From d06990469ff4e155d9773b5aa12e64345c8d41d3 Mon Sep 17 00:00:00 2001 From: Ma Mingrui Date: Thu, 7 May 2026 20:18:52 +0800 Subject: [PATCH 01/13] net/macb:Fix inconsistent lock usage between softirq and task context The tx_ptr_lock is used in both softirq context (via macb_tx_complete in NAPI poll) and task context (via macb_mac_link_down in workqueue). Currently, macb_tx_complete() uses spin_lock(), which is appropriate for softirq context, while macb_mac_link_down() also uses spin_lock(), which does **not** disable softirqs. This mismatch causes a potential deadlock if softirq interrupts the task context while it holds the lock, as detected by lockdep: WARNING: inconsistent lock state inconsistent {IN-SOFTIRQ-W} -> {SOFTIRQ-ON-W} usage. ... *** DEADLOCK *** Fix this by replacing spin_lock/unlock with spin_lock_bh/unlock_bh in macb_mac_link_down(), ensuring softirqs are disabled before acquiring the lock in task context. Signed-off-by: Ma Mingrui Signed-off-by: Li Wencheng Signed-off-by: Wang Yinfeng --- drivers/net/ethernet/cadence/macb_main.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/net/ethernet/cadence/macb_main.c b/drivers/net/ethernet/cadence/macb_main.c index fcfb5cad6c435..e134c4e7cc478 100644 --- a/drivers/net/ethernet/cadence/macb_main.c +++ b/drivers/net/ethernet/cadence/macb_main.c @@ -767,7 +767,7 @@ static void macb_mac_link_down(struct phylink_config *config, unsigned int mode, /* Tx clean */ for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { - spin_lock(&queue->tx_ptr_lock); + spin_lock_bh(&queue->tx_ptr_lock); for (i = 0; i < bp->tx_ring_size; i++) { tx_skb = macb_tx_skb(queue, i); /* free unsent skb buffers */ @@ -778,7 +778,7 @@ static void macb_mac_link_down(struct phylink_config *config, unsigned int mode, macb_set_addr(bp, tx_desc, 0); tx_desc->ctrl &= ~MACB_BIT(TX_USED); } - spin_unlock(&queue->tx_ptr_lock); + spin_unlock_bh(&queue->tx_ptr_lock); } netif_tx_stop_all_queues(ndev); From 93f88c257f3af4eac46df01f58ac147351d99c44 Mon Sep 17 00:00:00 2001 From: Ma Mingrui Date: Thu, 7 May 2026 20:31:58 +0800 Subject: [PATCH 02/13] net: phytmac: use memcpy_{from,to}io() for MMIO buffer access The driver was using plain memcpy() to copy data to/from __iomem msg_regs. When the MMIO address is not naturally aligned (e.g. ...904a), this triggers an alignment fault. With KASAN enabled, this fault is reliably reproducible during driver probe: FSC = 0x21: alignment fault pc : __memcpy+0x58/0x230 phytmac_v2_get_mac_addr+0xdc/0x180 [phytmac] Root cause: using memcpy() on MMIO regions both violates I/O access semantics and can lead to unaligned accesses. Fix it by replacing memcpy() with memcpy_fromio() when reading messages from msg ring and memcpy_toio() when writing messages to the msg ring. This avoids unaligned wide accesses and ensures proper MMIO semantics. Signed-off-by: Ma Mingrui Signed-off-by: Li Tongfeng Signed-off-by: Li Wencheng Signed-off-by: Wang Yinfeng --- drivers/net/ethernet/phytium/phytmac.h | 2 +- drivers/net/ethernet/phytium/phytmac_v2.c | 25 ++++++++++++----------- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/drivers/net/ethernet/phytium/phytmac.h b/drivers/net/ethernet/phytium/phytmac.h index c3feca3d9611a..e61e6a10efe21 100644 --- a/drivers/net/ethernet/phytium/phytmac.h +++ b/drivers/net/ethernet/phytium/phytmac.h @@ -17,7 +17,7 @@ #define PHYTMAC_PCI_DRV_NAME "phytmac_pci" #define PHYTMAC_PLAT_DRV_NAME "phytmac_platform" #define PHYTMAC_DRV_DESC "PHYTIUM Ethernet Driver" -#define PHYTMAC_DRIVER_VERSION "1.0.50" +#define PHYTMAC_DRIVER_VERSION "1.0.51" #define PHYTMAC_DEFAULT_MSG_ENABLE \ (NETIF_MSG_DRV | \ NETIF_MSG_PROBE | \ diff --git a/drivers/net/ethernet/phytium/phytmac_v2.c b/drivers/net/ethernet/phytium/phytmac_v2.c index 4dd3ddd18e714..518f606fa17de 100644 --- a/drivers/net/ethernet/phytium/phytmac_v2.c +++ b/drivers/net/ethernet/phytium/phytmac_v2.c @@ -68,7 +68,7 @@ static int phytmac_v2_msg_send(struct phytmac *pdata, u16 cmd_id, msg.cmd_type, msg.cmd_subid, msg.status0, len, tx_tail); } - memcpy(pdata->msg_regs + PHYTMAC_MSG(tx_tail), &msg, sizeof(msg)); + memcpy_toio(pdata->msg_regs + PHYTMAC_MSG(tx_tail), &msg, sizeof(msg)); tx_tail = phytmac_v2_tx_ring_wrap(pdata, ++tx_tail); PHYTMAC_WRITE(pdata, PHYTMAC_TX_MSG_TAIL, tx_tail | PHYTMAC_BIT(TX_MSG_INT)); pdata->msg_ring.tx_msg_wr_tail = tx_tail; @@ -86,8 +86,9 @@ static int phytmac_v2_msg_send(struct phytmac *pdata, u16 cmd_id, } } - memcpy(&msg_rx, pdata->msg_regs + PHYTMAC_MSG(pdata->msg_ring.tx_msg_rd_tail), - PHYTMAC_MSG_HDR_LEN); + memcpy_fromio(&msg_rx, pdata->msg_regs + + PHYTMAC_MSG(pdata->msg_ring.tx_msg_rd_tail), + PHYTMAC_MSG_HDR_LEN); if (!(msg_rx.status0 & PHYTMAC_CMD_PRC_SUCCESS)) { netdev_err(pdata->ndev, "Msg process error, cmdid:%d, subid:%d, status0:%d, tail:%d", msg.cmd_type, msg.cmd_subid, msg.status0, tx_tail); @@ -142,8 +143,8 @@ static int phytmac_v2_get_mac_addr(struct phytmac *pdata, u8 *addr) phytmac_v2_msg_send(pdata, cmd_id, cmd_subid, NULL, 0, 1); index = phytmac_v2_tx_ring_wrap(pdata, pdata->msg_ring.tx_msg_rd_tail); - memcpy(¶, pdata->msg_regs + PHYTMAC_MSG(index) + PHYTMAC_MSG_HDR_LEN, - sizeof(struct phytmac_mac)); + memcpy_fromio(¶, pdata->msg_regs + PHYTMAC_MSG(index) + PHYTMAC_MSG_HDR_LEN, + sizeof(struct phytmac_mac)); addr[0] = para.addrl & 0xff; addr[1] = (para.addrl >> 8) & 0xff; @@ -368,8 +369,8 @@ static int phytmac_v2_get_feature_all(struct phytmac *pdata) cmd_subid = PHYTMAC_MSG_CMD_GET_CAPS; phytmac_v2_msg_send(pdata, cmd_id, cmd_subid, NULL, 0, 1); index = phytmac_v2_tx_ring_wrap(pdata, pdata->msg_ring.tx_msg_rd_tail); - memcpy(¶, pdata->msg_regs + PHYTMAC_MSG(index) + PHYTMAC_MSG_HDR_LEN, - sizeof(struct phytmac_feature)); + memcpy_fromio(¶, pdata->msg_regs + PHYTMAC_MSG(index) + PHYTMAC_MSG_HDR_LEN, + sizeof(struct phytmac_feature)); pdata->queues_max_num = para.queue_num; if (para.dma_addr_width) @@ -409,15 +410,15 @@ static void phytmac_v2_get_regs(struct phytmac *pdata, u32 *reg_buff) msg.cnt = 0; phytmac_v2_msg_send(pdata, cmd_id, cmd_subid, (void *)(&msg), sizeof(msg), 1); index = phytmac_v2_tx_ring_wrap(pdata, pdata->msg_ring.tx_msg_rd_tail); - memcpy(reg_buff, pdata->msg_regs + PHYTMAC_MSG(index) + PHYTMAC_MSG_HDR_LEN, - PHYTMAC_MSG_PARA_LEN); + memcpy_fromio(reg_buff, pdata->msg_regs + PHYTMAC_MSG(index) + PHYTMAC_MSG_HDR_LEN, + PHYTMAC_MSG_PARA_LEN); msg.cnt = 1; phytmac_v2_msg_send(pdata, cmd_id, cmd_subid, (void *)(&msg), sizeof(msg), 1); index = phytmac_v2_tx_ring_wrap(pdata, pdata->msg_ring.tx_msg_rd_tail); - memcpy(reg_buff + PHYTMAC_MSG_PARA_LEN / sizeof(u32), pdata->msg_regs + - PHYTMAC_MSG(index) + PHYTMAC_MSG_HDR_LEN, - PHYTMAC_ETHTOOLD_REGS_LEN - PHYTMAC_MSG_PARA_LEN); + memcpy_fromio(reg_buff + PHYTMAC_MSG_PARA_LEN / sizeof(u32), pdata->msg_regs + + PHYTMAC_MSG(index) + PHYTMAC_MSG_HDR_LEN, + PHYTMAC_ETHTOOLD_REGS_LEN - PHYTMAC_MSG_PARA_LEN); } static void phytmac_v2_get_hw_stats(struct phytmac *pdata) From c3a3dd395ee5e281776dc44bd2078ab98996f38e Mon Sep 17 00:00:00 2001 From: Ma Mingrui Date: Thu, 7 May 2026 20:37:04 +0800 Subject: [PATCH 03/13] net/phytmac: Adjust the position of the MDC setting On certain machines, the phy_id cannot be read. Adjusting the MDC setting to occur before registering the mdiobus in enable_mdio_control resolves this issue. Signed-off-by: Ma Mingrui Signed-off-by: Ouyang Chenxing Signed-off-by: Wang Yinfeng --- drivers/net/ethernet/phytium/phytmac.h | 2 +- drivers/net/ethernet/phytium/phytmac_v2.c | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/net/ethernet/phytium/phytmac.h b/drivers/net/ethernet/phytium/phytmac.h index e61e6a10efe21..0abfe32a2b68b 100644 --- a/drivers/net/ethernet/phytium/phytmac.h +++ b/drivers/net/ethernet/phytium/phytmac.h @@ -17,7 +17,7 @@ #define PHYTMAC_PCI_DRV_NAME "phytmac_pci" #define PHYTMAC_PLAT_DRV_NAME "phytmac_platform" #define PHYTMAC_DRV_DESC "PHYTIUM Ethernet Driver" -#define PHYTMAC_DRIVER_VERSION "1.0.51" +#define PHYTMAC_DRIVER_VERSION "1.0.52" #define PHYTMAC_DEFAULT_MSG_ENABLE \ (NETIF_MSG_DRV | \ NETIF_MSG_PROBE | \ diff --git a/drivers/net/ethernet/phytium/phytmac_v2.c b/drivers/net/ethernet/phytium/phytmac_v2.c index 518f606fa17de..8dd3ecbec098d 100644 --- a/drivers/net/ethernet/phytium/phytmac_v2.c +++ b/drivers/net/ethernet/phytium/phytmac_v2.c @@ -195,7 +195,6 @@ static int phytmac_v2_init_hw(struct phytmac *pdata) struct phytmac_dma_info dma; struct phytmac_eth_info eth; u32 ptrconfig = 0; - u8 mdc; if (pdata->capacities & PHYTMAC_CAPS_TAILPTR) ptrconfig |= PHYTMAC_BIT(TXTAIL_EN); @@ -258,10 +257,6 @@ static int phytmac_v2_init_hw(struct phytmac *pdata) dma.hw_dma_cap |= HW_DMA_CAP_DDW128; phytmac_v2_msg_send(pdata, cmd_id, cmd_subid, (void *)&dma, sizeof(dma), 0); - cmd_subid = PHYTMAC_MSG_CMD_SET_MDC; - mdc = PHYTMAC_CLK_DIV96; - phytmac_v2_msg_send(pdata, cmd_id, cmd_subid, (void *)(&mdc), sizeof(mdc), 0); - memset(ð, 0, sizeof(eth)); cmd_subid = PHYTMAC_MSG_CMD_SET_ETH_MATCH; eth.index = 0; @@ -706,8 +701,13 @@ static int phytmac_v2_enable_txcsum(struct phytmac *pdata, int enable) static int phytmac_v2_enable_mdio(struct phytmac *pdata, int enable) { u16 cmd_id, cmd_subid; + u8 mdc; cmd_id = PHYTMAC_MSG_CMD_SET; + cmd_subid = PHYTMAC_MSG_CMD_SET_MDC; + mdc = PHYTMAC_CLK_DIV96; + phytmac_v2_msg_send(pdata, cmd_id, cmd_subid, (void *)(&mdc), sizeof(mdc), 0); + if (enable) cmd_subid = PHYTMAC_MSG_CMD_SET_ENABLE_MDIO; else From 2362efefa64a9f6be6a7f802083faa68dfef8cda Mon Sep 17 00:00:00 2001 From: Ma Mingrui Date: Thu, 7 May 2026 20:39:34 +0800 Subject: [PATCH 04/13] Resolve warnings when using jumbo frames Change the NULL return value to ERR_PTR(-EAGAIN) and handle it properly in the upper-layer function to break the processing loop, thereby avoiding warnings. Signed-off-by: Ma Mingrui Signed-off-by: Ouyang Chenxing Signed-off-by: Wang Yinfeng --- drivers/net/ethernet/phytium/phytmac.h | 2 +- drivers/net/ethernet/phytium/phytmac_main.c | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/drivers/net/ethernet/phytium/phytmac.h b/drivers/net/ethernet/phytium/phytmac.h index 0abfe32a2b68b..60ba6cf3c2257 100644 --- a/drivers/net/ethernet/phytium/phytmac.h +++ b/drivers/net/ethernet/phytium/phytmac.h @@ -17,7 +17,7 @@ #define PHYTMAC_PCI_DRV_NAME "phytmac_pci" #define PHYTMAC_PLAT_DRV_NAME "phytmac_platform" #define PHYTMAC_DRV_DESC "PHYTIUM Ethernet Driver" -#define PHYTMAC_DRIVER_VERSION "1.0.52" +#define PHYTMAC_DRIVER_VERSION "1.0.53" #define PHYTMAC_DEFAULT_MSG_ENABLE \ (NETIF_MSG_DRV | \ NETIF_MSG_PROBE | \ diff --git a/drivers/net/ethernet/phytium/phytmac_main.c b/drivers/net/ethernet/phytium/phytmac_main.c index ece14f0a7b47e..77a4dc5930815 100644 --- a/drivers/net/ethernet/phytium/phytmac_main.c +++ b/drivers/net/ethernet/phytium/phytmac_main.c @@ -1131,7 +1131,7 @@ static struct sk_buff *phytmac_rx_mbuffer(struct phytmac_queue *queue) for (rx_tail = queue->rx_tail; ; rx_tail++) { desc = phytmac_get_rx_desc(queue, rx_tail); if (!hw_if->rx_complete(desc)) - return NULL; + return ERR_PTR(-EAGAIN); if (hw_if->rx_pkt_start(desc)) { if (first_frag != -1) @@ -1218,6 +1218,9 @@ static int phytmac_rx(struct phytmac_queue *queue, struct napi_struct *napi, if (pdata->xdp_prog) netdev_warn(pdata->ndev, "xdp does not support multiple buffers!!\n"); skb = phytmac_rx_mbuffer(queue); + if (PTR_ERR(skb) == -EAGAIN) { + break; + } } if (!skb) { From c649fe423de14138820b557b0bfa18205b68a76d Mon Sep 17 00:00:00 2001 From: Ma Mingrui Date: Thu, 7 May 2026 20:41:40 +0800 Subject: [PATCH 05/13] net/phytmac: Support ipv6 TSO feature Add support for NETIF_F_TSO6 in features check and default config to enable IPv6 TSO functionality. Signed-off-by: Ma Mingrui Signed-off-by: Ouyang Chenxing Signed-off-by: Wang Yinfeng --- drivers/net/ethernet/phytium/phytmac.h | 2 +- drivers/net/ethernet/phytium/phytmac_main.c | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/net/ethernet/phytium/phytmac.h b/drivers/net/ethernet/phytium/phytmac.h index 60ba6cf3c2257..2fd43be183c95 100644 --- a/drivers/net/ethernet/phytium/phytmac.h +++ b/drivers/net/ethernet/phytium/phytmac.h @@ -17,7 +17,7 @@ #define PHYTMAC_PCI_DRV_NAME "phytmac_pci" #define PHYTMAC_PLAT_DRV_NAME "phytmac_platform" #define PHYTMAC_DRV_DESC "PHYTIUM Ethernet Driver" -#define PHYTMAC_DRIVER_VERSION "1.0.53" +#define PHYTMAC_DRIVER_VERSION "1.0.54" #define PHYTMAC_DEFAULT_MSG_ENABLE \ (NETIF_MSG_DRV | \ NETIF_MSG_PROBE | \ diff --git a/drivers/net/ethernet/phytium/phytmac_main.c b/drivers/net/ethernet/phytium/phytmac_main.c index 77a4dc5930815..1f4e9a2b97ee4 100644 --- a/drivers/net/ethernet/phytium/phytmac_main.c +++ b/drivers/net/ethernet/phytium/phytmac_main.c @@ -2368,7 +2368,7 @@ static netdev_features_t phytmac_features_check(struct sk_buff *skb, hdrlen = skb_transport_offset(skb); if (!IS_ALIGNED(skb_headlen(skb) - hdrlen, PHYTMAC_TX_LEN_ALIGN)) - return features & ~NETIF_F_TSO; + return features & ~(NETIF_F_TSO | NETIF_F_TSO6); nr_frags = skb_shinfo(skb)->nr_frags; /* No need to check last fragment */ @@ -2377,7 +2377,7 @@ static netdev_features_t phytmac_features_check(struct sk_buff *skb, const skb_frag_t *frag = &skb_shinfo(skb)->frags[f]; if (!IS_ALIGNED(skb_frag_size(frag), PHYTMAC_TX_LEN_ALIGN)) - return features & ~NETIF_F_TSO; + return features & ~(NETIF_F_TSO | NETIF_F_TSO6); } return features; } @@ -2667,7 +2667,7 @@ void phytmac_default_config(struct phytmac *pdata) ndev->hw_features = NETIF_F_SG; if (pdata->capacities & PHYTMAC_CAPS_LSO) - ndev->hw_features |= NETIF_F_TSO; + ndev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6; if (pdata->use_ncsi) { ndev->hw_features &= ~(NETIF_F_HW_CSUM | NETIF_F_RXCSUM); From ee22999a9846224ab1f8a96d8022c1216ae40167 Mon Sep 17 00:00:00 2001 From: Ma Mingrui Date: Thu, 7 May 2026 20:43:59 +0800 Subject: [PATCH 06/13] net/macb: Support ipv6 TSO feature Add NETIF_F_TSO6 support to MACB_NETIF_LSO to enable IPv6 TSO functionality. Signed-off-by: Ma Mingrui Signed-off-by: Ouyang Chenxing Signed-off-by: Wang Yinfeng --- drivers/net/ethernet/cadence/macb_main.c | 2 +- drivers/net/ethernet/cadence/macb_main.c.orig | 6258 +++++++++++++++++ 2 files changed, 6259 insertions(+), 1 deletion(-) create mode 100644 drivers/net/ethernet/cadence/macb_main.c.orig diff --git a/drivers/net/ethernet/cadence/macb_main.c b/drivers/net/ethernet/cadence/macb_main.c index e134c4e7cc478..0b1c9610d4db5 100644 --- a/drivers/net/ethernet/cadence/macb_main.c +++ b/drivers/net/ethernet/cadence/macb_main.c @@ -89,7 +89,7 @@ struct sifive_fu540_macb_mgmt { #define GEM_MAX_TX_LEN (unsigned int)(0x3FC0) #define GEM_MTU_MIN_SIZE ETH_MIN_MTU -#define MACB_NETIF_LSO NETIF_F_TSO +#define MACB_NETIF_LSO (NETIF_F_TSO | NETIF_F_TSO6) #define MACB_WOL_HAS_MAGIC_PACKET (0x1 << 0) #define MACB_WOL_ENABLED (0x1 << 1) diff --git a/drivers/net/ethernet/cadence/macb_main.c.orig b/drivers/net/ethernet/cadence/macb_main.c.orig new file mode 100644 index 0000000000000..e134c4e7cc478 --- /dev/null +++ b/drivers/net/ethernet/cadence/macb_main.c.orig @@ -0,0 +1,6258 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Cadence MACB/GEM Ethernet Controller driver + * + * Copyright (C) 2004-2006 Atmel Corporation + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "macb.h" + +/* This structure is only used for MACB on SiFive FU540 devices */ +struct sifive_fu540_macb_mgmt { + void __iomem *reg; + unsigned long rate; + struct clk_hw hw; +}; + +#define MAX_RING_ADDR_ALLOC_TIMES 3 +#define RING_ADDR_INTERVAL 128 + +#define MACB_RX_BUFFER_SIZE 128 +#define RX_BUFFER_MULTIPLE 64 /* bytes */ + +#define DEFAULT_RX_RING_SIZE 512 /* must be power of 2 */ +#define MIN_RX_RING_SIZE 64 +#define MAX_RX_RING_SIZE 8192 +#define RX_RING_BYTES(bp) (macb_dma_desc_get_size(bp) \ + * (bp)->rx_ring_size) + +#define DEFAULT_TX_RING_SIZE 512 /* must be power of 2 */ +#define MIN_TX_RING_SIZE 64 +#define MAX_TX_RING_SIZE 4096 +#define TX_RING_BYTES(bp) (macb_dma_desc_get_size(bp) \ + * (bp)->tx_ring_size) + +/* level of occupied TX descriptors under which we wake up TX process */ +#define MACB_TX_WAKEUP_THRESH(bp) (3 * (bp)->tx_ring_size / 4) + +#define MACB_RX_INT_FLAGS (MACB_BIT(RCOMP) | MACB_BIT(ISR_ROVR)) +#define MACB_TX_ERR_FLAGS (MACB_BIT(ISR_TUND) \ + | MACB_BIT(ISR_RLE) \ + | MACB_BIT(TXERR)) +#define MACB_TX_INT_FLAGS (MACB_TX_ERR_FLAGS | MACB_BIT(TCOMP) \ + | MACB_BIT(TXUBR)) + +/* Max length of transmit frame must be a multiple of 8 bytes */ +#define MACB_TX_LEN_ALIGN 8 +#define MACB_MAX_TX_LEN ((unsigned int)((1 << MACB_TX_FRMLEN_SIZE) - 1) & ~((unsigned int)(MACB_TX_LEN_ALIGN - 1))) +/* Limit maximum TX length as per Cadence TSO errata. This is to avoid a + * false amba_error in TX path from the DMA assuming there is not enough + * space in the SRAM (16KB) even when there is. + */ +#define GEM_MAX_TX_LEN (unsigned int)(0x3FC0) + +#define GEM_MTU_MIN_SIZE ETH_MIN_MTU +#define MACB_NETIF_LSO NETIF_F_TSO + +#define MACB_WOL_HAS_MAGIC_PACKET (0x1 << 0) +#define MACB_WOL_ENABLED (0x1 << 1) + +#define HS_SPEED_100M 0 +#define HS_SPEED_1000M 1 +#define HS_SPEED_2500M 2 +#define HS_SPEED_5000M 3 +#define HS_SPEED_10000M 4 +#define MACB_SERDES_RATE_5G 0 +#define MACB_SERDES_RATE_10G 1 + +/* Graceful stop timeouts in us. We should allow up to + * 1 frame time (10 Mbits/s, full-duplex, ignoring collisions) + */ +#define MACB_HALT_TIMEOUT 14000 +#define MACB_PM_TIMEOUT 100 /* ms */ + +#define MACB_MDIO_TIMEOUT 1000000 /* in usecs */ + +static void macb_tx_unmap(struct macb *bp, + struct macb_tx_skb *tx_skb, + int budget); +static void macb_set_addr(struct macb *bp, struct macb_dma_desc *desc, + dma_addr_t addr); + +/* DMA buffer descriptor might be different size + * depends on hardware configuration: + * + * 1. dma address width 32 bits: + * word 1: 32 bit address of Data Buffer + * word 2: control + * + * 2. dma address width 64 bits: + * word 1: 32 bit address of Data Buffer + * word 2: control + * word 3: upper 32 bit address of Data Buffer + * word 4: unused + * + * 3. dma address width 32 bits with hardware timestamping: + * word 1: 32 bit address of Data Buffer + * word 2: control + * word 3: timestamp word 1 + * word 4: timestamp word 2 + * + * 4. dma address width 64 bits with hardware timestamping: + * word 1: 32 bit address of Data Buffer + * word 2: control + * word 3: upper 32 bit address of Data Buffer + * word 4: unused + * word 5: timestamp word 1 + * word 6: timestamp word 2 + */ +static unsigned int macb_dma_desc_get_size(struct macb *bp) +{ +#ifdef MACB_EXT_DESC + unsigned int desc_size; + + switch (bp->hw_dma_cap) { + case HW_DMA_CAP_64B: + desc_size = sizeof(struct macb_dma_desc) + + sizeof(struct macb_dma_desc_64); + break; + case HW_DMA_CAP_PTP: + desc_size = sizeof(struct macb_dma_desc) + + sizeof(struct macb_dma_desc_ptp); + break; + case HW_DMA_CAP_64B_PTP: + desc_size = sizeof(struct macb_dma_desc) + + sizeof(struct macb_dma_desc_64) + + sizeof(struct macb_dma_desc_ptp); + break; + default: + desc_size = sizeof(struct macb_dma_desc); + } + return desc_size; +#endif + return sizeof(struct macb_dma_desc); +} + +static unsigned int macb_adj_dma_desc_idx(struct macb *bp, unsigned int desc_idx) +{ +#ifdef MACB_EXT_DESC + switch (bp->hw_dma_cap) { + case HW_DMA_CAP_64B: + case HW_DMA_CAP_PTP: + desc_idx <<= 1; + break; + case HW_DMA_CAP_64B_PTP: + desc_idx *= 3; + break; + default: + break; + } +#endif + return desc_idx; +} + +#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT +static struct macb_dma_desc_64 *macb_64b_desc(struct macb *bp, struct macb_dma_desc *desc) +{ + return (struct macb_dma_desc_64 *)((void *)desc + + sizeof(struct macb_dma_desc)); +} +#endif + +/* Ring buffer accessors */ +static unsigned int macb_tx_ring_wrap(struct macb *bp, unsigned int index) +{ + return index & (bp->tx_ring_size - 1); +} + +static struct macb_dma_desc *macb_tx_desc(struct macb_queue *queue, + unsigned int index) +{ + index = macb_tx_ring_wrap(queue->bp, index); + index = macb_adj_dma_desc_idx(queue->bp, index); + return &queue->tx_ring[index]; +} + +static struct macb_tx_skb *macb_tx_skb(struct macb_queue *queue, + unsigned int index) +{ + return &queue->tx_skb[macb_tx_ring_wrap(queue->bp, index)]; +} + +static dma_addr_t macb_tx_dma(struct macb_queue *queue, unsigned int index) +{ + dma_addr_t offset; + + offset = macb_tx_ring_wrap(queue->bp, index) * + macb_dma_desc_get_size(queue->bp); + + return queue->tx_ring_dma + offset; +} + +static unsigned int macb_rx_ring_wrap(struct macb *bp, unsigned int index) +{ + return index & (bp->rx_ring_size - 1); +} + +static struct macb_dma_desc *macb_rx_desc(struct macb_queue *queue, unsigned int index) +{ + index = macb_rx_ring_wrap(queue->bp, index); + index = macb_adj_dma_desc_idx(queue->bp, index); + return &queue->rx_ring[index]; +} + +static void *macb_rx_buffer(struct macb_queue *queue, unsigned int index) +{ + return queue->rx_buffers + queue->bp->rx_buffer_size * + macb_rx_ring_wrap(queue->bp, index); +} + +/* I/O accessors */ +static u32 hw_readl_native(struct macb *bp, int offset) +{ + return __raw_readl(bp->regs + offset); +} + +static void hw_writel_native(struct macb *bp, int offset, u32 value) +{ + __raw_writel(value, bp->regs + offset); +} + +static u32 hw_readl(struct macb *bp, int offset) +{ + return readl_relaxed(bp->regs + offset); +} + +static void hw_writel(struct macb *bp, int offset, u32 value) +{ + writel_relaxed(value, bp->regs + offset); +} + +/* Find the CPU endianness by using the loopback bit of NCR register. When the + * CPU is in big endian we need to program swapped mode for management + * descriptor access. + */ +static bool hw_is_native_io(void __iomem *addr) +{ + u32 value = MACB_BIT(LLB); + + __raw_writel(value, addr + MACB_NCR); + value = __raw_readl(addr + MACB_NCR); + + /* Write 0 back to disable everything */ + __raw_writel(0, addr + MACB_NCR); + + return value == MACB_BIT(LLB); +} + +static bool hw_is_gem(void __iomem *addr, bool native_io) +{ + u32 id; + + if (native_io) + id = __raw_readl(addr + MACB_MID); + else + id = readl_relaxed(addr + MACB_MID); + + return MACB_BFEXT(IDNUM, id) >= 0x2; +} + +static void macb_set_hwaddr(struct macb *bp) +{ + u32 bottom; + u16 top; + + bottom = get_unaligned_le32(bp->dev->dev_addr); + macb_or_gem_writel(bp, SA1B, bottom); + top = get_unaligned_le16(bp->dev->dev_addr + 4); + macb_or_gem_writel(bp, SA1T, top); + + if (gem_has_ptp(bp)) { + gem_writel(bp, RXPTPUNI, bottom); + gem_writel(bp, TXPTPUNI, bottom); + } + + /* Clear unused address register sets */ + macb_or_gem_writel(bp, SA2B, 0); + macb_or_gem_writel(bp, SA2T, 0); + macb_or_gem_writel(bp, SA3B, 0); + macb_or_gem_writel(bp, SA3T, 0); + macb_or_gem_writel(bp, SA4B, 0); + macb_or_gem_writel(bp, SA4T, 0); +} + +static void macb_get_hwaddr(struct macb *bp) +{ + u32 bottom; + u16 top; + u8 addr[6]; + int i; + + /* Check all 4 address register for valid address */ + for (i = 0; i < 4; i++) { + bottom = macb_or_gem_readl(bp, SA1B + i * 8); + top = macb_or_gem_readl(bp, SA1T + i * 8); + + addr[0] = bottom & 0xff; + addr[1] = (bottom >> 8) & 0xff; + addr[2] = (bottom >> 16) & 0xff; + addr[3] = (bottom >> 24) & 0xff; + addr[4] = top & 0xff; + addr[5] = (top >> 8) & 0xff; + + if (is_valid_ether_addr(addr)) { + eth_hw_addr_set(bp->dev, addr); + return; + } + } + + dev_info(&bp->pdev->dev, "invalid hw address, using random\n"); + eth_hw_addr_random(bp->dev); +} + +static int macb_mdio_wait_for_idle(struct macb *bp) +{ + u32 val; + + return readx_poll_timeout(MACB_READ_NSR, bp, val, val & MACB_BIT(IDLE), + 1, MACB_MDIO_TIMEOUT); +} + +static int macb_mdio_read_c22(struct mii_bus *bus, int mii_id, int regnum) +{ + struct macb *bp = bus->priv; + int status; + + status = pm_runtime_resume_and_get(&bp->pdev->dev); + if (status < 0) + goto mdio_pm_exit; + + status = macb_mdio_wait_for_idle(bp); + if (status < 0) + goto mdio_read_exit; + + macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C22_SOF) + | MACB_BF(RW, MACB_MAN_C22_READ) + | MACB_BF(PHYA, mii_id) + | MACB_BF(REGA, regnum) + | MACB_BF(CODE, MACB_MAN_C22_CODE))); + + status = macb_mdio_wait_for_idle(bp); + if (status < 0) + goto mdio_read_exit; + + status = MACB_BFEXT(DATA, macb_readl(bp, MAN)); + +mdio_read_exit: + pm_runtime_mark_last_busy(&bp->pdev->dev); + pm_runtime_put_autosuspend(&bp->pdev->dev); +mdio_pm_exit: + return status; +} + +static int macb_mdio_read_c45(struct mii_bus *bus, int mii_id, int devad, + int regnum) +{ + struct macb *bp = bus->priv; + int status; + + status = pm_runtime_get_sync(&bp->pdev->dev); + if (status < 0) { + pm_runtime_put_noidle(&bp->pdev->dev); + goto mdio_pm_exit; + } + + status = macb_mdio_wait_for_idle(bp); + if (status < 0) + goto mdio_read_exit; + + macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C45_SOF) + | MACB_BF(RW, MACB_MAN_C45_ADDR) + | MACB_BF(PHYA, mii_id) + | MACB_BF(REGA, devad & 0x1F) + | MACB_BF(DATA, regnum & 0xFFFF) + | MACB_BF(CODE, MACB_MAN_C45_CODE))); + + status = macb_mdio_wait_for_idle(bp); + if (status < 0) + goto mdio_read_exit; + + macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C45_SOF) + | MACB_BF(RW, MACB_MAN_C45_READ) + | MACB_BF(PHYA, mii_id) + | MACB_BF(REGA, devad & 0x1F) + | MACB_BF(CODE, MACB_MAN_C45_CODE))); + + status = macb_mdio_wait_for_idle(bp); + if (status < 0) + goto mdio_read_exit; + + status = MACB_BFEXT(DATA, macb_readl(bp, MAN)); + +mdio_read_exit: + pm_runtime_mark_last_busy(&bp->pdev->dev); + pm_runtime_put_autosuspend(&bp->pdev->dev); +mdio_pm_exit: + return status; +} + +static int macb_mdio_write_c22(struct mii_bus *bus, int mii_id, int regnum, + u16 value) +{ + struct macb *bp = bus->priv; + int status; + + status = pm_runtime_resume_and_get(&bp->pdev->dev); + if (status < 0) + goto mdio_pm_exit; + + status = macb_mdio_wait_for_idle(bp); + if (status < 0) + goto mdio_write_exit; + + macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C22_SOF) + | MACB_BF(RW, MACB_MAN_C22_WRITE) + | MACB_BF(PHYA, mii_id) + | MACB_BF(REGA, regnum) + | MACB_BF(CODE, MACB_MAN_C22_CODE) + | MACB_BF(DATA, value))); + + status = macb_mdio_wait_for_idle(bp); + if (status < 0) + goto mdio_write_exit; + +mdio_write_exit: + pm_runtime_mark_last_busy(&bp->pdev->dev); + pm_runtime_put_autosuspend(&bp->pdev->dev); +mdio_pm_exit: + return status; +} + +static int macb_mdio_write_c45(struct mii_bus *bus, int mii_id, + int devad, int regnum, + u16 value) +{ + struct macb *bp = bus->priv; + int status; + + status = pm_runtime_get_sync(&bp->pdev->dev); + if (status < 0) { + pm_runtime_put_noidle(&bp->pdev->dev); + goto mdio_pm_exit; + } + + status = macb_mdio_wait_for_idle(bp); + if (status < 0) + goto mdio_write_exit; + + macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C45_SOF) + | MACB_BF(RW, MACB_MAN_C45_ADDR) + | MACB_BF(PHYA, mii_id) + | MACB_BF(REGA, devad & 0x1F) + | MACB_BF(DATA, regnum & 0xFFFF) + | MACB_BF(CODE, MACB_MAN_C45_CODE))); + + status = macb_mdio_wait_for_idle(bp); + if (status < 0) + goto mdio_write_exit; + + macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C45_SOF) + | MACB_BF(RW, MACB_MAN_C45_WRITE) + | MACB_BF(PHYA, mii_id) + | MACB_BF(REGA, devad & 0x1F) + | MACB_BF(CODE, MACB_MAN_C45_CODE) + | MACB_BF(DATA, value))); + + status = macb_mdio_wait_for_idle(bp); + if (status < 0) + goto mdio_write_exit; + +mdio_write_exit: + pm_runtime_mark_last_busy(&bp->pdev->dev); + pm_runtime_put_autosuspend(&bp->pdev->dev); +mdio_pm_exit: + return status; +} + +static void macb_init_buffers(struct macb *bp) +{ + struct macb_queue *queue; + unsigned int q; + + for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { + queue_writel(queue, RBQP, lower_32_bits(queue->rx_ring_dma)); +#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT + if (bp->hw_dma_cap & HW_DMA_CAP_64B) + queue_writel(queue, RBQPH, + upper_32_bits(queue->rx_ring_dma)); +#endif + queue_writel(queue, TBQP, lower_32_bits(queue->tx_ring_dma)); +#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT + if (bp->hw_dma_cap & HW_DMA_CAP_64B) + queue_writel(queue, TBQPH, + upper_32_bits(queue->tx_ring_dma)); +#endif + } +} + +/** + * macb_set_tx_clk() - Set a clock to a new frequency + * @bp: pointer to struct macb + * @speed: New frequency in Hz + */ +static void macb_set_tx_clk(struct macb *bp, int speed) +{ + long ferr, rate, rate_rounded; + + if (!bp->tx_clk || (bp->caps & MACB_CAPS_CLK_HW_CHG)) + return; + + /* In case of MII the PHY is the clock master */ + if (bp->phy_interface == PHY_INTERFACE_MODE_MII) + return; + + switch (speed) { + case SPEED_10: + rate = 2500000; + break; + case SPEED_100: + rate = 25000000; + break; + case SPEED_1000: + rate = 125000000; + break; + default: + return; + } + + rate_rounded = clk_round_rate(bp->tx_clk, rate); + if (rate_rounded < 0) + return; + + /* RGMII allows 50 ppm frequency error. Test and warn if this limit + * is not satisfied. + */ + ferr = abs(rate_rounded - rate); + ferr = DIV_ROUND_UP(ferr, rate / 100000); + if (ferr > 5) + netdev_warn(bp->dev, + "unable to generate target frequency: %ld Hz\n", + rate); + + if (clk_set_rate(bp->tx_clk, rate_rounded)) + netdev_err(bp->dev, "adjusting tx_clk failed.\n"); +} + +static void macb_usx_pcs_link_up(struct phylink_pcs *pcs, unsigned int mode, + phy_interface_t interface, int speed, + int duplex) +{ + struct macb *bp = container_of(pcs, struct macb, phylink_usx_pcs); + u32 config; + + config = gem_readl(bp, USX_CONTROL); + if (speed == SPEED_10000) { + config = GEM_BFINS(SERDES_RATE, MACB_SERDES_RATE_10G, config); + config = GEM_BFINS(USX_CTRL_SPEED, HS_SPEED_10000M, config); + } else if (speed == SPEED_5000) { + config = GEM_BFINS(SERDES_RATE, MACB_SERDES_RATE_5G, config); + config = GEM_BFINS(USX_CTRL_SPEED, HS_SPEED_5000M, config); + } + + config &= ~(GEM_BIT(TX_SCR_BYPASS) | GEM_BIT(RX_SCR_BYPASS)); + /* reset */ + config &= ~(GEM_BIT(SIGNAL_OK) | GEM_BIT(TX_EN)); + config |= GEM_BIT(RX_SYNC_RESET); + + gem_writel(bp, USX_CONTROL, config); + + /* enable rx and tx */ + config &= ~(GEM_BIT(RX_SYNC_RESET)); + config |= GEM_BIT(SIGNAL_OK) | GEM_BIT(TX_EN); + + gem_writel(bp, USX_CONTROL, config); +} + +static void macb_usx_pcs_get_state(struct phylink_pcs *pcs, + struct phylink_link_state *state) +{ + struct macb *bp = container_of(pcs, struct macb, phylink_usx_pcs); + u32 val; + + if (state->interface == PHY_INTERFACE_MODE_5GBASER) + state->speed = SPEED_5000; + else if (state->interface == PHY_INTERFACE_MODE_10GBASER || + state->interface == PHY_INTERFACE_MODE_USXGMII) + state->speed = bp->speed; + + state->duplex = 1; + state->an_complete = 1; + + val = gem_readl(bp, USX_STATUS); + state->link = !!(val & GEM_BIT(USX_BLOCK_LOCK)); + val = gem_readl(bp, NCFGR); + if (val & GEM_BIT(PAE)) + state->pause = MLO_PAUSE_RX; +} + +static int macb_usx_pcs_config(struct phylink_pcs *pcs, + unsigned int neg_mode, + phy_interface_t interface, + const unsigned long *advertising, + bool permit_pause_to_mac) +{ + struct macb *bp = container_of(pcs, struct macb, phylink_usx_pcs); + + gem_writel(bp, USX_CONTROL, gem_readl(bp, USX_CONTROL) | + GEM_BIT(SIGNAL_OK) | GEM_BIT(TX_EN)); + + return 0; +} + +static void macb_pcs_get_state(struct phylink_pcs *pcs, + struct phylink_link_state *state) +{ + state->link = 0; +} + +static void macb_pcs_an_restart(struct phylink_pcs *pcs) +{ + /* Not supported */ +} + +static int macb_pcs_config(struct phylink_pcs *pcs, + unsigned int neg_mode, + phy_interface_t interface, + const unsigned long *advertising, + bool permit_pause_to_mac) +{ + return 0; +} + +static const struct phylink_pcs_ops macb_phylink_usx_pcs_ops = { + .pcs_get_state = macb_usx_pcs_get_state, + .pcs_config = macb_usx_pcs_config, + .pcs_link_up = macb_usx_pcs_link_up, +}; + +static const struct phylink_pcs_ops macb_phylink_pcs_ops = { + .pcs_get_state = macb_pcs_get_state, + .pcs_an_restart = macb_pcs_an_restart, + .pcs_config = macb_pcs_config, +}; + +static void macb_mac_config(struct phylink_config *config, unsigned int mode, + const struct phylink_link_state *state) +{ + struct net_device *ndev = to_net_dev(config->dev); + struct macb *bp = netdev_priv(ndev); + unsigned long flags; + u32 old_ctrl, ctrl; + u32 old_ncr, ncr; + + spin_lock_irqsave(&bp->lock, flags); + + old_ctrl = ctrl = macb_or_gem_readl(bp, NCFGR); + old_ncr = ncr = macb_or_gem_readl(bp, NCR); + + if (bp->caps & MACB_CAPS_MACB_IS_EMAC) { + if (state->interface == PHY_INTERFACE_MODE_RMII) + ctrl |= MACB_BIT(RM9200_RMII); + } else if (macb_is_gem(bp)) { + ctrl &= ~(GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL)); + ncr &= ~GEM_BIT(ENABLE_HS_MAC); + + if (state->interface == PHY_INTERFACE_MODE_SGMII || + state->interface == PHY_INTERFACE_MODE_2500BASEX) { + ctrl |= GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL); + } else if (state->interface == PHY_INTERFACE_MODE_10GBASER || + state->interface == PHY_INTERFACE_MODE_USXGMII || + state->interface == PHY_INTERFACE_MODE_5GBASER) { + ctrl |= GEM_BIT(PCSSEL); + ncr |= GEM_BIT(ENABLE_HS_MAC); + } else if (bp->caps & MACB_CAPS_MIIONRGMII && + bp->phy_interface == PHY_INTERFACE_MODE_MII) { + ncr |= MACB_BIT(MIIONRGMII); + } + } + + /* Apply the new configuration, if any */ + if (old_ctrl ^ ctrl) + macb_or_gem_writel(bp, NCFGR, ctrl); + + if (old_ncr ^ ncr) + macb_or_gem_writel(bp, NCR, ncr); + + /* Disable AN for SGMII fixed link configuration, enable otherwise. + * Must be written after PCSSEL is set in NCFGR, + * otherwise writes will not take effect. + */ + if (macb_is_gem(bp) && (state->interface == PHY_INTERFACE_MODE_SGMII || + state->interface == PHY_INTERFACE_MODE_2500BASEX)) { + u32 pcsctrl, old_pcsctrl; + + old_pcsctrl = gem_readl(bp, PCSCNTRL); + if (mode == MLO_AN_FIXED) + pcsctrl = old_pcsctrl & ~GEM_BIT(PCSAUTONEG); + else + pcsctrl = old_pcsctrl | GEM_BIT(PCSAUTONEG); + if (old_pcsctrl != pcsctrl) + gem_writel(bp, PCSCNTRL, pcsctrl); + } + + spin_unlock_irqrestore(&bp->lock, flags); +} + +static void macb_mac_link_down(struct phylink_config *config, unsigned int mode, + phy_interface_t interface) +{ + struct net_device *ndev = to_net_dev(config->dev); + struct macb *bp = netdev_priv(ndev); + struct macb_tx_skb *tx_skb; + struct macb_queue *queue; + struct macb_dma_desc *tx_desc = NULL; + unsigned int q; + u32 ctrl; + int i; + + if (bp->use_ncsi) + ncsi_stop_dev(bp->ndev); + + if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC)) + for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) + queue_writel(queue, IDR, + bp->rx_intr_mask | MACB_TX_INT_FLAGS | MACB_BIT(HRESP)); + + /* Disable Rx and Tx */ + ctrl = macb_readl(bp, NCR) & ~(MACB_BIT(RE) | + MACB_BIT(TE)) & ~(MACB_BIT(2PT5G)); + macb_writel(bp, NCR, ctrl); + + /* Tx clean */ + for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { + spin_lock_bh(&queue->tx_ptr_lock); + for (i = 0; i < bp->tx_ring_size; i++) { + tx_skb = macb_tx_skb(queue, i); + /* free unsent skb buffers */ + if (tx_skb) + macb_tx_unmap(bp, tx_skb, 0); + + tx_desc = macb_tx_desc(queue, i); + macb_set_addr(bp, tx_desc, 0); + tx_desc->ctrl &= ~MACB_BIT(TX_USED); + } + spin_unlock_bh(&queue->tx_ptr_lock); + } + + netif_tx_stop_all_queues(ndev); +} + +static void phytium_gem1p0_sel_clk(struct macb *bp, int speed) +{ + if (bp->phy_interface == PHY_INTERFACE_MODE_10GBASER || + bp->phy_interface == PHY_INTERFACE_MODE_USXGMII) { + gem_writel(bp, SRC_SEL_LN, 0x1); /*0x1c04*/ + if (speed == SPEED_5000) { + gem_writel(bp, DIV_SEL0_LN, 0x8); /*0x1c08*/ + gem_writel(bp, DIV_SEL1_LN, 0x2); /*0x1c0c*/ + gem_writel(bp, PMA_XCVR_POWER_STATE, 0x0); /*0x1c10*/ + } else { + gem_writel(bp, DIV_SEL0_LN, 0x4); /*0x1c08*/ + gem_writel(bp, DIV_SEL1_LN, 0x1); /*0x1c0c*/ + gem_writel(bp, PMA_XCVR_POWER_STATE, 0x1); /*0x1c10*/ + } + } else if (bp->phy_interface == PHY_INTERFACE_MODE_5GBASER) { + gem_writel(bp, SRC_SEL_LN, 0x1); /*0x1c04*/ + gem_writel(bp, DIV_SEL0_LN, 0x8); /*0x1c08*/ + gem_writel(bp, DIV_SEL1_LN, 0x2); /*0x1c0c*/ + gem_writel(bp, PMA_XCVR_POWER_STATE, 0x0); /*0x1c10*/ + } else if (bp->phy_interface == PHY_INTERFACE_MODE_2500BASEX) { + gem_writel(bp, SRC_SEL_LN, 0x1); /*0x1c04*/ + gem_writel(bp, DIV_SEL0_LN, 0x1); /*0x1c08*/ + gem_writel(bp, DIV_SEL1_LN, 0x2); /*0x1c0c*/ + gem_writel(bp, PMA_XCVR_POWER_STATE, 0x1); /*0x1c10*/ + gem_writel(bp, TX_CLK_SEL0, 0x0); /*0x1c20*/ + gem_writel(bp, TX_CLK_SEL1, 0x1); /*0x1c24*/ + gem_writel(bp, TX_CLK_SEL2, 0x1); /*0x1c28*/ + gem_writel(bp, TX_CLK_SEL3, 0x1); /*0x1c2c*/ + gem_writel(bp, RX_CLK_SEL0, 0x1); /*0x1c30*/ + gem_writel(bp, RX_CLK_SEL1, 0x0); /*0x1c34*/ + gem_writel(bp, TX_CLK_SEL3_0, 0x0); /*0x1c70*/ + gem_writel(bp, TX_CLK_SEL4_0, 0x0); /*0x1c74*/ + gem_writel(bp, RX_CLK_SEL3_0, 0x0); /*0x1c78*/ + gem_writel(bp, RX_CLK_SEL4_0, 0x0); /*0x1c7c*/ + } else if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) { + if (speed == SPEED_2500) { + gem_writel(bp, DIV_SEL0_LN, 0x1); /*0x1c08*/ + gem_writel(bp, DIV_SEL1_LN, 0x2); /*0x1c0c*/ + gem_writel(bp, PMA_XCVR_POWER_STATE, 0x1); /*0x1c10*/ + gem_writel(bp, TX_CLK_SEL0, 0x0); /*0x1c20*/ + gem_writel(bp, TX_CLK_SEL1, 0x1); /*0x1c24*/ + gem_writel(bp, TX_CLK_SEL2, 0x1); /*0x1c28*/ + gem_writel(bp, TX_CLK_SEL3, 0x1); /*0x1c2c*/ + gem_writel(bp, RX_CLK_SEL0, 0x1); /*0x1c30*/ + gem_writel(bp, RX_CLK_SEL1, 0x0); /*0x1c34*/ + gem_writel(bp, TX_CLK_SEL3_0, 0x0); /*0x1c70*/ + gem_writel(bp, TX_CLK_SEL4_0, 0x0); /*0x1c74*/ + gem_writel(bp, RX_CLK_SEL3_0, 0x0); /*0x1c78*/ + gem_writel(bp, RX_CLK_SEL4_0, 0x0); /*0x1c7c*/ + } else if (speed == SPEED_1000) { + gem_writel(bp, SRC_SEL_LN, 0x1); /*0x1c04*/ + gem_writel(bp, DIV_SEL0_LN, 0x4); /*0x1c08*/ + gem_writel(bp, DIV_SEL1_LN, 0x8); /*0x1c0c*/ + gem_writel(bp, PMA_XCVR_POWER_STATE, 0x1); /*0x1c10*/ + gem_writel(bp, TX_CLK_SEL0, 0x0); /*0x1c20*/ + gem_writel(bp, TX_CLK_SEL1, 0x0); /*0x1c24*/ + gem_writel(bp, TX_CLK_SEL2, 0x0); /*0x1c28*/ + gem_writel(bp, TX_CLK_SEL3, 0x1); /*0x1c2c*/ + gem_writel(bp, RX_CLK_SEL0, 0x1); /*0x1c30*/ + gem_writel(bp, RX_CLK_SEL1, 0x0); /*0x1c34*/ + gem_writel(bp, TX_CLK_SEL3_0, 0x0); /*0x1c70*/ + gem_writel(bp, TX_CLK_SEL4_0, 0x0); /*0x1c74*/ + gem_writel(bp, RX_CLK_SEL3_0, 0x0); /*0x1c78*/ + gem_writel(bp, RX_CLK_SEL4_0, 0x0); /*0x1c7c*/ + } else if (speed == SPEED_100 || speed == SPEED_10) { + gem_writel(bp, SRC_SEL_LN, 0x1); /*0x1c04*/ + gem_writel(bp, DIV_SEL0_LN, 0x4); /*0x1c08*/ + gem_writel(bp, DIV_SEL1_LN, 0x8); /*0x1c0c*/ + gem_writel(bp, PMA_XCVR_POWER_STATE, 0x1); /*0x1c10*/ + gem_writel(bp, TX_CLK_SEL0, 0x0); /*0x1c20*/ + gem_writel(bp, TX_CLK_SEL1, 0x0); /*0x1c24*/ + gem_writel(bp, TX_CLK_SEL2, 0x1); /*0x1c28*/ + gem_writel(bp, TX_CLK_SEL3, 0x1); /*0x1c2c*/ + gem_writel(bp, RX_CLK_SEL0, 0x1); /*0x1c30*/ + gem_writel(bp, RX_CLK_SEL1, 0x0); /*0x1c34*/ + gem_writel(bp, TX_CLK_SEL3_0, 0x1); /*0x1c70*/ + gem_writel(bp, TX_CLK_SEL4_0, 0x0); /*0x1c74*/ + gem_writel(bp, RX_CLK_SEL3_0, 0x0); /*0x1c78*/ + gem_writel(bp, RX_CLK_SEL4_0, 0x1); /*0x1c7c*/ + } + } else if (bp->phy_interface == PHY_INTERFACE_MODE_RGMII || + bp->phy_interface == PHY_INTERFACE_MODE_RGMII_ID) { + if (speed == SPEED_1000) { + gem_writel(bp, MII_SELECT, 0x1); /*0x1c18*/ + gem_writel(bp, SEL_MII_ON_RGMII, 0x0); /*0x1c1c*/ + gem_writel(bp, TX_CLK_SEL0, 0x0); /*0x1c20*/ + gem_writel(bp, TX_CLK_SEL1, 0x1); /*0x1c24*/ + gem_writel(bp, TX_CLK_SEL2, 0x0); /*0x1c28*/ + gem_writel(bp, TX_CLK_SEL3, 0x0); /*0x1c2c*/ + gem_writel(bp, RX_CLK_SEL0, 0x0); /*0x1c30*/ + gem_writel(bp, RX_CLK_SEL1, 0x1); /*0x1c34*/ + gem_writel(bp, CLK_250M_DIV10_DIV100_SEL, 0x0); /*0x1c38*/ + gem_writel(bp, RX_CLK_SEL5, 0x1); /*0x1c48*/ + gem_writel(bp, RGMII_TX_CLK_SEL0, 0x1); /*0x1c80*/ + gem_writel(bp, RGMII_TX_CLK_SEL1, 0x0); /*0x1c84*/ + } else if (speed == SPEED_100) { + gem_writel(bp, MII_SELECT, 0x1); /*0x1c18*/ + gem_writel(bp, SEL_MII_ON_RGMII, 0x0); /*0x1c1c*/ + gem_writel(bp, TX_CLK_SEL0, 0x0); /*0x1c20*/ + gem_writel(bp, TX_CLK_SEL1, 0x1); /*0x1c24*/ + gem_writel(bp, TX_CLK_SEL2, 0x0); /*0x1c28*/ + gem_writel(bp, TX_CLK_SEL3, 0x0); /*0x1c2c*/ + gem_writel(bp, RX_CLK_SEL0, 0x0); /*0x1c30*/ + gem_writel(bp, RX_CLK_SEL1, 0x1); /*0x1c34*/ + gem_writel(bp, CLK_250M_DIV10_DIV100_SEL, 0x0); /*0x1c38*/ + gem_writel(bp, RX_CLK_SEL5, 0x1); /*0x1c48*/ + gem_writel(bp, RGMII_TX_CLK_SEL0, 0x0); /*0x1c80*/ + gem_writel(bp, RGMII_TX_CLK_SEL1, 0x0); /*0x1c84*/ + } else { + gem_writel(bp, MII_SELECT, 0x1); /*0x1c18*/ + gem_writel(bp, SEL_MII_ON_RGMII, 0x0); /*0x1c1c*/ + gem_writel(bp, TX_CLK_SEL0, 0x0); /*0x1c20*/ + gem_writel(bp, TX_CLK_SEL1, 0x1); /*0x1c24*/ + gem_writel(bp, TX_CLK_SEL2, 0x0); /*0x1c28*/ + gem_writel(bp, TX_CLK_SEL3, 0x0); /*0x1c2c*/ + gem_writel(bp, RX_CLK_SEL0, 0x0); /*0x1c30*/ + gem_writel(bp, RX_CLK_SEL1, 0x1); /*0x1c34*/ + gem_writel(bp, CLK_250M_DIV10_DIV100_SEL, 0x1); /*0x1c38*/ + gem_writel(bp, RX_CLK_SEL5, 0x1); /*0x1c48*/ + gem_writel(bp, RGMII_TX_CLK_SEL0, 0x0); /*0x1c80*/ + gem_writel(bp, RGMII_TX_CLK_SEL1, 0x0); /*0x1c84*/ + } + } else if (bp->phy_interface == PHY_INTERFACE_MODE_RMII) { + gem_writel(bp, RX_CLK_SEL5, 0x1); /*0x1c48*/ + } + + if (speed == SPEED_100) + gem_writel(bp, HS_MAC_CONFIG, GEM_BFINS(HS_MAC_SPEED, HS_SPEED_100M, + gem_readl(bp, HS_MAC_CONFIG))); + else if (speed == SPEED_1000) + gem_writel(bp, HS_MAC_CONFIG, GEM_BFINS(HS_MAC_SPEED, HS_SPEED_1000M, + gem_readl(bp, HS_MAC_CONFIG))); + else if (speed == SPEED_2500) + gem_writel(bp, HS_MAC_CONFIG, GEM_BFINS(HS_MAC_SPEED, HS_SPEED_2500M, + gem_readl(bp, HS_MAC_CONFIG))); + else if (speed == SPEED_5000) + gem_writel(bp, HS_MAC_CONFIG, GEM_BFINS(HS_MAC_SPEED, HS_SPEED_5000M, + gem_readl(bp, HS_MAC_CONFIG))); + else if (speed == SPEED_10000) + gem_writel(bp, HS_MAC_CONFIG, GEM_BFINS(HS_MAC_SPEED, HS_SPEED_10000M, + gem_readl(bp, HS_MAC_CONFIG))); +} + +static void phytium_gem2p0_sel_clk(struct macb *bp, int speed) +{ + if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) { + if (speed == SPEED_100 || speed == SPEED_10) { + gem_writel(bp, SRC_SEL_LN, 0x1); /*0x1c04*/ + gem_writel(bp, DIV_SEL1_LN, 0x1); /*0x1c0c*/ + } + } + + if (speed == SPEED_100 || speed == SPEED_10) + gem_writel(bp, HS_MAC_CONFIG, GEM_BFINS(HS_MAC_SPEED, HS_SPEED_100M, + gem_readl(bp, HS_MAC_CONFIG))); + else if (speed == SPEED_1000) + gem_writel(bp, HS_MAC_CONFIG, GEM_BFINS(HS_MAC_SPEED, HS_SPEED_1000M, + gem_readl(bp, HS_MAC_CONFIG))); + else if (speed == SPEED_2500) + gem_writel(bp, HS_MAC_CONFIG, GEM_BFINS(HS_MAC_SPEED, HS_SPEED_2500M, + gem_readl(bp, HS_MAC_CONFIG))); +} + +/* Use juggling algorithm to left rotate tx ring and tx skb array */ +static void gem_shuffle_tx_one_ring(struct macb_queue *queue) +{ + unsigned int head, tail, count, ring_size, desc_size; + struct macb_tx_skb tx_skb, *skb_curr, *skb_next; + struct macb_dma_desc *desc_curr, *desc_next; + unsigned int i, cycles, shift, curr, next; + struct macb *bp = queue->bp; + unsigned char desc[24]; + unsigned long flags; + + desc_size = macb_dma_desc_get_size(bp); + + if (WARN_ON_ONCE(desc_size > ARRAY_SIZE(desc))) + return; + + spin_lock_irqsave(&queue->tx_ptr_lock, flags); + head = queue->tx_head; + tail = queue->tx_tail; + ring_size = bp->tx_ring_size; + count = CIRC_CNT(head, tail, ring_size); + + if (!(tail % ring_size)) + goto unlock; + + if (!count) { + queue->tx_head = 0; + queue->tx_tail = 0; + goto unlock; + } + + shift = tail % ring_size; + cycles = gcd(ring_size, shift); + + for (i = 0; i < cycles; i++) { + memcpy(&desc, macb_tx_desc(queue, i), desc_size); + memcpy(&tx_skb, macb_tx_skb(queue, i), + sizeof(struct macb_tx_skb)); + + curr = i; + next = (curr + shift) % ring_size; + + while (next != i) { + desc_curr = macb_tx_desc(queue, curr); + desc_next = macb_tx_desc(queue, next); + + memcpy(desc_curr, desc_next, desc_size); + + if (next == ring_size - 1) + desc_curr->ctrl &= ~MACB_BIT(TX_WRAP); + if (curr == ring_size - 1) + desc_curr->ctrl |= MACB_BIT(TX_WRAP); + + skb_curr = macb_tx_skb(queue, curr); + skb_next = macb_tx_skb(queue, next); + memcpy(skb_curr, skb_next, sizeof(struct macb_tx_skb)); + + curr = next; + next = (curr + shift) % ring_size; + } + + desc_curr = macb_tx_desc(queue, curr); + memcpy(desc_curr, &desc, desc_size); + if (i == ring_size - 1) + desc_curr->ctrl &= ~MACB_BIT(TX_WRAP); + if (curr == ring_size - 1) + desc_curr->ctrl |= MACB_BIT(TX_WRAP); + memcpy(macb_tx_skb(queue, curr), &tx_skb, + sizeof(struct macb_tx_skb)); + } + + queue->tx_head = count; + queue->tx_tail = 0; + + /* Make descriptor updates visible to hardware */ + wmb(); + +unlock: + spin_unlock_irqrestore(&queue->tx_ptr_lock, flags); +} + +/* Rotate the queue so that the tail is at index 0 */ +static void gem_shuffle_tx_rings(struct macb *bp) +{ + struct macb_queue *queue; + int q; + + for (q = 0, queue = bp->queues; q < bp->num_queues; q++, queue++) + gem_shuffle_tx_one_ring(queue); +} + +static void macb_mac_link_up(struct phylink_config *config, + struct phy_device *phy, + unsigned int mode, phy_interface_t interface, + int speed, int duplex, + bool tx_pause, bool rx_pause) +{ + struct net_device *ndev = to_net_dev(config->dev); + struct macb *bp = netdev_priv(ndev); + struct macb_queue *queue; + unsigned long flags; + unsigned int q; + u32 ctrl; + int err; + + spin_lock_irqsave(&bp->lock, flags); + + if (bp->caps & MACB_CAPS_SEL_CLK) + bp->sel_clk_hw(bp, speed); + + ctrl = macb_or_gem_readl(bp, NCFGR); + + ctrl &= ~(MACB_BIT(SPD) | MACB_BIT(FD)); + + if (speed == SPEED_100) + ctrl |= MACB_BIT(SPD); + + if (duplex) + ctrl |= MACB_BIT(FD); + + if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC)) { + ctrl &= ~MACB_BIT(PAE); + if (macb_is_gem(bp)) { + ctrl &= ~GEM_BIT(GBE); + + if (speed == SPEED_1000 || speed == SPEED_2500) + ctrl |= GEM_BIT(GBE); + } + + if (rx_pause) + ctrl |= MACB_BIT(PAE); + + macb_set_tx_clk(bp, speed); + + for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { + queue_writel(queue, IER, + bp->rx_intr_mask | MACB_TX_INT_FLAGS | MACB_BIT(HRESP)); + } + } + + macb_or_gem_writel(bp, NCFGR, ctrl); + + if (speed == SPEED_2500) { + u32 network_ctrl; + u32 pcsctrl, old_pcsctrl; + + network_ctrl = macb_readl(bp, NCR); + network_ctrl |= MACB_BIT(2PT5G); + macb_writel(bp, NCR, network_ctrl); + + old_pcsctrl = gem_readl(bp, PCSCNTRL); + pcsctrl = old_pcsctrl & ~GEM_BIT(PCSAUTONEG); + if (old_pcsctrl != pcsctrl) + gem_writel(bp, PCSCNTRL, pcsctrl); + } + + if (bp->phy_interface == PHY_INTERFACE_MODE_10GBASER || + bp->phy_interface == PHY_INTERFACE_MODE_USXGMII) { + if (speed == SPEED_5000) + gem_writel(bp, HS_MAC_CONFIG, + GEM_BFINS(HS_MAC_SPEED, HS_SPEED_5000M, + gem_readl(bp, HS_MAC_CONFIG))); + else + gem_writel(bp, HS_MAC_CONFIG, + GEM_BFINS(HS_MAC_SPEED, HS_SPEED_10000M, + gem_readl(bp, HS_MAC_CONFIG))); + } else if (bp->phy_interface == PHY_INTERFACE_MODE_5GBASER) + gem_writel(bp, HS_MAC_CONFIG, + GEM_BFINS(HS_MAC_SPEED, HS_SPEED_5000M, + gem_readl(bp, HS_MAC_CONFIG))); + + spin_unlock_irqrestore(&bp->lock, flags); + + if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC)) { + macb_set_tx_clk(bp, speed); + gem_shuffle_tx_rings(bp); + } + + /* Enable Rx and Tx; Enable PTP unicast */ + ctrl = macb_readl(bp, NCR); + if (gem_has_ptp(bp)) + ctrl |= MACB_BIT(PTPUNI); + + macb_writel(bp, NCR, ctrl | MACB_BIT(RE) | MACB_BIT(TE)); + + if (bp->use_ncsi) { + /* Start the NCSI device */ + err = ncsi_start_dev(bp->ndev); + if (err) { + netdev_err(bp->dev, "Ncsi start dev failed (error %d)\n", err); + return; + } + } + + netif_tx_wake_all_queues(ndev); +} + +static struct phylink_pcs *macb_mac_select_pcs(struct phylink_config *config, + phy_interface_t interface) +{ + struct net_device *ndev = to_net_dev(config->dev); + struct macb *bp = netdev_priv(ndev); + + if (interface == PHY_INTERFACE_MODE_10GBASER || + interface == PHY_INTERFACE_MODE_5GBASER || + interface == PHY_INTERFACE_MODE_USXGMII) { + return &bp->phylink_usx_pcs; + } else if (interface == PHY_INTERFACE_MODE_SGMII || + interface == PHY_INTERFACE_MODE_2500BASEX) { + return &bp->phylink_sgmii_pcs; + } else { + return NULL; + } +} + +static const struct phylink_mac_ops macb_phylink_ops = { + .mac_select_pcs = macb_mac_select_pcs, + .mac_config = macb_mac_config, + .mac_link_down = macb_mac_link_down, + .mac_link_up = macb_mac_link_up, +}; + +static bool macb_phy_handle_exists(struct device_node *dn) +{ + dn = of_parse_phandle(dn, "phy-handle", 0); + of_node_put(dn); + return dn != NULL; +} + +static int macb_phylink_connect(struct macb *bp) +{ + struct device_node *dn = bp->pdev->dev.of_node; + struct net_device *dev = bp->dev; + struct phy_device *phydev; + int ret; + + if (dn) + ret = phylink_of_phy_connect(bp->phylink, dn, 0); + + if (!dn || (ret && !macb_phy_handle_exists(dn))) { + phydev = phy_find_first(bp->mii_bus); + if (!phydev) { + netdev_err(dev, "no PHY found\n"); + return -ENXIO; + } + + /* attach the mac to the phy */ + ret = phylink_connect_phy(bp->phylink, phydev); + } + + if (ret) { + netdev_err(dev, "Could not attach PHY (%d)\n", ret); + return ret; + } + + phylink_start(bp->phylink); + + return 0; +} + +static void macb_get_pcs_fixed_state(struct phylink_config *config, + struct phylink_link_state *state) +{ + struct net_device *ndev = to_net_dev(config->dev); + struct macb *bp = netdev_priv(ndev); + + state->link = (macb_readl(bp, NSR) & MACB_BIT(NSR_LINK)) != 0; +} + +static void macb_get_usx_pcs_fixed_state(struct phylink_config *config, + struct phylink_link_state *state) +{ + u32 val; + struct net_device *ndev = to_net_dev(config->dev); + struct macb *bp = netdev_priv(ndev); + + val = gem_readl(bp, USX_STATUS); + state->link = !!(val & GEM_BIT(USX_BLOCK_LOCK)); + val = gem_readl(bp, NCFGR); + if (val & GEM_BIT(PAE)) + state->pause = MLO_PAUSE_RX; +} + +/* based on au1000_eth. c*/ +static int macb_mii_probe(struct net_device *dev) +{ + struct macb *bp = netdev_priv(dev); + + bp->phylink_sgmii_pcs.ops = &macb_phylink_pcs_ops; + bp->phylink_sgmii_pcs.neg_mode = true; + bp->phylink_usx_pcs.ops = &macb_phylink_usx_pcs_ops; + bp->phylink_usx_pcs.neg_mode = true; + + bp->phylink_config.dev = &dev->dev; + bp->phylink_config.type = PHYLINK_NETDEV; + bp->phylink_config.mac_managed_pm = true; + + if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII || + bp->phy_interface == PHY_INTERFACE_MODE_2500BASEX) { + bp->phylink_config.poll_fixed_state = true; + bp->phylink_config.get_fixed_state = macb_get_pcs_fixed_state; + } else if (bp->phy_interface == PHY_INTERFACE_MODE_USXGMII) { + bp->phylink_config.poll_fixed_state = true; + bp->phylink_config.get_fixed_state = macb_get_usx_pcs_fixed_state; + } + + bp->phylink_config.mac_capabilities = MAC_ASYM_PAUSE | + MAC_10 | MAC_100; + + __set_bit(PHY_INTERFACE_MODE_MII, + bp->phylink_config.supported_interfaces); + __set_bit(PHY_INTERFACE_MODE_RMII, + bp->phylink_config.supported_interfaces); + + /* Determine what modes are supported */ + if (macb_is_gem(bp) && (bp->caps & MACB_CAPS_GIGABIT_MODE_AVAILABLE)) { + bp->phylink_config.mac_capabilities |= MAC_1000FD; + if (!(bp->caps & MACB_CAPS_NO_GIGABIT_HALF)) + bp->phylink_config.mac_capabilities |= MAC_1000HD; + + __set_bit(PHY_INTERFACE_MODE_GMII, + bp->phylink_config.supported_interfaces); + phy_interface_set_rgmii(bp->phylink_config.supported_interfaces); + + if (bp->caps & MACB_CAPS_PCS) + __set_bit(PHY_INTERFACE_MODE_SGMII, + bp->phylink_config.supported_interfaces); + + if (bp->caps & MACB_CAPS_HIGH_SPEED) { + __set_bit(PHY_INTERFACE_MODE_10GBASER, + bp->phylink_config.supported_interfaces); + __set_bit(PHY_INTERFACE_MODE_USXGMII, + bp->phylink_config.supported_interfaces); + bp->phylink_config.mac_capabilities |= MAC_10000FD; + } + } + + bp->phylink = phylink_create(&bp->phylink_config, bp->pdev->dev.fwnode, + bp->phy_interface, &macb_phylink_ops); + if (IS_ERR(bp->phylink)) { + netdev_err(dev, "Could not create a phylink instance (%ld)\n", + PTR_ERR(bp->phylink)); + return PTR_ERR(bp->phylink); + } + + return 0; +} + +static int macb_mdiobus_register(struct macb *bp) +{ + struct device_node *child, *np = bp->pdev->dev.of_node; + + /* If we have a child named mdio, probe it instead of looking for PHYs + * directly under the MAC node + */ + child = of_get_child_by_name(np, "mdio"); + if (child) { + int ret = of_mdiobus_register(bp->mii_bus, child); + + of_node_put(child); + return ret; + } + + /* Only create the PHY from the device tree if at least one PHY is + * described. Otherwise scan the entire MDIO bus. We do this to support + * old device tree that did not follow the best practices and did not + * describe their network PHYs. + */ + for_each_available_child_of_node(np, child) + if (of_mdiobus_child_is_phy(child)) { + /* The loop increments the child refcount, + * decrement it before returning. + */ + of_node_put(child); + + return of_mdiobus_register(bp->mii_bus, np); + } + + return mdiobus_register(bp->mii_bus); +} + +static int macb_mii_init(struct macb *bp) +{ + struct device_node *child, *np = bp->pdev->dev.of_node; + int err = -ENXIO; + + /* With fixed-link, we don't need to register the MDIO bus, + * except if we have a child named "mdio" in the device tree. + * In that case, some devices may be attached to the MACB's MDIO bus. + */ + child = of_get_child_by_name(np, "mdio"); + if (child) + of_node_put(child); + else if (of_phy_is_fixed_link(np)) + return macb_mii_probe(bp->dev); + + /* Enable management port */ + macb_writel(bp, NCR, MACB_BIT(MPE)); + + bp->mii_bus = mdiobus_alloc(); + if (!bp->mii_bus) { + err = -ENOMEM; + goto err_out; + } + + bp->mii_bus->name = "MACB_mii_bus"; + bp->mii_bus->read = &macb_mdio_read_c22; + bp->mii_bus->write = &macb_mdio_write_c22; + bp->mii_bus->read_c45 = &macb_mdio_read_c45; + bp->mii_bus->write_c45 = &macb_mdio_write_c45; + snprintf(bp->mii_bus->id, MII_BUS_ID_SIZE, "%s-%x", + bp->pdev->name, bp->pdev->id); + bp->mii_bus->priv = bp; + bp->mii_bus->parent = &bp->pdev->dev; + + dev_set_drvdata(&bp->dev->dev, bp->mii_bus); + + err = macb_mdiobus_register(bp); + if (err) + goto err_out_free_mdiobus; + + err = macb_mii_probe(bp->dev); + if (err) + goto err_out_unregister_bus; + + return 0; + +err_out_unregister_bus: + mdiobus_unregister(bp->mii_bus); +err_out_free_mdiobus: + mdiobus_free(bp->mii_bus); +err_out: + return err; +} + +static void macb_update_stats(struct macb *bp) +{ + u32 *p = &bp->hw_stats.macb.rx_pause_frames; + u32 *end = &bp->hw_stats.macb.tx_pause_frames + 1; + int offset = MACB_PFR; + + WARN_ON((unsigned long)(end - p - 1) != (MACB_TPF - MACB_PFR) / 4); + + for (; p < end; p++, offset += 4) + *p += bp->macb_reg_readl(bp, offset); +} + +static int macb_halt_tx(struct macb *bp) +{ + u32 status; + + macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(THALT)); + + /* Poll TSR until TGO is cleared or timeout. */ + return read_poll_timeout_atomic(macb_readl, status, + !(status & MACB_BIT(TGO)), + 250, MACB_HALT_TIMEOUT, false, + bp, TSR); +} + +static void macb_tx_unmap(struct macb *bp, struct macb_tx_skb *tx_skb, int budget) +{ + if (tx_skb->mapping) { + if (tx_skb->mapped_as_page) + dma_unmap_page(&bp->pdev->dev, tx_skb->mapping, + tx_skb->size, DMA_TO_DEVICE); + else + dma_unmap_single(&bp->pdev->dev, tx_skb->mapping, + tx_skb->size, DMA_TO_DEVICE); + tx_skb->mapping = 0; + } + + if (tx_skb->skb) { + dev_consume_skb_any(tx_skb->skb); + tx_skb->skb = NULL; + } +} + +static void macb_set_addr(struct macb *bp, struct macb_dma_desc *desc, dma_addr_t addr) +{ +#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT + struct macb_dma_desc_64 *desc_64; + + if (bp->hw_dma_cap & HW_DMA_CAP_64B) { + desc_64 = macb_64b_desc(bp, desc); + desc_64->addrh = upper_32_bits(addr); + /* The low bits of RX address contain the RX_USED bit, clearing + * of which allows packet RX. Make sure the high bits are also + * visible to HW at that point. + */ + dma_wmb(); + } +#endif + desc->addr = lower_32_bits(addr); +} + +static dma_addr_t macb_get_addr(struct macb *bp, struct macb_dma_desc *desc) +{ + dma_addr_t addr = 0; +#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT + struct macb_dma_desc_64 *desc_64; + + if (bp->hw_dma_cap & HW_DMA_CAP_64B) { + desc_64 = macb_64b_desc(bp, desc); + addr = ((u64)(desc_64->addrh) << 32); + } +#endif + addr |= MACB_BF(RX_WADDR, MACB_BFEXT(RX_WADDR, desc->addr)); +#ifdef CONFIG_MACB_USE_HWSTAMP + if (bp->hw_dma_cap & HW_DMA_CAP_PTP) + addr &= ~GEM_BIT(DMA_RXVALID); +#endif + return addr; +} + +static void macb_tx_error_task(struct work_struct *work) +{ + struct macb_queue *queue = container_of(work, struct macb_queue, + tx_error_task); + bool halt_timeout = false; + struct macb *bp = queue->bp; + struct macb_tx_skb *tx_skb; + struct macb_dma_desc *desc; + struct sk_buff *skb; + unsigned int tail; + unsigned long flags; + + netdev_vdbg(bp->dev, "macb_tx_error_task: q = %u, t = %u, h = %u\n", + (unsigned int)(queue - bp->queues), + queue->tx_tail, queue->tx_head); + + /* Prevent the queue NAPI TX poll from running, as it calls + * macb_tx_complete(), which in turn may call netif_wake_subqueue(). + * As explained below, we have to halt the transmission before updating + * TBQP registers so we call netif_tx_stop_all_queues() to notify the + * network engine about the macb/gem being halted. + */ + napi_disable(&queue->napi_tx); + spin_lock_irqsave(&bp->lock, flags); + + /* Make sure nobody is trying to queue up new packets */ + netif_tx_stop_all_queues(bp->dev); + + /* Stop transmission now + * (in case we have just queued new packets) + * macb/gem must be halted to write TBQP register + */ + if (macb_halt_tx(bp)) { + netdev_err(bp->dev, "BUG: halt tx timed out\n"); + macb_writel(bp, NCR, macb_readl(bp, NCR) & (~MACB_BIT(TE))); + halt_timeout = true; + } + + /* Treat frames in TX queue including the ones that caused the error. + * Free transmit buffers in upper layer. + */ + for (tail = queue->tx_tail; tail != queue->tx_head; tail++) { + u32 ctrl; + + desc = macb_tx_desc(queue, tail); + ctrl = desc->ctrl; + tx_skb = macb_tx_skb(queue, tail); + skb = tx_skb->skb; + + if (ctrl & MACB_BIT(TX_USED)) { + /* skb is set for the last buffer of the frame */ + while (!skb) { + macb_tx_unmap(bp, tx_skb, 0); + tail++; + tx_skb = macb_tx_skb(queue, tail); + skb = tx_skb->skb; + } + + /* ctrl still refers to the first buffer descriptor + * since it's the only one written back by the hardware + */ + if (!(ctrl & MACB_BIT(TX_BUF_EXHAUSTED))) { + netdev_vdbg(bp->dev, "txerr skb %u (data %p) TX complete\n", + macb_tx_ring_wrap(bp, tail), + skb->data); + bp->dev->stats.tx_packets++; + queue->stats.tx_packets++; + bp->dev->stats.tx_bytes += skb->len; + queue->stats.tx_bytes += skb->len; + } + } else { + /* "Buffers exhausted mid-frame" errors may only happen + * if the driver is buggy, so complain loudly about + * those. Statistics are updated by hardware. + */ + if (ctrl & MACB_BIT(TX_BUF_EXHAUSTED)) + netdev_err(bp->dev, + "BUG: TX buffers exhausted mid-frame\n"); + + desc->ctrl = ctrl | MACB_BIT(TX_USED); + } + + macb_tx_unmap(bp, tx_skb, 0); + } + + /* Set end of TX queue */ + desc = macb_tx_desc(queue, 0); + macb_set_addr(bp, desc, 0); + desc->ctrl = MACB_BIT(TX_USED); + + /* Make descriptor updates visible to hardware */ + wmb(); + + /* Reinitialize the TX desc queue */ + queue_writel(queue, TBQP, lower_32_bits(queue->tx_ring_dma)); +#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT + if (bp->hw_dma_cap & HW_DMA_CAP_64B) + queue_writel(queue, TBQPH, upper_32_bits(queue->tx_ring_dma)); +#endif + /* Make TX ring reflect state of hardware */ + queue->tx_head = 0; + queue->tx_tail = 0; + + /* Housework before enabling TX IRQ */ + macb_writel(bp, TSR, macb_readl(bp, TSR)); + queue_writel(queue, IER, MACB_TX_INT_FLAGS); + + if (halt_timeout) + macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TE)); + + /* Now we are ready to start transmission again */ + netif_tx_start_all_queues(bp->dev); + macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART)); + + spin_unlock_irqrestore(&bp->lock, flags); + napi_enable(&queue->napi_tx); +} + +static bool ptp_one_step_sync(struct sk_buff *skb) +{ + struct ptp_header *hdr; + unsigned int ptp_class; + u8 msgtype; + + /* No need to parse packet if PTP TS is not involved */ + if (likely(!(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP))) + goto not_oss; + + /* Identify and return whether PTP one step sync is being processed */ + ptp_class = ptp_classify_raw(skb); + if (ptp_class == PTP_CLASS_NONE) + goto not_oss; + + hdr = ptp_parse_header(skb, ptp_class); + if (!hdr) + goto not_oss; + + if (hdr->flag_field[0] & PTP_FLAG_TWOSTEP) + goto not_oss; + + msgtype = ptp_get_msgtype(hdr, ptp_class); + if (msgtype == PTP_MSGTYPE_SYNC) + return true; + +not_oss: + return false; +} + +static int macb_tx_complete(struct macb_queue *queue, int budget) +{ + struct macb *bp = queue->bp; + u16 queue_index = queue - bp->queues; + unsigned long flags; + unsigned int tail; + unsigned int head; + int packets = 0; + + spin_lock_irqsave(&queue->tx_ptr_lock, flags); + head = queue->tx_head; + for (tail = queue->tx_tail; tail != head && packets < budget; tail++) { + struct macb_tx_skb *tx_skb; + struct sk_buff *skb; + struct macb_dma_desc *desc; + u32 ctrl; + + desc = macb_tx_desc(queue, tail); + + /* Make hw descriptor updates visible to CPU */ + rmb(); + + ctrl = desc->ctrl; + + /* TX_USED bit is only set by hardware on the very first buffer + * descriptor of the transmitted frame. + */ + if (!(ctrl & MACB_BIT(TX_USED))) + break; + + /* Process all buffers of the current transmitted frame */ + for (;; tail++) { + tx_skb = macb_tx_skb(queue, tail); + skb = tx_skb->skb; + + /* First, update TX stats if needed */ + if (skb) { + if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && + !ptp_one_step_sync(skb)) + gem_ptp_do_txstamp(bp, skb, desc); + + netdev_vdbg(bp->dev, "skb %u (data %p) TX complete\n", + macb_tx_ring_wrap(bp, tail), + skb->data); + bp->dev->stats.tx_packets++; + queue->stats.tx_packets++; + bp->dev->stats.tx_bytes += skb->len; + queue->stats.tx_bytes += skb->len; + packets++; + } + + /* Now we can safely release resources */ + macb_tx_unmap(bp, tx_skb, budget); + + /* skb is set only for the last buffer of the frame. + * WARNING: at this point skb has been freed by + * macb_tx_unmap(). + */ + if (skb) + break; + } + } + + queue->tx_tail = tail; + if (__netif_subqueue_stopped(bp->dev, queue_index) && + CIRC_CNT(queue->tx_head, queue->tx_tail, + bp->tx_ring_size) <= MACB_TX_WAKEUP_THRESH(bp)) + netif_wake_subqueue(bp->dev, queue_index); + spin_unlock_irqrestore(&queue->tx_ptr_lock, flags); + + return packets; +} + +static void gem_rx_refill(struct macb_queue *queue) +{ + unsigned int entry; + struct sk_buff *skb; + dma_addr_t paddr; + struct macb *bp = queue->bp; + struct macb_dma_desc *desc; + + while (CIRC_SPACE(queue->rx_prepared_head, queue->rx_tail, + bp->rx_ring_size) > 0) { + entry = macb_rx_ring_wrap(bp, queue->rx_prepared_head); + + /* Make hw descriptor updates visible to CPU */ + rmb(); + + desc = macb_rx_desc(queue, entry); + + if (!queue->rx_skbuff[entry]) { + /* allocate sk_buff for this free entry in ring */ + skb = netdev_alloc_skb(bp->dev, bp->rx_buffer_size); + if (unlikely(!skb)) { + netdev_err(bp->dev, + "Unable to allocate sk_buff\n"); + break; + } + + /* now fill corresponding descriptor entry */ + paddr = dma_map_single(&bp->pdev->dev, skb->data, + bp->rx_buffer_size, + DMA_FROM_DEVICE); + if (dma_mapping_error(&bp->pdev->dev, paddr)) { + dev_kfree_skb(skb); + break; + } + + queue->rx_skbuff[entry] = skb; + + if (entry == bp->rx_ring_size - 1) + paddr |= MACB_BIT(RX_WRAP); + desc->ctrl = 0; + /* Setting addr clears RX_USED and allows reception, + * make sure ctrl is cleared first to avoid a race. + */ + dma_wmb(); + macb_set_addr(bp, desc, paddr); + + /* properly align Ethernet header */ + skb_reserve(skb, NET_IP_ALIGN); + } else { + desc->ctrl = 0; + dma_wmb(); + desc->addr &= ~MACB_BIT(RX_USED); + } + queue->rx_prepared_head++; + } + + /* Make descriptor updates visible to hardware */ + wmb(); + + netdev_vdbg(bp->dev, "rx ring: queue: %p, prepared head %d, tail %d\n", + queue, queue->rx_prepared_head, queue->rx_tail); +} + +/* Mark DMA descriptors from begin up to and not including end as unused */ +static void discard_partial_frame(struct macb_queue *queue, unsigned int begin, + unsigned int end) +{ + unsigned int frag; + + for (frag = begin; frag != end; frag++) { + struct macb_dma_desc *desc = macb_rx_desc(queue, frag); + + desc->addr &= ~MACB_BIT(RX_USED); + } + + /* Make descriptor updates visible to hardware */ + wmb(); + + /* When this happens, the hardware stats registers for + * whatever caused this is updated, so we don't have to record + * anything. + */ +} + +static int gem_rx(struct macb_queue *queue, struct napi_struct *napi, + int budget) +{ + struct macb *bp = queue->bp; + unsigned int len; + unsigned int entry; + struct sk_buff *skb; + struct macb_dma_desc *desc; + int count = 0; + + while (count < budget) { + u32 ctrl; + dma_addr_t addr; + bool rxused; + + entry = macb_rx_ring_wrap(bp, queue->rx_tail); + desc = macb_rx_desc(queue, entry); + + /* Make hw descriptor updates visible to CPU */ + rmb(); + + rxused = (desc->addr & MACB_BIT(RX_USED)) ? true : false; + addr = macb_get_addr(bp, desc); + + if (!rxused) + break; + + /* Ensure ctrl is at least as up-to-date as rxused */ + dma_rmb(); + + ctrl = desc->ctrl; + + queue->rx_tail++; + count++; + + if (!(ctrl & MACB_BIT(RX_SOF) && ctrl & MACB_BIT(RX_EOF))) { + netdev_err(bp->dev, + "not whole frame pointed by descriptor\n"); + bp->dev->stats.rx_dropped++; + queue->stats.rx_dropped++; + break; + } + skb = queue->rx_skbuff[entry]; + if (unlikely(!skb)) { + netdev_err(bp->dev, + "inconsistent Rx descriptor chain\n"); + bp->dev->stats.rx_dropped++; + queue->stats.rx_dropped++; + break; + } + /* now everything is ready for receiving packet */ + queue->rx_skbuff[entry] = NULL; + len = ctrl & bp->rx_frm_len_mask; + + netdev_vdbg(bp->dev, "gem_rx %u (len %u)\n", entry, len); + + skb_put(skb, len); + dma_unmap_single(&bp->pdev->dev, addr, + bp->rx_buffer_size, DMA_FROM_DEVICE); + + skb->protocol = eth_type_trans(skb, bp->dev); + skb_checksum_none_assert(skb); + if (bp->dev->features & NETIF_F_RXCSUM && + !(bp->dev->flags & IFF_PROMISC) && + GEM_BFEXT(RX_CSUM, ctrl) & GEM_RX_CSUM_CHECKED_MASK) + skb->ip_summed = CHECKSUM_UNNECESSARY; + + bp->dev->stats.rx_packets++; + queue->stats.rx_packets++; + bp->dev->stats.rx_bytes += skb->len; + queue->stats.rx_bytes += skb->len; + + gem_ptp_do_rxstamp(bp, skb, desc); + +#if defined(DEBUG) && defined(VERBOSE_DEBUG) + netdev_vdbg(bp->dev, "received skb of length %u, csum: %08x\n", + skb->len, skb->csum); + print_hex_dump(KERN_DEBUG, " mac: ", DUMP_PREFIX_ADDRESS, 16, 1, + skb_mac_header(skb), 16, true); + print_hex_dump(KERN_DEBUG, "data: ", DUMP_PREFIX_ADDRESS, 16, 1, + skb->data, 32, true); +#endif + + napi_gro_receive(napi, skb); + } + + gem_rx_refill(queue); + + return count; +} + +static int macb_rx_frame(struct macb_queue *queue, struct napi_struct *napi, + unsigned int first_frag, unsigned int last_frag) +{ + unsigned int len; + unsigned int frag; + unsigned int offset; + struct sk_buff *skb; + struct macb_dma_desc *desc; + struct macb *bp = queue->bp; + + desc = macb_rx_desc(queue, last_frag); + len = desc->ctrl & bp->rx_frm_len_mask; + + netdev_vdbg(bp->dev, "macb_rx_frame frags %u - %u (len %u)\n", + macb_rx_ring_wrap(bp, first_frag), + macb_rx_ring_wrap(bp, last_frag), len); + + /* The ethernet header starts NET_IP_ALIGN bytes into the + * first buffer. Since the header is 14 bytes, this makes the + * payload word-aligned. + * + * Instead of calling skb_reserve(NET_IP_ALIGN), we just copy + * the two padding bytes into the skb so that we avoid hitting + * the slowpath in memcpy(), and pull them off afterwards. + */ + skb = netdev_alloc_skb(bp->dev, len + NET_IP_ALIGN); + if (!skb) { + bp->dev->stats.rx_dropped++; + for (frag = first_frag; ; frag++) { + desc = macb_rx_desc(queue, frag); + desc->addr &= ~MACB_BIT(RX_USED); + if (frag == last_frag) + break; + } + + /* Make descriptor updates visible to hardware */ + wmb(); + + return 1; + } + + offset = 0; + len += NET_IP_ALIGN; + skb_checksum_none_assert(skb); + skb_put(skb, len); + + for (frag = first_frag; ; frag++) { + unsigned int frag_len = bp->rx_buffer_size; + + if (offset + frag_len > len) { + if (unlikely(frag != last_frag)) { + dev_kfree_skb_any(skb); + return -1; + } + frag_len = len - offset; + } + skb_copy_to_linear_data_offset(skb, offset, + macb_rx_buffer(queue, frag), + frag_len); + offset += bp->rx_buffer_size; + desc = macb_rx_desc(queue, frag); + desc->addr &= ~MACB_BIT(RX_USED); + + if (frag == last_frag) + break; + } + + /* Make descriptor updates visible to hardware */ + wmb(); + + __skb_pull(skb, NET_IP_ALIGN); + skb->protocol = eth_type_trans(skb, bp->dev); + + bp->dev->stats.rx_packets++; + bp->dev->stats.rx_bytes += skb->len; + netdev_vdbg(bp->dev, "received skb of length %u, csum: %08x\n", + skb->len, skb->csum); + napi_gro_receive(napi, skb); + + return 0; +} + +static inline void macb_init_rx_ring(struct macb_queue *queue) +{ + struct macb *bp = queue->bp; + dma_addr_t addr; + struct macb_dma_desc *desc = NULL; + int i; + + addr = queue->rx_buffers_dma; + for (i = 0; i < bp->rx_ring_size; i++) { + desc = macb_rx_desc(queue, i); + macb_set_addr(bp, desc, addr); + desc->ctrl = 0; + addr += bp->rx_buffer_size; + } + desc->addr |= MACB_BIT(RX_WRAP); + queue->rx_tail = 0; +} + +static int macb_rx(struct macb_queue *queue, struct napi_struct *napi, + int budget) +{ + struct macb *bp = queue->bp; + bool reset_rx_queue = false; + int received = 0; + unsigned int tail; + int first_frag = -1; + + for (tail = queue->rx_tail; budget > 0; tail++) { + struct macb_dma_desc *desc = macb_rx_desc(queue, tail); + u32 ctrl; + + /* Make hw descriptor updates visible to CPU */ + rmb(); + + if (!(desc->addr & MACB_BIT(RX_USED))) + break; + + /* Ensure ctrl is at least as up-to-date as addr */ + dma_rmb(); + + ctrl = desc->ctrl; + + if (ctrl & MACB_BIT(RX_SOF)) { + if (first_frag != -1) + discard_partial_frame(queue, first_frag, tail); + first_frag = tail; + } + + if (ctrl & MACB_BIT(RX_EOF)) { + int dropped; + + if (unlikely(first_frag == -1)) { + reset_rx_queue = true; + continue; + } + + dropped = macb_rx_frame(queue, napi, first_frag, tail); + first_frag = -1; + if (unlikely(dropped < 0)) { + reset_rx_queue = true; + continue; + } + if (!dropped) { + received++; + budget--; + } + } + } + + if (unlikely(reset_rx_queue)) { + unsigned long flags; + u32 ctrl; + + netdev_err(bp->dev, "RX queue corruption: reset it\n"); + + spin_lock_irqsave(&bp->lock, flags); + + ctrl = macb_readl(bp, NCR); + macb_writel(bp, NCR, ctrl & ~MACB_BIT(RE)); + + macb_init_rx_ring(queue); + queue_writel(queue, RBQP, queue->rx_ring_dma); + + macb_writel(bp, NCR, ctrl | MACB_BIT(RE)); + + spin_unlock_irqrestore(&bp->lock, flags); + return received; + } + + if (first_frag != -1) + queue->rx_tail = first_frag; + else + queue->rx_tail = tail; + + return received; +} + +static bool macb_rx_pending(struct macb_queue *queue) +{ + struct macb *bp = queue->bp; + unsigned int entry; + struct macb_dma_desc *desc; + + entry = macb_rx_ring_wrap(bp, queue->rx_tail); + desc = macb_rx_desc(queue, entry); + + /* Make hw descriptor updates visible to CPU */ + rmb(); + + return (desc->addr & MACB_BIT(RX_USED)) != 0; +} + +static int macb_rx_poll(struct napi_struct *napi, int budget) +{ + struct macb_queue *queue = container_of(napi, struct macb_queue, napi_rx); + struct macb *bp = queue->bp; + int work_done; + + work_done = bp->macbgem_ops.mog_rx(queue, napi, budget); + + netdev_vdbg(bp->dev, "RX poll: queue = %u, work_done = %d, budget = %d\n", + (unsigned int)(queue - bp->queues), work_done, budget); + + if (work_done < budget && napi_complete_done(napi, work_done)) { + queue_writel(queue, IER, bp->rx_intr_mask); + + /* Packet completions only seem to propagate to raise + * interrupts when interrupts are enabled at the time, so if + * packets were received while interrupts were disabled, + * they will not cause another interrupt to be generated when + * interrupts are re-enabled. + * Check for this case here to avoid losing a wakeup. This can + * potentially race with the interrupt handler doing the same + * actions if an interrupt is raised just after enabling them, + * but this should be harmless. + */ + if (macb_rx_pending(queue)) { + queue_writel(queue, IDR, bp->rx_intr_mask); + if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) + queue_writel(queue, ISR, MACB_BIT(RCOMP)); + netdev_vdbg(bp->dev, "poll: packets pending, reschedule\n"); + napi_schedule(napi); + } + } + + /* TODO: Handle errors */ + + return work_done; +} + +static void macb_tx_restart(struct macb_queue *queue) +{ + struct macb *bp = queue->bp; + unsigned int head_idx, tbqp; + unsigned long flags; + + spin_lock_irqsave(&queue->tx_ptr_lock, flags); + + if (queue->tx_head == queue->tx_tail) + goto out_tx_ptr_unlock; + + tbqp = queue_readl(queue, TBQP) - lower_32_bits(queue->tx_ring_dma); + tbqp = tbqp / macb_dma_desc_get_size(bp); + tbqp = macb_adj_dma_desc_idx(bp, macb_tx_ring_wrap(bp, tbqp)); + head_idx = macb_adj_dma_desc_idx(bp, macb_tx_ring_wrap(bp, queue->tx_head)); + + if (tbqp == head_idx) + goto out_tx_ptr_unlock; + + spin_lock(&bp->lock); + macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART)); + spin_unlock(&bp->lock); + +out_tx_ptr_unlock: + spin_unlock_irqrestore(&queue->tx_ptr_lock, flags); +} + +static bool macb_tx_complete_pending(struct macb_queue *queue) +{ + bool retval = false; + unsigned long flags; + + spin_lock_irqsave(&queue->tx_ptr_lock, flags); + if (queue->tx_head != queue->tx_tail) { + /* Make hw descriptor updates visible to CPU */ + rmb(); + + if (macb_tx_desc(queue, queue->tx_tail)->ctrl & MACB_BIT(TX_USED)) + retval = true; + } + spin_unlock_irqrestore(&queue->tx_ptr_lock, flags); + return retval; +} + +static int macb_tx_poll(struct napi_struct *napi, int budget) +{ + struct macb_queue *queue = container_of(napi, struct macb_queue, napi_tx); + struct macb *bp = queue->bp; + int work_done; + + work_done = macb_tx_complete(queue, budget); + + rmb(); // ensure txubr_pending is up to date + if (queue->txubr_pending) { + queue->txubr_pending = false; + netdev_vdbg(bp->dev, "poll: tx restart\n"); + macb_tx_restart(queue); + } + + netdev_vdbg(bp->dev, "TX poll: queue = %u, work_done = %d, budget = %d\n", + (unsigned int)(queue - bp->queues), work_done, budget); + + if (work_done < budget && napi_complete_done(napi, work_done)) { + queue_writel(queue, IER, MACB_BIT(TCOMP)); + + /* Packet completions only seem to propagate to raise + * interrupts when interrupts are enabled at the time, so if + * packets were sent while interrupts were disabled, + * they will not cause another interrupt to be generated when + * interrupts are re-enabled. + * Check for this case here to avoid losing a wakeup. This can + * potentially race with the interrupt handler doing the same + * actions if an interrupt is raised just after enabling them, + * but this should be harmless. + */ + if (macb_tx_complete_pending(queue)) { + queue_writel(queue, IDR, MACB_BIT(TCOMP)); + if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) + queue_writel(queue, ISR, MACB_BIT(TCOMP)); + netdev_vdbg(bp->dev, "TX poll: packets pending, reschedule\n"); + napi_schedule(napi); + } + } + + return work_done; +} + +static void macb_hresp_error_task(struct tasklet_struct *t) +{ + struct macb *bp = from_tasklet(bp, t, hresp_err_tasklet); + struct net_device *dev = bp->dev; + struct macb_queue *queue; + unsigned int q; + u32 ctrl; + + for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { + queue_writel(queue, IDR, bp->rx_intr_mask | + MACB_TX_INT_FLAGS | + MACB_BIT(HRESP)); + } + ctrl = macb_readl(bp, NCR); + ctrl &= ~(MACB_BIT(RE) | MACB_BIT(TE)); + macb_writel(bp, NCR, ctrl); + + netif_tx_stop_all_queues(dev); + netif_carrier_off(dev); + + bp->macbgem_ops.mog_init_rings(bp); + + /* Initialize TX and RX buffers */ + macb_init_buffers(bp); + + /* Enable interrupts */ + for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) + queue_writel(queue, IER, + bp->rx_intr_mask | + MACB_TX_INT_FLAGS | + MACB_BIT(HRESP)); + + ctrl |= MACB_BIT(RE) | MACB_BIT(TE); + macb_writel(bp, NCR, ctrl); + + netif_carrier_on(dev); + netif_tx_start_all_queues(dev); +} + +static irqreturn_t macb_wol_interrupt(int irq, void *dev_id) +{ + struct macb_queue *queue = dev_id; + struct macb *bp = queue->bp; + u32 status; + + status = queue_readl(queue, ISR); + + if (unlikely(!status)) + return IRQ_NONE; + + spin_lock(&bp->lock); + + if (status & MACB_BIT(WOL)) { + queue_writel(queue, IDR, MACB_BIT(WOL)); + macb_writel(bp, WOL, 0); + netdev_vdbg(bp->dev, "MACB WoL: queue = %u, isr = 0x%08lx\n", + (unsigned int)(queue - bp->queues), + (unsigned long)status); + if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) + queue_writel(queue, ISR, MACB_BIT(WOL)); + pm_wakeup_event(&bp->pdev->dev, 0); + } + + spin_unlock(&bp->lock); + + return IRQ_HANDLED; +} + +static irqreturn_t gem_wol_interrupt(int irq, void *dev_id) +{ + struct macb_queue *queue = dev_id; + struct macb *bp = queue->bp; + u32 status; + + status = queue_readl(queue, ISR); + + if (unlikely(!status)) + return IRQ_NONE; + + spin_lock(&bp->lock); + + if (status & GEM_BIT(WOL)) { + queue_writel(queue, IDR, GEM_BIT(WOL)); + gem_writel(bp, WOL, 0); + netdev_vdbg(bp->dev, "GEM WoL: queue = %u, isr = 0x%08lx\n", + (unsigned int)(queue - bp->queues), + (unsigned long)status); + if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) + queue_writel(queue, ISR, GEM_BIT(WOL)); + pm_wakeup_event(&bp->pdev->dev, 0); + } + + spin_unlock(&bp->lock); + + return IRQ_HANDLED; +} + +static irqreturn_t macb_interrupt(int irq, void *dev_id) +{ + struct macb_queue *queue = dev_id; + struct macb *bp = queue->bp; + struct net_device *dev = bp->dev; + u32 status, ctrl; + + status = queue_readl(queue, ISR); + + if (unlikely(!status)) + return IRQ_NONE; + + spin_lock(&bp->lock); + + while (status) { + /* close possible race with dev_close */ + if (unlikely(!netif_running(dev))) { + queue_writel(queue, IDR, -1); + if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) + queue_writel(queue, ISR, -1); + break; + } + + netdev_vdbg(bp->dev, "queue = %u, isr = 0x%08lx\n", + (unsigned int)(queue - bp->queues), + (unsigned long)status); + + if (status & bp->rx_intr_mask) { + /* There's no point taking any more interrupts + * until we have processed the buffers. The + * scheduling call may fail if the poll routine + * is already scheduled, so disable interrupts + * now. + */ + queue_writel(queue, IDR, bp->rx_intr_mask); + if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) + queue_writel(queue, ISR, MACB_BIT(RCOMP)); + + if (napi_schedule_prep(&queue->napi_rx)) { + netdev_vdbg(bp->dev, "scheduling RX softirq\n"); + __napi_schedule(&queue->napi_rx); + } + } + + if (status & (MACB_BIT(TCOMP) | + MACB_BIT(TXUBR))) { + queue_writel(queue, IDR, MACB_BIT(TCOMP)); + if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) + queue_writel(queue, ISR, MACB_BIT(TCOMP) | + MACB_BIT(TXUBR)); + + if (status & MACB_BIT(TXUBR)) { + queue->txubr_pending = true; + wmb(); // ensure softirq can see update + } + + if (napi_schedule_prep(&queue->napi_tx)) { + netdev_vdbg(bp->dev, "scheduling TX softirq\n"); + __napi_schedule(&queue->napi_tx); + } + } + + if (unlikely(status & (MACB_TX_ERR_FLAGS))) { + queue_writel(queue, IDR, MACB_TX_INT_FLAGS); + schedule_work(&queue->tx_error_task); + + if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) + queue_writel(queue, ISR, MACB_TX_ERR_FLAGS); + + break; + } + + /* Link change detection isn't possible with RMII, so we'll + * add that if/when we get our hands on a full-blown MII PHY. + */ + + /* There is a hardware issue under heavy load where DMA can + * stop, this causes endless "used buffer descriptor read" + * interrupts but it can be cleared by re-enabling RX. See + * the at91rm9200 manual, section 41.3.1 or the Zynq manual + * section 16.7.4 for details. RXUBR is only enabled for + * these two versions. + */ + if (status & MACB_BIT(RXUBR)) { + ctrl = macb_readl(bp, NCR); + macb_writel(bp, NCR, ctrl & ~MACB_BIT(RE)); + wmb(); + macb_writel(bp, NCR, ctrl | MACB_BIT(RE)); + + if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) + queue_writel(queue, ISR, MACB_BIT(RXUBR)); + } + + if (status & MACB_BIT(ISR_ROVR)) { + /* We missed at least one packet */ + spin_lock(&bp->stats_lock); + if (macb_is_gem(bp)) + bp->hw_stats.gem.rx_overruns++; + else + bp->hw_stats.macb.rx_overruns++; + spin_unlock(&bp->stats_lock); + + if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) + queue_writel(queue, ISR, MACB_BIT(ISR_ROVR)); + } + + if (status & MACB_BIT(HRESP)) { + tasklet_schedule(&bp->hresp_err_tasklet); + netdev_err(dev, "DMA bus error: HRESP not OK\n"); + + if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) + queue_writel(queue, ISR, MACB_BIT(HRESP)); + } + status = queue_readl(queue, ISR); + } + + spin_unlock(&bp->lock); + + return IRQ_HANDLED; +} + +#ifdef CONFIG_NET_POLL_CONTROLLER +/* Polling receive - used by netconsole and other diagnostic tools + * to allow network i/o with interrupts disabled. + */ +static void macb_poll_controller(struct net_device *dev) +{ + struct macb *bp = netdev_priv(dev); + struct macb_queue *queue; + unsigned long flags; + unsigned int q; + + local_irq_save(flags); + for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) + macb_interrupt(dev->irq, queue); + local_irq_restore(flags); +} +#endif + +static unsigned int macb_tx_map(struct macb *bp, + struct macb_queue *queue, + struct sk_buff *skb, + unsigned int hdrlen) +{ + dma_addr_t mapping; + unsigned int len, entry, i, tx_head = queue->tx_head; + struct macb_tx_skb *tx_skb = NULL; + struct macb_dma_desc *desc; + unsigned int offset, size, count = 0; + unsigned int f, nr_frags = skb_shinfo(skb)->nr_frags; + unsigned int eof = 1, mss_mfs = 0; + u32 ctrl, lso_ctrl = 0, seq_ctrl = 0; + + /* LSO */ + if (skb_shinfo(skb)->gso_size != 0) { + if (ip_hdr(skb)->protocol == IPPROTO_UDP) + /* UDP - UFO */ + lso_ctrl = MACB_LSO_UFO_ENABLE; + else + /* TCP - TSO */ + lso_ctrl = MACB_LSO_TSO_ENABLE; + } + + /* First, map non-paged data */ + len = skb_headlen(skb); + + /* first buffer length */ + size = hdrlen; + + offset = 0; + while (len) { + entry = macb_tx_ring_wrap(bp, tx_head); + tx_skb = &queue->tx_skb[entry]; + + mapping = dma_map_single(&bp->pdev->dev, + skb->data + offset, + size, DMA_TO_DEVICE); + if (dma_mapping_error(&bp->pdev->dev, mapping)) + goto dma_error; + + /* Save info to properly release resources */ + tx_skb->skb = NULL; + tx_skb->mapping = mapping; + tx_skb->size = size; + tx_skb->mapped_as_page = false; + + len -= size; + offset += size; + count++; + tx_head++; + + size = min(len, bp->max_tx_length); + } + + /* Then, map paged data from fragments */ + for (f = 0; f < nr_frags; f++) { + const skb_frag_t *frag = &skb_shinfo(skb)->frags[f]; + + len = skb_frag_size(frag); + offset = 0; + while (len) { + size = min(len, bp->max_tx_length); + entry = macb_tx_ring_wrap(bp, tx_head); + tx_skb = &queue->tx_skb[entry]; + + mapping = skb_frag_dma_map(&bp->pdev->dev, frag, + offset, size, DMA_TO_DEVICE); + if (dma_mapping_error(&bp->pdev->dev, mapping)) + goto dma_error; + + /* Save info to properly release resources */ + tx_skb->skb = NULL; + tx_skb->mapping = mapping; + tx_skb->size = size; + tx_skb->mapped_as_page = true; + + len -= size; + offset += size; + count++; + tx_head++; + } + } + + /* Should never happen */ + if (unlikely(!tx_skb)) { + netdev_err(bp->dev, "BUG! empty skb!\n"); + return 0; + } + + /* This is the last buffer of the frame: save socket buffer */ + tx_skb->skb = skb; + + /* Update TX ring: update buffer descriptors in reverse order + * to avoid race condition + */ + + /* Set 'TX_USED' bit in buffer descriptor at tx_head position + * to set the end of TX queue + */ + i = tx_head; + entry = macb_tx_ring_wrap(bp, i); + ctrl = MACB_BIT(TX_USED); + desc = macb_tx_desc(queue, entry); + desc->ctrl = ctrl; + + if (lso_ctrl) { + if (lso_ctrl == MACB_LSO_UFO_ENABLE) + /* include header and FCS in value given to h/w */ + mss_mfs = skb_shinfo(skb)->gso_size + + skb_transport_offset(skb) + + ETH_FCS_LEN; + else /* TSO */ { + mss_mfs = skb_shinfo(skb)->gso_size; + /* TCP Sequence Number Source Select + * can be set only for TSO + */ + seq_ctrl = 0; + } + } + + do { + i--; + entry = macb_tx_ring_wrap(bp, i); + tx_skb = &queue->tx_skb[entry]; + desc = macb_tx_desc(queue, entry); + + ctrl = (u32)tx_skb->size; + if (eof) { + ctrl |= MACB_BIT(TX_LAST); + eof = 0; + } + if (unlikely(entry == (bp->tx_ring_size - 1))) + ctrl |= MACB_BIT(TX_WRAP); + + /* First descriptor is header descriptor */ + if (i == queue->tx_head) { + ctrl |= MACB_BF(TX_LSO, lso_ctrl); + ctrl |= MACB_BF(TX_TCP_SEQ_SRC, seq_ctrl); + if ((bp->dev->features & NETIF_F_HW_CSUM) && + skb->ip_summed != CHECKSUM_PARTIAL && !lso_ctrl && + !ptp_one_step_sync(skb)) + ctrl |= MACB_BIT(TX_NOCRC); + } else + /* Only set MSS/MFS on payload descriptors + * (second or later descriptor) + */ + ctrl |= MACB_BF(MSS_MFS, mss_mfs); + + /* Set TX buffer descriptor */ + macb_set_addr(bp, desc, tx_skb->mapping); + /* desc->addr must be visible to hardware before clearing + * 'TX_USED' bit in desc->ctrl. + */ + wmb(); + desc->ctrl = ctrl; + } while (i != queue->tx_head); + + queue->tx_head = tx_head; + + return count; + +dma_error: + netdev_err(bp->dev, "TX DMA map failed\n"); + + for (i = queue->tx_head; i != tx_head; i++) { + tx_skb = macb_tx_skb(queue, i); + + macb_tx_unmap(bp, tx_skb, 0); + } + + return 0; +} + +static netdev_features_t macb_features_check(struct sk_buff *skb, + struct net_device *dev, + netdev_features_t features) +{ + unsigned int nr_frags, f; + unsigned int hdrlen; + + /* Validate LSO compatibility */ + + /* there is only one buffer or protocol is not UDP */ + if (!skb_is_nonlinear(skb) || (ip_hdr(skb)->protocol != IPPROTO_UDP)) + return features; + + /* length of header */ + hdrlen = skb_transport_offset(skb); + + /* For UFO only: + * When software supplies two or more payload buffers all payload buffers + * apart from the last must be a multiple of 8 bytes in size. + */ + if (!IS_ALIGNED(skb_headlen(skb) - hdrlen, MACB_TX_LEN_ALIGN)) + return features & ~MACB_NETIF_LSO; + + nr_frags = skb_shinfo(skb)->nr_frags; + /* No need to check last fragment */ + nr_frags--; + for (f = 0; f < nr_frags; f++) { + const skb_frag_t *frag = &skb_shinfo(skb)->frags[f]; + + if (!IS_ALIGNED(skb_frag_size(frag), MACB_TX_LEN_ALIGN)) + return features & ~MACB_NETIF_LSO; + } + return features; +} + +static inline int macb_clear_csum(struct sk_buff *skb) +{ + /* no change for packets without checksum offloading */ + if (skb->ip_summed != CHECKSUM_PARTIAL) + return 0; + + /* make sure we can modify the header */ + if (unlikely(skb_cow_head(skb, 0))) + return -1; + + /* initialize checksum field + * This is required - at least for Zynq, which otherwise calculates + * wrong UDP header checksums for UDP packets with UDP data len <=2 + */ + *(__sum16 *)(skb_checksum_start(skb) + skb->csum_offset) = 0; + return 0; +} + +static int macb_pad_and_fcs(struct sk_buff **skb, struct net_device *ndev) +{ + bool cloned = skb_cloned(*skb) || skb_header_cloned(*skb) || + skb_is_nonlinear(*skb); + int padlen = ETH_ZLEN - (*skb)->len; + int tailroom = skb_tailroom(*skb); + struct sk_buff *nskb; + u32 fcs; + + if (!(ndev->features & NETIF_F_HW_CSUM) || + !((*skb)->ip_summed != CHECKSUM_PARTIAL) || + skb_shinfo(*skb)->gso_size || ptp_one_step_sync(*skb)) + return 0; + + if (padlen <= 0) { + /* FCS could be appeded to tailroom. */ + if (tailroom >= ETH_FCS_LEN) + goto add_fcs; + /* No room for FCS, need to reallocate skb. */ + else + padlen = ETH_FCS_LEN; + } else { + /* Add room for FCS. */ + padlen += ETH_FCS_LEN; + } + + if (cloned || tailroom < padlen) { + nskb = skb_copy_expand(*skb, 0, padlen, GFP_ATOMIC); + if (!nskb) + return -ENOMEM; + + dev_consume_skb_any(*skb); + *skb = nskb; + } + + if (padlen > ETH_FCS_LEN) + skb_put_zero(*skb, padlen - ETH_FCS_LEN); + +add_fcs: + /* set FCS to packet */ + fcs = crc32_le(~0, (*skb)->data, (*skb)->len); + fcs = ~fcs; + + skb_put_u8(*skb, fcs & 0xff); + skb_put_u8(*skb, (fcs >> 8) & 0xff); + skb_put_u8(*skb, (fcs >> 16) & 0xff); + skb_put_u8(*skb, (fcs >> 24) & 0xff); + + return 0; +} + +static netdev_tx_t macb_start_xmit(struct sk_buff *skb, struct net_device *dev) +{ + u16 queue_index = skb_get_queue_mapping(skb); + struct macb *bp = netdev_priv(dev); + struct macb_queue *queue = &bp->queues[queue_index]; + unsigned int desc_cnt, nr_frags, frag_size, f; + unsigned int hdrlen; + unsigned long flags; + bool is_lso; + netdev_tx_t ret = NETDEV_TX_OK; + + if (macb_clear_csum(skb)) { + dev_kfree_skb_any(skb); + return ret; + } + + if (macb_pad_and_fcs(&skb, dev)) { + dev_kfree_skb_any(skb); + return ret; + } + +#ifdef CONFIG_MACB_USE_HWSTAMP + if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && + (bp->hw_dma_cap & HW_DMA_CAP_PTP)) + skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; +#endif + + is_lso = (skb_shinfo(skb)->gso_size != 0); + + if (is_lso) { + /* length of headers */ + if (ip_hdr(skb)->protocol == IPPROTO_UDP) + /* only queue eth + ip headers separately for UDP */ + hdrlen = skb_transport_offset(skb); + else + hdrlen = skb_tcp_all_headers(skb); + if (skb_headlen(skb) < hdrlen) { + netdev_err(bp->dev, "Error - LSO headers fragmented!!!\n"); + /* if this is required, would need to copy to single buffer */ + return NETDEV_TX_BUSY; + } + } else + hdrlen = min(skb_headlen(skb), bp->max_tx_length); + +#if defined(DEBUG) && defined(VERBOSE_DEBUG) + netdev_vdbg(bp->dev, + "start_xmit: queue %hu len %u head %p data %p tail %p end %p\n", + queue_index, skb->len, skb->head, skb->data, + skb_tail_pointer(skb), skb_end_pointer(skb)); + print_hex_dump(KERN_DEBUG, "data: ", DUMP_PREFIX_OFFSET, 16, 1, + skb->data, 16, true); +#endif + + /* Count how many TX buffer descriptors are needed to send this + * socket buffer: skb fragments of jumbo frames may need to be + * split into many buffer descriptors. + */ + if (is_lso && (skb_headlen(skb) > hdrlen)) + /* extra header descriptor if also payload in first buffer */ + desc_cnt = DIV_ROUND_UP((skb_headlen(skb) - hdrlen), bp->max_tx_length) + 1; + else + desc_cnt = DIV_ROUND_UP(skb_headlen(skb), bp->max_tx_length); + nr_frags = skb_shinfo(skb)->nr_frags; + for (f = 0; f < nr_frags; f++) { + frag_size = skb_frag_size(&skb_shinfo(skb)->frags[f]); + desc_cnt += DIV_ROUND_UP(frag_size, bp->max_tx_length); + } + + spin_lock_irqsave(&queue->tx_ptr_lock, flags); + + /* This is a hard error, log it. */ + if (CIRC_SPACE(queue->tx_head, queue->tx_tail, + bp->tx_ring_size) < desc_cnt) { + netif_stop_subqueue(dev, queue_index); + netdev_dbg(bp->dev, "tx_head = %u, tx_tail = %u\n", + queue->tx_head, queue->tx_tail); + ret = NETDEV_TX_BUSY; + goto unlock; + } + + /* Map socket buffer for DMA transfer */ + if (!macb_tx_map(bp, queue, skb, hdrlen)) { + dev_kfree_skb_any(skb); + goto unlock; + } + + /* Make newly initialized descriptor visible to hardware */ + wmb(); + skb_tx_timestamp(skb); + + spin_lock(&bp->lock); + macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART)); + spin_unlock(&bp->lock); + + if (CIRC_SPACE(queue->tx_head, queue->tx_tail, bp->tx_ring_size) < 1) + netif_stop_subqueue(dev, queue_index); + +unlock: + spin_unlock_irqrestore(&queue->tx_ptr_lock, flags); + + return ret; +} + +static void macb_init_rx_buffer_size(struct macb *bp, size_t size) +{ + if (!macb_is_gem(bp)) { + bp->rx_buffer_size = MACB_RX_BUFFER_SIZE; + } else { + bp->rx_buffer_size = size; + + if (bp->rx_buffer_size % RX_BUFFER_MULTIPLE) { + netdev_dbg(bp->dev, + "RX buffer must be multiple of %d bytes, expanding\n", + RX_BUFFER_MULTIPLE); + bp->rx_buffer_size = + roundup(bp->rx_buffer_size, RX_BUFFER_MULTIPLE); + } + } + + netdev_dbg(bp->dev, "mtu [%u] rx_buffer_size [%zu]\n", + bp->dev->mtu, bp->rx_buffer_size); +} + +static void gem_free_rx_buffers(struct macb *bp) +{ + struct sk_buff *skb; + struct macb_dma_desc *desc; + struct macb_queue *queue; + dma_addr_t addr; + unsigned int q; + int i; + + for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { + if (!queue->rx_skbuff) + continue; + + for (i = 0; i < bp->rx_ring_size; i++) { + skb = queue->rx_skbuff[i]; + + if (!skb) + continue; + + desc = macb_rx_desc(queue, i); + addr = macb_get_addr(bp, desc); + + dma_unmap_single(&bp->pdev->dev, addr, bp->rx_buffer_size, + DMA_FROM_DEVICE); + dev_kfree_skb_any(skb); + skb = NULL; + } + + kfree(queue->rx_skbuff); + queue->rx_skbuff = NULL; + } +} + +static void macb_free_rx_buffers(struct macb *bp) +{ + struct macb_queue *queue = &bp->queues[0]; + + if (queue->rx_buffers) { + dma_free_coherent(&bp->pdev->dev, + bp->rx_ring_size * bp->rx_buffer_size, + queue->rx_buffers, queue->rx_buffers_dma); + queue->rx_buffers = NULL; + } +} + +static void macb_free_consistent(struct macb *bp) +{ + struct macb_dma_desc *tx_ring_base = NULL; + struct macb_dma_desc *rx_ring_base = NULL; + dma_addr_t tx_ring_base_addr; + dma_addr_t rx_ring_base_addr; + struct macb_queue *queue; + unsigned int q; + int size; + + if (bp->rx_ring_tieoff) { + dma_free_coherent(&bp->pdev->dev, macb_dma_desc_get_size(bp), + bp->rx_ring_tieoff, bp->rx_ring_tieoff_dma); + bp->rx_ring_tieoff = NULL; + } + + bp->macbgem_ops.mog_free_rx_buffers(bp); + + queue = bp->queues; + if (queue->tx_ring) { + tx_ring_base = queue->tx_ring; + tx_ring_base_addr = queue->tx_ring_dma; + } + if (queue->rx_ring) { + rx_ring_base = queue->rx_ring; + rx_ring_base_addr = queue->rx_ring_dma; + } + + for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { + kfree(queue->tx_skb); + queue->tx_skb = NULL; + if (queue->tx_ring) + queue->tx_ring = NULL; + if (queue->rx_ring) + queue->rx_ring = NULL; + } + + if (tx_ring_base) { + size = bp->num_queues * (TX_RING_BYTES(bp) + + bp->tx_bd_rd_prefetch + + RING_ADDR_INTERVAL); + dma_free_coherent(&bp->pdev->dev, size, tx_ring_base, + tx_ring_base_addr); + } + if (rx_ring_base) { + size = bp->num_queues * (RX_RING_BYTES(bp) + + bp->rx_bd_rd_prefetch + + RING_ADDR_INTERVAL); + dma_free_coherent(&bp->pdev->dev, size, rx_ring_base, + rx_ring_base_addr); + } +} + +static int gem_alloc_rx_buffers(struct macb *bp) +{ + struct macb_queue *queue; + unsigned int q; + int size; + + for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { + size = bp->rx_ring_size * sizeof(struct sk_buff *); + queue->rx_skbuff = kzalloc(size, GFP_KERNEL); + if (!queue->rx_skbuff) + return -ENOMEM; + else + netdev_dbg(bp->dev, + "Allocated %d RX struct sk_buff entries at %p\n", + bp->rx_ring_size, queue->rx_skbuff); + } + return 0; +} + +static int macb_alloc_rx_buffers(struct macb *bp) +{ + struct macb_queue *queue = &bp->queues[0]; + int size; + + size = bp->rx_ring_size * bp->rx_buffer_size; + queue->rx_buffers = dma_alloc_coherent(&bp->pdev->dev, size, + &queue->rx_buffers_dma, GFP_KERNEL); + if (!queue->rx_buffers) + return -ENOMEM; + + netdev_dbg(bp->dev, + "Allocated RX buffers of %d bytes at %08lx (mapped %p)\n", + size, (unsigned long)queue->rx_buffers_dma, queue->rx_buffers); + return 0; +} + +static int macb_queue_phyaddr_check(struct macb *bp, dma_addr_t ring_base_addr, + int offset) +{ + u32 bus_addr_high; + int i; + + bus_addr_high = upper_32_bits(ring_base_addr); + for (i = 1; i < bp->num_queues; i++) { + ring_base_addr += offset; + if (bus_addr_high != upper_32_bits(ring_base_addr)) + return -1; + } + + return 0; +} + +static int macb_alloc_consistent(struct macb *bp) +{ + struct macb_dma_desc *tx_ring_base, *rx_ring_base; + dma_addr_t tx_ring_base_addr, rx_ring_base_addr; + struct macb_queue *queue; + int tx_offset, rx_offset; + int tx_size, rx_size; + unsigned int q; + int ret, i; + int size; + + tx_offset = TX_RING_BYTES(bp) + bp->tx_bd_rd_prefetch + + RING_ADDR_INTERVAL; + tx_size = bp->num_queues * tx_offset; + for (i = 0; i < MAX_RING_ADDR_ALLOC_TIMES + 1; i++) { + if (i == MAX_RING_ADDR_ALLOC_TIMES) + return -ENOMEM; + + tx_ring_base = dma_alloc_coherent(&bp->pdev->dev, tx_size, + &tx_ring_base_addr, + GFP_KERNEL); + if (!tx_ring_base) + continue; + + ret = macb_queue_phyaddr_check(bp, tx_ring_base_addr, + tx_offset); + if (ret) { + dma_free_coherent(&bp->pdev->dev, tx_size, tx_ring_base, + tx_ring_base_addr); + continue; + } else { + break; + } + } + + rx_offset = RX_RING_BYTES(bp) + bp->rx_bd_rd_prefetch + + RING_ADDR_INTERVAL; + rx_size = bp->num_queues * rx_offset; + for (i = 0; i < MAX_RING_ADDR_ALLOC_TIMES + 1; i++) { + if (i == MAX_RING_ADDR_ALLOC_TIMES) { + dma_free_coherent(&bp->pdev->dev, tx_size, tx_ring_base, + tx_ring_base_addr); + return -ENOMEM; + } + + rx_ring_base = dma_alloc_coherent(&bp->pdev->dev, rx_size, + &rx_ring_base_addr, + GFP_KERNEL); + if (!rx_ring_base) + continue; + + ret = macb_queue_phyaddr_check(bp, rx_ring_base_addr, + rx_offset); + if (ret) { + dma_free_coherent(&bp->pdev->dev, rx_size, rx_ring_base, + rx_ring_base_addr); + continue; + } else { + break; + } + } + + for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { + queue->tx_ring = (void *)tx_ring_base + q * tx_offset; + queue->tx_ring_dma = tx_ring_base_addr + q * tx_offset; + if (!queue->tx_ring) + goto out_err; + netdev_dbg(bp->dev, + "Allocated TX ring for queue %u of %d bytes at %08lx (mapped %p)\n", + q, size, (unsigned long)queue->tx_ring_dma, + queue->tx_ring); + + size = bp->tx_ring_size * sizeof(struct macb_tx_skb); + queue->tx_skb = kzalloc(size, GFP_KERNEL); + if (!queue->tx_skb) + goto out_err; + + queue->rx_ring = (void *)rx_ring_base + q * rx_offset; + queue->rx_ring_dma = rx_ring_base_addr + q * rx_offset; + if (!queue->rx_ring) + goto out_err; + netdev_dbg(bp->dev, + "Allocated RX ring of %d bytes at %08lx (mapped %p)\n", + size, (unsigned long)queue->rx_ring_dma, queue->rx_ring); + } + if (bp->macbgem_ops.mog_alloc_rx_buffers(bp)) + goto out_err; + + /* Required for tie off descriptor for PM cases */ + if (!(bp->caps & MACB_CAPS_QUEUE_DISABLE)) { + bp->rx_ring_tieoff = dma_alloc_coherent(&bp->pdev->dev, + macb_dma_desc_get_size(bp), + &bp->rx_ring_tieoff_dma, + GFP_KERNEL); + if (!bp->rx_ring_tieoff) + goto out_err; + } + + return 0; + +out_err: + macb_free_consistent(bp); + return -ENOMEM; +} + +static void macb_init_tieoff(struct macb *bp) +{ + struct macb_dma_desc *desc = bp->rx_ring_tieoff; + + if (bp->caps & MACB_CAPS_QUEUE_DISABLE) + return; + /* Setup a wrapping descriptor with no free slots + * (WRAP and USED) to tie off/disable unused RX queues. + */ + macb_set_addr(bp, desc, MACB_BIT(RX_WRAP) | MACB_BIT(RX_USED)); + desc->ctrl = 0; +} + +static void gem_init_rx_ring(struct macb_queue *queue) +{ + queue->rx_tail = 0; + queue->rx_prepared_head = 0; + + gem_rx_refill(queue); +} + +static void gem_init_rings(struct macb *bp) +{ + struct macb_queue *queue; + struct macb_dma_desc *desc = NULL; + unsigned int q; + int i; + + for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { + for (i = 0; i < bp->tx_ring_size; i++) { + desc = macb_tx_desc(queue, i); + macb_set_addr(bp, desc, 0); + desc->ctrl = MACB_BIT(TX_USED); + } + desc->ctrl |= MACB_BIT(TX_WRAP); + queue->tx_head = 0; + queue->tx_tail = 0; + + for (i = 0; i < bp->rx_ring_size; i++) { + desc = macb_rx_desc(queue, i); + desc->ctrl = 0; + /* make sure ctrl is cleared first, + * and bit RX_USED is set to avoid a race. + */ + dma_wmb(); + desc->addr |= MACB_BIT(RX_USED); + } + + gem_init_rx_ring(queue); + } + + macb_init_tieoff(bp); +} + +static void macb_init_rings(struct macb *bp) +{ + int i; + struct macb_dma_desc *desc = NULL; + + macb_init_rx_ring(&bp->queues[0]); + + for (i = 0; i < bp->tx_ring_size; i++) { + desc = macb_tx_desc(&bp->queues[0], i); + macb_set_addr(bp, desc, 0); + desc->ctrl = MACB_BIT(TX_USED); + } + bp->queues[0].tx_head = 0; + bp->queues[0].tx_tail = 0; + desc->ctrl |= MACB_BIT(TX_WRAP); + + macb_init_tieoff(bp); +} + +static void macb_reset_hw(struct macb *bp) +{ + struct macb_queue *queue; + unsigned int q; + u32 ctrl = macb_readl(bp, NCR); + + /* Disable RX and TX (XXX: Should we halt the transmission + * more gracefully?) + */ + ctrl &= ~(MACB_BIT(RE) | MACB_BIT(TE)); + + /* Clear the stats registers (XXX: Update stats first?) */ + ctrl |= MACB_BIT(CLRSTAT); + + macb_writel(bp, NCR, ctrl); + + /* Clear all status flags */ + macb_writel(bp, TSR, -1); + macb_writel(bp, RSR, -1); + + /* Disable RX partial store and forward and reset watermark value */ + gem_writel(bp, PBUFRXCUT, 0); + + /* Disable all interrupts */ + for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { + queue_writel(queue, IDR, -1); + queue_readl(queue, ISR); + if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) + queue_writel(queue, ISR, -1); + } +} + +static u32 gem_mdc_clk_div(struct macb *bp) +{ + u32 config; + unsigned long pclk_hz = clk_get_rate(bp->pclk); + + if (pclk_hz <= 20000000) + config = GEM_BF(CLK, GEM_CLK_DIV8); + else if (pclk_hz <= 40000000) + config = GEM_BF(CLK, GEM_CLK_DIV16); + else if (pclk_hz <= 80000000) + config = GEM_BF(CLK, GEM_CLK_DIV32); + else if (pclk_hz <= 120000000) + config = GEM_BF(CLK, GEM_CLK_DIV48); + else if (pclk_hz <= 160000000) + config = GEM_BF(CLK, GEM_CLK_DIV64); + else if (pclk_hz <= 240000000) + config = GEM_BF(CLK, GEM_CLK_DIV96); + else if (pclk_hz <= 320000000) + config = GEM_BF(CLK, GEM_CLK_DIV128); + else + config = GEM_BF(CLK, GEM_CLK_DIV224); + + return config; +} + +static u32 macb_mdc_clk_div(struct macb *bp) +{ + u32 config; + unsigned long pclk_hz; + + if (macb_is_gem(bp)) + return gem_mdc_clk_div(bp); + + pclk_hz = clk_get_rate(bp->pclk); + if (pclk_hz <= 20000000) + config = MACB_BF(CLK, MACB_CLK_DIV8); + else if (pclk_hz <= 40000000) + config = MACB_BF(CLK, MACB_CLK_DIV16); + else if (pclk_hz <= 80000000) + config = MACB_BF(CLK, MACB_CLK_DIV32); + else + config = MACB_BF(CLK, MACB_CLK_DIV64); + + return config; +} + +/* Get the DMA bus width field of the network configuration register that we + * should program. We find the width from decoding the design configuration + * register to find the maximum supported data bus width. + */ +static u32 macb_dbw(struct macb *bp) +{ + if (!macb_is_gem(bp)) + return 0; + + switch (GEM_BFEXT(DBWDEF, gem_readl(bp, DCFG1))) { + case 4: + return GEM_BF(DBW, GEM_DBW128); + case 2: + return GEM_BF(DBW, GEM_DBW64); + case 1: + default: + return GEM_BF(DBW, GEM_DBW32); + } +} + +/* Configure the receive DMA engine + * - use the correct receive buffer size + * - set best burst length for DMA operations + * (if not supported by FIFO, it will fallback to default) + * - set both rx/tx packet buffers to full memory size + * These are configurable parameters for GEM. + */ +static void macb_configure_dma(struct macb *bp) +{ + struct macb_queue *queue; + u32 buffer_size; + unsigned int q; + u32 dmacfg; + + buffer_size = bp->rx_buffer_size / RX_BUFFER_MULTIPLE; + if (macb_is_gem(bp)) { + dmacfg = gem_readl(bp, DMACFG) & ~GEM_BF(RXBS, -1L); + for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { + if (q) + queue_writel(queue, RBQS, buffer_size); + else + dmacfg |= GEM_BF(RXBS, buffer_size); + } + if (bp->dma_burst_length) + dmacfg = GEM_BFINS(FBLDO, bp->dma_burst_length, dmacfg); + dmacfg |= GEM_BIT(TXPBMS) | GEM_BF(RXBMS, -1L); + dmacfg &= ~GEM_BIT(ENDIA_PKT); + + if (bp->native_io) + dmacfg &= ~GEM_BIT(ENDIA_DESC); + else + dmacfg |= GEM_BIT(ENDIA_DESC); /* CPU in big endian */ + + if (bp->dev->features & NETIF_F_HW_CSUM) + dmacfg |= GEM_BIT(TXCOEN); + else + dmacfg &= ~GEM_BIT(TXCOEN); + + dmacfg &= ~GEM_BIT(ADDR64); +#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT + if (bp->hw_dma_cap & HW_DMA_CAP_64B) + dmacfg |= GEM_BIT(ADDR64); +#endif +#ifdef CONFIG_MACB_USE_HWSTAMP + if (bp->hw_dma_cap & HW_DMA_CAP_PTP) + dmacfg |= GEM_BIT(RXEXT) | GEM_BIT(TXEXT); +#endif + netdev_dbg(bp->dev, "Cadence configure DMA with 0x%08x\n", + dmacfg); + gem_writel(bp, DMACFG, dmacfg); + } +} + +static int phytium_mac_config(struct macb *bp) +{ + u32 old_ctrl, ctrl; + u32 old_ncr, ncr; + + netdev_dbg(bp->dev, "phytium mac config"); + + ncr = macb_readl(bp, NCR); + old_ncr = ncr; + ctrl = macb_or_gem_readl(bp, NCFGR); + old_ctrl = ctrl; + + ncr &= ~(GEM_BIT(ENABLE_HS_MAC) | MACB_BIT(2PT5G)); + ctrl &= ~(GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL) | + MACB_BIT(SPD) | MACB_BIT(FD)); + if (macb_is_gem(bp)) + ctrl &= ~GEM_BIT(GBE); + + if (bp->phy_interface == PHY_INTERFACE_MODE_2500BASEX) { + ctrl |= GEM_BIT(PCSSEL) | GEM_BIT(SGMIIEN); + ncr |= MACB_BIT(2PT5G); + } else if (bp->phy_interface == PHY_INTERFACE_MODE_USXGMII || + bp->phy_interface == PHY_INTERFACE_MODE_5GBASER) { + ctrl |= GEM_BIT(PCSSEL); + ncr |= GEM_BIT(ENABLE_HS_MAC); + } + + if (bp->duplex) + ctrl |= MACB_BIT(FD); + + /* Apply the new configuration, if any */ + if (old_ctrl ^ ctrl) + macb_or_gem_writel(bp, NCFGR, ctrl); + + if (old_ncr ^ ncr) + macb_or_gem_writel(bp, NCR, ncr); + + return 0; +} + +static void macb_init_hw(struct macb *bp) +{ + u32 config; + + macb_reset_hw(bp); + macb_set_hwaddr(bp); + + config = macb_readl(bp, NCR); + config |= MACB_BIT(MPE); + macb_writel(bp, NCR, config); + + config = macb_mdc_clk_div(bp); + config |= MACB_BF(RBOF, NET_IP_ALIGN); /* Make eth data aligned */ + config |= MACB_BIT(DRFCS); /* Discard Rx FCS */ + if (bp->caps & MACB_CAPS_JUMBO) + config |= MACB_BIT(JFRAME); /* Enable jumbo frames */ + else + config |= MACB_BIT(BIG); /* Receive oversized frames */ + if (bp->dev->flags & IFF_PROMISC) + config |= MACB_BIT(CAF); /* Copy All Frames */ + else if (macb_is_gem(bp) && bp->dev->features & NETIF_F_RXCSUM) + config |= GEM_BIT(RXCOEN); + if (!(bp->dev->flags & IFF_BROADCAST)) + config |= MACB_BIT(NBC); /* No BroadCast */ + config |= macb_dbw(bp); + macb_writel(bp, NCFGR, config); + if ((bp->caps & MACB_CAPS_JUMBO) && bp->jumbo_max_len) + gem_writel(bp, JML, bp->jumbo_max_len); + bp->rx_frm_len_mask = MACB_RX_FRMLEN_MASK; + if (bp->caps & MACB_CAPS_JUMBO) + bp->rx_frm_len_mask = MACB_RX_JFRMLEN_MASK; + + gem_writel(bp, AXI_PIPE, 0x1010); + + if (bp->phy_interface == PHY_INTERFACE_MODE_USXGMII || + bp->phy_interface == PHY_INTERFACE_MODE_5GBASER || + bp->phy_interface == PHY_INTERFACE_MODE_2500BASEX) { + /* phytium need hwclock */ + if (bp->caps & MACB_CAPS_SEL_CLK) + bp->sel_clk_hw(bp, bp->speed); + phytium_mac_config(bp); + if (bp->link) + macb_usx_pcs_link_up(&bp->phylink_usx_pcs, 0, + bp->phy_interface, bp->speed, bp->duplex); + } else { + bp->speed = SPEED_10; + bp->duplex = DUPLEX_HALF; + } + + macb_configure_dma(bp); + + /* Enable RX partial store and forward and set watermark */ + if (bp->rx_watermark) + gem_writel(bp, PBUFRXCUT, (bp->rx_watermark | GEM_BIT(ENCUTTHRU))); +} + +/* The hash address register is 64 bits long and takes up two + * locations in the memory map. The least significant bits are stored + * in EMAC_HSL and the most significant bits in EMAC_HSH. + * + * The unicast hash enable and the multicast hash enable bits in the + * network configuration register enable the reception of hash matched + * frames. The destination address is reduced to a 6 bit index into + * the 64 bit hash register using the following hash function. The + * hash function is an exclusive or of every sixth bit of the + * destination address. + * + * hi[5] = da[5] ^ da[11] ^ da[17] ^ da[23] ^ da[29] ^ da[35] ^ da[41] ^ da[47] + * hi[4] = da[4] ^ da[10] ^ da[16] ^ da[22] ^ da[28] ^ da[34] ^ da[40] ^ da[46] + * hi[3] = da[3] ^ da[09] ^ da[15] ^ da[21] ^ da[27] ^ da[33] ^ da[39] ^ da[45] + * hi[2] = da[2] ^ da[08] ^ da[14] ^ da[20] ^ da[26] ^ da[32] ^ da[38] ^ da[44] + * hi[1] = da[1] ^ da[07] ^ da[13] ^ da[19] ^ da[25] ^ da[31] ^ da[37] ^ da[43] + * hi[0] = da[0] ^ da[06] ^ da[12] ^ da[18] ^ da[24] ^ da[30] ^ da[36] ^ da[42] + * + * da[0] represents the least significant bit of the first byte + * received, that is, the multicast/unicast indicator, and da[47] + * represents the most significant bit of the last byte received. If + * the hash index, hi[n], points to a bit that is set in the hash + * register then the frame will be matched according to whether the + * frame is multicast or unicast. A multicast match will be signalled + * if the multicast hash enable bit is set, da[0] is 1 and the hash + * index points to a bit set in the hash register. A unicast match + * will be signalled if the unicast hash enable bit is set, da[0] is 0 + * and the hash index points to a bit set in the hash register. To + * receive all multicast frames, the hash register should be set with + * all ones and the multicast hash enable bit should be set in the + * network configuration register. + */ + +static inline int hash_bit_value(int bitnr, __u8 *addr) +{ + if (addr[bitnr / 8] & (1 << (bitnr % 8))) + return 1; + return 0; +} + +/* Return the hash index value for the specified address. */ +static int hash_get_index(__u8 *addr) +{ + int i, j, bitval; + int hash_index = 0; + + for (j = 0; j < 6; j++) { + for (i = 0, bitval = 0; i < 8; i++) + bitval ^= hash_bit_value(i * 6 + j, addr); + + hash_index |= (bitval << j); + } + + return hash_index; +} + +/* Add multicast addresses to the internal multicast-hash table. */ +static void macb_sethashtable(struct net_device *dev) +{ + struct netdev_hw_addr *ha; + unsigned long mc_filter[2]; + unsigned int bitnr; + struct macb *bp = netdev_priv(dev); + + mc_filter[0] = 0; + mc_filter[1] = 0; + + netdev_for_each_mc_addr(ha, dev) { + bitnr = hash_get_index(ha->addr); + mc_filter[bitnr >> 5] |= 1 << (bitnr & 31); + } + + macb_or_gem_writel(bp, HRB, mc_filter[0]); + macb_or_gem_writel(bp, HRT, mc_filter[1]); +} + +/* Enable/Disable promiscuous and multicast modes. */ +static void macb_set_rx_mode(struct net_device *dev) +{ + unsigned long cfg; + struct macb *bp = netdev_priv(dev); + + cfg = macb_readl(bp, NCFGR); + + if (dev->flags & IFF_PROMISC) { + /* Enable promiscuous mode */ + cfg |= MACB_BIT(CAF); + + /* Disable RX checksum offload */ + if (macb_is_gem(bp)) + cfg &= ~GEM_BIT(RXCOEN); + } else { + /* Disable promiscuous mode */ + cfg &= ~MACB_BIT(CAF); + + /* Enable RX checksum offload only if requested */ + if (macb_is_gem(bp) && dev->features & NETIF_F_RXCSUM) + cfg |= GEM_BIT(RXCOEN); + } + + if (dev->flags & IFF_ALLMULTI) { + /* Enable all multicast mode */ + macb_or_gem_writel(bp, HRB, -1); + macb_or_gem_writel(bp, HRT, -1); + cfg |= MACB_BIT(NCFGR_MTI); + } else if (!netdev_mc_empty(dev)) { + /* Enable specific multicasts */ + macb_sethashtable(dev); + cfg |= MACB_BIT(NCFGR_MTI); + } else if (dev->flags & (~IFF_ALLMULTI)) { + /* Disable all multicast mode */ + macb_or_gem_writel(bp, HRB, 0); + macb_or_gem_writel(bp, HRT, 0); + cfg &= ~MACB_BIT(NCFGR_MTI); + } + + macb_writel(bp, NCFGR, cfg); +} + +static int macb_open(struct net_device *dev) +{ + size_t bufsz = dev->mtu + ETH_HLEN + ETH_FCS_LEN + NET_IP_ALIGN; + struct macb *bp = netdev_priv(dev); + struct macb_queue *queue; + unsigned int q; + int err; + + netdev_dbg(bp->dev, "open\n"); + + err = pm_runtime_resume_and_get(&bp->pdev->dev); + if (err < 0) + return err; + + /* RX buffers initialization */ + macb_init_rx_buffer_size(bp, bufsz); + + err = macb_alloc_consistent(bp); + if (err) { + netdev_err(dev, "Unable to allocate DMA memory (error %d)\n", + err); + goto pm_exit; + } + + bp->macbgem_ops.mog_init_rings(bp); + macb_init_buffers(bp); + + for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { + napi_enable(&queue->napi_rx); + napi_enable(&queue->napi_tx); + } + + macb_init_hw(bp); + + err = phy_power_on(bp->sgmii_phy); + if (err) + goto reset_hw; + + err = macb_phylink_connect(bp); + if (err) + goto phy_off; + + netif_tx_start_all_queues(dev); + + if (bp->ptp_info) + bp->ptp_info->ptp_init(dev); + + return 0; + +phy_off: + phy_power_off(bp->sgmii_phy); + +reset_hw: + macb_reset_hw(bp); + for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { + napi_disable(&queue->napi_rx); + napi_disable(&queue->napi_tx); + } + macb_free_consistent(bp); +pm_exit: + pm_runtime_put_sync(&bp->pdev->dev); + return err; +} + +static int macb_close(struct net_device *dev) +{ + struct macb *bp = netdev_priv(dev); + struct macb_queue *queue; + unsigned long flags; + unsigned int q; + + netif_tx_stop_all_queues(dev); + + for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { + napi_disable(&queue->napi_rx); + napi_disable(&queue->napi_tx); + } + + phylink_stop(bp->phylink); + phylink_disconnect_phy(bp->phylink); + + phy_power_off(bp->sgmii_phy); + + spin_lock_irqsave(&bp->lock, flags); + macb_reset_hw(bp); + netif_carrier_off(dev); + spin_unlock_irqrestore(&bp->lock, flags); + + macb_free_consistent(bp); + + if (bp->ptp_info) + bp->ptp_info->ptp_remove(dev); + + pm_runtime_put(&bp->pdev->dev); + + return 0; +} + +static int macb_change_mtu(struct net_device *dev, int new_mtu) +{ + if (netif_running(dev)) + return -EBUSY; + + dev->mtu = new_mtu; + + return 0; +} + +static int macb_set_mac_addr(struct net_device *dev, void *addr) +{ + int err; + + err = eth_mac_addr(dev, addr); + if (err < 0) + return err; + + macb_set_hwaddr(netdev_priv(dev)); + return 0; +} + +static void gem_update_stats(struct macb *bp) +{ + struct macb_queue *queue; + unsigned int i, q, idx; + unsigned long *stat; + + u32 *p = &bp->hw_stats.gem.tx_octets_31_0; + + for (i = 0; i < GEM_STATS_LEN; ++i, ++p) { + u32 offset = gem_statistics[i].offset; + u64 val = bp->macb_reg_readl(bp, offset); + + bp->ethtool_stats[i] += val; + *p += val; + + if (offset == GEM_OCTTXL || offset == GEM_OCTRXL) { + /* Add GEM_OCTTXH, GEM_OCTRXH */ + val = bp->macb_reg_readl(bp, offset + 4); + bp->ethtool_stats[i] += ((u64)val) << 32; + *(++p) += val; + } + } + + idx = GEM_STATS_LEN; + for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) + for (i = 0, stat = &queue->stats.first; i < QUEUE_STATS_LEN; ++i, ++stat) + bp->ethtool_stats[idx++] = *stat; +} + +static struct net_device_stats *gem_get_stats(struct macb *bp) +{ + struct gem_stats *hwstat = &bp->hw_stats.gem; + struct net_device_stats *nstat = &bp->dev->stats; + + if (!netif_running(bp->dev)) + return nstat; + + spin_lock_irq(&bp->stats_lock); + gem_update_stats(bp); + + nstat->rx_errors = (hwstat->rx_frame_check_sequence_errors + + hwstat->rx_alignment_errors + + hwstat->rx_resource_errors + + hwstat->rx_overruns + + hwstat->rx_oversize_frames + + hwstat->rx_jabbers + + hwstat->rx_undersized_frames + + hwstat->rx_length_field_frame_errors); + nstat->tx_errors = (hwstat->tx_late_collisions + + hwstat->tx_excessive_collisions + + hwstat->tx_underrun + + hwstat->tx_carrier_sense_errors); + nstat->multicast = hwstat->rx_multicast_frames; + nstat->collisions = (hwstat->tx_single_collision_frames + + hwstat->tx_multiple_collision_frames + + hwstat->tx_excessive_collisions); + nstat->rx_length_errors = (hwstat->rx_oversize_frames + + hwstat->rx_jabbers + + hwstat->rx_undersized_frames + + hwstat->rx_length_field_frame_errors); + nstat->rx_over_errors = hwstat->rx_resource_errors; + nstat->rx_crc_errors = hwstat->rx_frame_check_sequence_errors; + nstat->rx_frame_errors = hwstat->rx_alignment_errors; + nstat->rx_fifo_errors = hwstat->rx_overruns; + nstat->tx_aborted_errors = hwstat->tx_excessive_collisions; + nstat->tx_carrier_errors = hwstat->tx_carrier_sense_errors; + nstat->tx_fifo_errors = hwstat->tx_underrun; + spin_unlock_irq(&bp->stats_lock); + + return nstat; +} + +static void gem_get_ethtool_stats(struct net_device *dev, + struct ethtool_stats *stats, u64 *data) +{ + struct macb *bp = netdev_priv(dev); + + spin_lock_irq(&bp->stats_lock); + gem_update_stats(bp); + memcpy(data, &bp->ethtool_stats, sizeof(u64) + * (GEM_STATS_LEN + QUEUE_STATS_LEN * bp->num_queues)); + spin_unlock_irq(&bp->stats_lock); +} + +static int gem_get_sset_count(struct net_device *dev, int sset) +{ + struct macb *bp = netdev_priv(dev); + + switch (sset) { + case ETH_SS_STATS: + return GEM_STATS_LEN + bp->num_queues * QUEUE_STATS_LEN; + default: + return -EOPNOTSUPP; + } +} + +static void gem_get_ethtool_strings(struct net_device *dev, u32 sset, u8 *p) +{ + char stat_string[ETH_GSTRING_LEN]; + struct macb *bp = netdev_priv(dev); + struct macb_queue *queue; + unsigned int i; + unsigned int q; + + switch (sset) { + case ETH_SS_STATS: + for (i = 0; i < GEM_STATS_LEN; i++, p += ETH_GSTRING_LEN) + memcpy(p, gem_statistics[i].stat_string, + ETH_GSTRING_LEN); + + for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { + for (i = 0; i < QUEUE_STATS_LEN; i++, p += ETH_GSTRING_LEN) { + snprintf(stat_string, ETH_GSTRING_LEN, "q%d_%s", + q, queue_statistics[i].stat_string); + memcpy(p, stat_string, ETH_GSTRING_LEN); + } + } + break; + } +} + +static struct net_device_stats *macb_get_stats(struct net_device *dev) +{ + struct macb *bp = netdev_priv(dev); + struct net_device_stats *nstat = &bp->dev->stats; + struct macb_stats *hwstat = &bp->hw_stats.macb; + + if (macb_is_gem(bp)) + return gem_get_stats(bp); + + /* read stats from hardware */ + spin_lock_irq(&bp->stats_lock); + macb_update_stats(bp); + + /* Convert HW stats into netdevice stats */ + nstat->rx_errors = (hwstat->rx_fcs_errors + + hwstat->rx_align_errors + + hwstat->rx_resource_errors + + hwstat->rx_overruns + + hwstat->rx_oversize_pkts + + hwstat->rx_jabbers + + hwstat->rx_undersize_pkts + + hwstat->rx_length_mismatch); + nstat->tx_errors = (hwstat->tx_late_cols + + hwstat->tx_excessive_cols + + hwstat->tx_underruns + + hwstat->tx_carrier_errors + + hwstat->sqe_test_errors); + nstat->collisions = (hwstat->tx_single_cols + + hwstat->tx_multiple_cols + + hwstat->tx_excessive_cols); + nstat->rx_length_errors = (hwstat->rx_oversize_pkts + + hwstat->rx_jabbers + + hwstat->rx_undersize_pkts + + hwstat->rx_length_mismatch); + nstat->rx_over_errors = hwstat->rx_resource_errors + + hwstat->rx_overruns; + nstat->rx_crc_errors = hwstat->rx_fcs_errors; + nstat->rx_frame_errors = hwstat->rx_align_errors; + nstat->rx_fifo_errors = hwstat->rx_overruns; + /* XXX: What does "missed" mean? */ + nstat->tx_aborted_errors = hwstat->tx_excessive_cols; + nstat->tx_carrier_errors = hwstat->tx_carrier_errors; + nstat->tx_fifo_errors = hwstat->tx_underruns; + /* Don't know about heartbeat or window errors... */ + spin_unlock_irq(&bp->stats_lock); + + return nstat; +} + +static int macb_get_regs_len(struct net_device *netdev) +{ + return MACB_GREGS_NBR * sizeof(u32); +} + +static void macb_get_regs(struct net_device *dev, struct ethtool_regs *regs, + void *p) +{ + struct macb *bp = netdev_priv(dev); + unsigned int tail, head; + u32 *regs_buff = p; + + regs->version = (macb_readl(bp, MID) & ((1 << MACB_REV_SIZE) - 1)) + | MACB_GREGS_VERSION; + + tail = macb_tx_ring_wrap(bp, bp->queues[0].tx_tail); + head = macb_tx_ring_wrap(bp, bp->queues[0].tx_head); + + regs_buff[0] = macb_readl(bp, NCR); + regs_buff[1] = macb_or_gem_readl(bp, NCFGR); + regs_buff[2] = macb_readl(bp, NSR); + regs_buff[3] = macb_readl(bp, TSR); + regs_buff[4] = macb_readl(bp, RBQP); + regs_buff[5] = macb_readl(bp, TBQP); + regs_buff[6] = macb_readl(bp, RSR); + regs_buff[7] = macb_readl(bp, IMR); + + regs_buff[8] = tail; + regs_buff[9] = head; + regs_buff[10] = macb_tx_dma(&bp->queues[0], tail); + regs_buff[11] = macb_tx_dma(&bp->queues[0], head); + + if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) + regs_buff[12] = macb_or_gem_readl(bp, USRIO); + if (macb_is_gem(bp)) + regs_buff[13] = gem_readl(bp, DMACFG); +} + +static void macb_get_wol(struct net_device *netdev, struct ethtool_wolinfo *wol) +{ + struct macb *bp = netdev_priv(netdev); + + if (bp->wol & MACB_WOL_HAS_MAGIC_PACKET) { + phylink_ethtool_get_wol(bp->phylink, wol); + wol->supported |= WAKE_MAGIC; + + if (bp->wol & MACB_WOL_ENABLED) + wol->wolopts |= WAKE_MAGIC; + } +} + +static int macb_set_wol(struct net_device *netdev, struct ethtool_wolinfo *wol) +{ + struct macb *bp = netdev_priv(netdev); + int ret; + + /* Pass the order to phylink layer */ + ret = phylink_ethtool_set_wol(bp->phylink, wol); + /* Don't manage WoL on MAC if handled by the PHY + * or if there's a failure in talking to the PHY + */ + if (!ret || ret != -EOPNOTSUPP) + return ret; + + if (!(bp->wol & MACB_WOL_HAS_MAGIC_PACKET) || + (wol->wolopts & ~WAKE_MAGIC)) + return -EOPNOTSUPP; + + if (wol->wolopts & WAKE_MAGIC) + bp->wol |= MACB_WOL_ENABLED; + else + bp->wol &= ~MACB_WOL_ENABLED; + + device_set_wakeup_enable(&bp->pdev->dev, bp->wol & MACB_WOL_ENABLED); + + return 0; +} + +static int macb_get_link_ksettings(struct net_device *netdev, + struct ethtool_link_ksettings *kset) +{ + struct macb *bp = netdev_priv(netdev); + + return phylink_ethtool_ksettings_get(bp->phylink, kset); +} + +static int macb_set_link_ksettings(struct net_device *netdev, + const struct ethtool_link_ksettings *kset) +{ + struct macb *bp = netdev_priv(netdev); + + return phylink_ethtool_ksettings_set(bp->phylink, kset); +} + +static void macb_get_ringparam(struct net_device *netdev, + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) +{ + struct macb *bp = netdev_priv(netdev); + + ring->rx_max_pending = MAX_RX_RING_SIZE; + ring->tx_max_pending = MAX_TX_RING_SIZE; + + ring->rx_pending = bp->rx_ring_size; + ring->tx_pending = bp->tx_ring_size; +} + +static int macb_set_ringparam(struct net_device *netdev, + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) +{ + struct macb *bp = netdev_priv(netdev); + u32 new_rx_size, new_tx_size; + unsigned int reset = 0; + + if ((ring->rx_mini_pending) || (ring->rx_jumbo_pending)) + return -EINVAL; + + new_rx_size = clamp_t(u32, ring->rx_pending, + MIN_RX_RING_SIZE, MAX_RX_RING_SIZE); + new_rx_size = roundup_pow_of_two(new_rx_size); + + new_tx_size = clamp_t(u32, ring->tx_pending, + MIN_TX_RING_SIZE, MAX_TX_RING_SIZE); + new_tx_size = roundup_pow_of_two(new_tx_size); + + if ((new_tx_size == bp->tx_ring_size) && + (new_rx_size == bp->rx_ring_size)) { + /* nothing to do */ + return 0; + } + + if (netif_running(bp->dev)) { + reset = 1; + macb_close(bp->dev); + } + + bp->rx_ring_size = new_rx_size; + bp->tx_ring_size = new_tx_size; + + if (reset) + macb_open(bp->dev); + + return 0; +} + +#ifdef CONFIG_MACB_USE_HWSTAMP +static unsigned int gem_get_tsu_rate(struct macb *bp) +{ + struct clk *tsu_clk; + unsigned int tsu_rate; + + tsu_clk = devm_clk_get(&bp->pdev->dev, "tsu_clk"); + if (!IS_ERR(tsu_clk)) + tsu_rate = clk_get_rate(tsu_clk); + /* try pclk instead */ + else if (!IS_ERR(bp->pclk)) { + tsu_clk = bp->pclk; + tsu_rate = clk_get_rate(tsu_clk); + } else + return -ENOTSUPP; + return tsu_rate; +} + +static s32 gem_get_ptp_max_adj(void) +{ + return 64000000; +} + +static int gem_get_ts_info(struct net_device *dev, + struct ethtool_ts_info *info) +{ + struct macb *bp = netdev_priv(dev); + + if ((bp->hw_dma_cap & HW_DMA_CAP_PTP) == 0) { + ethtool_op_get_ts_info(dev, info); + return 0; + } + + info->so_timestamping = + SOF_TIMESTAMPING_TX_SOFTWARE | + SOF_TIMESTAMPING_RX_SOFTWARE | + SOF_TIMESTAMPING_SOFTWARE | + SOF_TIMESTAMPING_TX_HARDWARE | + SOF_TIMESTAMPING_RX_HARDWARE | + SOF_TIMESTAMPING_RAW_HARDWARE; + info->tx_types = + (1 << HWTSTAMP_TX_ONESTEP_SYNC) | + (1 << HWTSTAMP_TX_OFF) | + (1 << HWTSTAMP_TX_ON); + info->rx_filters = + (1 << HWTSTAMP_FILTER_NONE) | + (1 << HWTSTAMP_FILTER_ALL); + + info->phc_index = bp->ptp_clock ? ptp_clock_index(bp->ptp_clock) : -1; + + return 0; +} + +static struct macb_ptp_info gem_ptp_info = { + .ptp_init = gem_ptp_init, + .ptp_remove = gem_ptp_remove, + .get_ptp_max_adj = gem_get_ptp_max_adj, + .get_tsu_rate = gem_get_tsu_rate, + .get_ts_info = gem_get_ts_info, + .get_hwtst = gem_get_hwtst, + .set_hwtst = gem_set_hwtst, +}; +#endif + +static int macb_get_ts_info(struct net_device *netdev, + struct ethtool_ts_info *info) +{ + struct macb *bp = netdev_priv(netdev); + + if (bp->ptp_info) + return bp->ptp_info->get_ts_info(netdev, info); + + return ethtool_op_get_ts_info(netdev, info); +} + +static void gem_enable_flow_filters(struct macb *bp, bool enable) +{ + struct net_device *netdev = bp->dev; + struct ethtool_rx_fs_item *item; + u32 t2_scr; + int num_t2_scr; + + if (!(netdev->features & NETIF_F_NTUPLE)) + return; + + num_t2_scr = GEM_BFEXT(T2SCR, gem_readl(bp, DCFG8)); + + list_for_each_entry(item, &bp->rx_fs_list.list, list) { + struct ethtool_rx_flow_spec *fs = &item->fs; + struct ethtool_tcpip4_spec *tp4sp_m; + + if (fs->location >= num_t2_scr) + continue; + + t2_scr = gem_readl_n(bp, SCRT2, fs->location); + + /* enable/disable screener regs for the flow entry */ + t2_scr = GEM_BFINS(ETHTEN, enable, t2_scr); + + /* only enable fields with no masking */ + tp4sp_m = &(fs->m_u.tcp_ip4_spec); + + if (enable && (tp4sp_m->ip4src == 0xFFFFFFFF)) + t2_scr = GEM_BFINS(CMPAEN, 1, t2_scr); + else + t2_scr = GEM_BFINS(CMPAEN, 0, t2_scr); + + if (enable && (tp4sp_m->ip4dst == 0xFFFFFFFF)) + t2_scr = GEM_BFINS(CMPBEN, 1, t2_scr); + else + t2_scr = GEM_BFINS(CMPBEN, 0, t2_scr); + + if (enable && ((tp4sp_m->psrc == 0xFFFF) || (tp4sp_m->pdst == 0xFFFF))) + t2_scr = GEM_BFINS(CMPCEN, 1, t2_scr); + else + t2_scr = GEM_BFINS(CMPCEN, 0, t2_scr); + + gem_writel_n(bp, SCRT2, fs->location, t2_scr); + } +} + +static void gem_prog_cmp_regs(struct macb *bp, struct ethtool_rx_flow_spec *fs) +{ + struct ethtool_tcpip4_spec *tp4sp_v, *tp4sp_m; + uint16_t index = fs->location; + u32 w0, w1, t2_scr; + bool cmp_a = false; + bool cmp_b = false; + bool cmp_c = false; + + if (!macb_is_gem(bp)) + return; + + tp4sp_v = &(fs->h_u.tcp_ip4_spec); + tp4sp_m = &(fs->m_u.tcp_ip4_spec); + + /* ignore field if any masking set */ + if (tp4sp_m->ip4src == 0xFFFFFFFF) { + /* 1st compare reg - IP source address */ + w0 = 0; + w1 = 0; + w0 = tp4sp_v->ip4src; + w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */ + w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_ETYPE, w1); + w1 = GEM_BFINS(T2OFST, ETYPE_SRCIP_OFFSET, w1); + gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_IP4SRC_CMP(index)), w0); + gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_IP4SRC_CMP(index)), w1); + cmp_a = true; + } + + /* ignore field if any masking set */ + if (tp4sp_m->ip4dst == 0xFFFFFFFF) { + /* 2nd compare reg - IP destination address */ + w0 = 0; + w1 = 0; + w0 = tp4sp_v->ip4dst; + w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */ + w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_ETYPE, w1); + w1 = GEM_BFINS(T2OFST, ETYPE_DSTIP_OFFSET, w1); + gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_IP4DST_CMP(index)), w0); + gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_IP4DST_CMP(index)), w1); + cmp_b = true; + } + + /* ignore both port fields if masking set in both */ + if ((tp4sp_m->psrc == 0xFFFF) || (tp4sp_m->pdst == 0xFFFF)) { + /* 3rd compare reg - source port, destination port */ + w0 = 0; + w1 = 0; + w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_IPHDR, w1); + if (tp4sp_m->psrc == tp4sp_m->pdst) { + w0 = GEM_BFINS(T2MASK, tp4sp_v->psrc, w0); + w0 = GEM_BFINS(T2CMP, tp4sp_v->pdst, w0); + w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */ + w1 = GEM_BFINS(T2OFST, IPHDR_SRCPORT_OFFSET, w1); + } else { + /* only one port definition */ + w1 = GEM_BFINS(T2DISMSK, 0, w1); /* 16-bit compare */ + w0 = GEM_BFINS(T2MASK, 0xFFFF, w0); + if (tp4sp_m->psrc == 0xFFFF) { /* src port */ + w0 = GEM_BFINS(T2CMP, tp4sp_v->psrc, w0); + w1 = GEM_BFINS(T2OFST, IPHDR_SRCPORT_OFFSET, w1); + } else { /* dst port */ + w0 = GEM_BFINS(T2CMP, tp4sp_v->pdst, w0); + w1 = GEM_BFINS(T2OFST, IPHDR_DSTPORT_OFFSET, w1); + } + } + gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_PORT_CMP(index)), w0); + gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_PORT_CMP(index)), w1); + cmp_c = true; + } + + t2_scr = 0; + t2_scr = GEM_BFINS(QUEUE, (fs->ring_cookie) & 0xFF, t2_scr); + t2_scr = GEM_BFINS(ETHT2IDX, SCRT2_ETHT, t2_scr); + if (cmp_a) + t2_scr = GEM_BFINS(CMPA, GEM_IP4SRC_CMP(index), t2_scr); + if (cmp_b) + t2_scr = GEM_BFINS(CMPB, GEM_IP4DST_CMP(index), t2_scr); + if (cmp_c) + t2_scr = GEM_BFINS(CMPC, GEM_PORT_CMP(index), t2_scr); + gem_writel_n(bp, SCRT2, index, t2_scr); +} + +static int gem_add_flow_filter(struct net_device *netdev, + struct ethtool_rxnfc *cmd) +{ + struct macb *bp = netdev_priv(netdev); + struct ethtool_rx_flow_spec *fs = &cmd->fs; + struct ethtool_rx_fs_item *item, *newfs; + unsigned long flags; + int ret = -EINVAL; + bool added = false; + + newfs = kmalloc(sizeof(*newfs), GFP_KERNEL); + if (newfs == NULL) + return -ENOMEM; + memcpy(&newfs->fs, fs, sizeof(newfs->fs)); + + netdev_dbg(netdev, + "Adding flow filter entry,type=%u,queue=%u,loc=%u,src=%08X,dst=%08X,ps=%u,pd=%u\n", + fs->flow_type, (int)fs->ring_cookie, fs->location, + htonl(fs->h_u.tcp_ip4_spec.ip4src), + htonl(fs->h_u.tcp_ip4_spec.ip4dst), + be16_to_cpu(fs->h_u.tcp_ip4_spec.psrc), + be16_to_cpu(fs->h_u.tcp_ip4_spec.pdst)); + + spin_lock_irqsave(&bp->rx_fs_lock, flags); + + /* find correct place to add in list */ + list_for_each_entry(item, &bp->rx_fs_list.list, list) { + if (item->fs.location > newfs->fs.location) { + list_add_tail(&newfs->list, &item->list); + added = true; + break; + } else if (item->fs.location == fs->location) { + netdev_err(netdev, "Rule not added: location %d not free!\n", + fs->location); + ret = -EBUSY; + goto err; + } + } + if (!added) + list_add_tail(&newfs->list, &bp->rx_fs_list.list); + + gem_prog_cmp_regs(bp, fs); + bp->rx_fs_list.count++; + /* enable filtering if NTUPLE on */ + gem_enable_flow_filters(bp, 1); + + spin_unlock_irqrestore(&bp->rx_fs_lock, flags); + return 0; + +err: + spin_unlock_irqrestore(&bp->rx_fs_lock, flags); + kfree(newfs); + return ret; +} + +static int gem_del_flow_filter(struct net_device *netdev, + struct ethtool_rxnfc *cmd) +{ + struct macb *bp = netdev_priv(netdev); + struct ethtool_rx_fs_item *item; + struct ethtool_rx_flow_spec *fs; + unsigned long flags; + + spin_lock_irqsave(&bp->rx_fs_lock, flags); + + list_for_each_entry(item, &bp->rx_fs_list.list, list) { + if (item->fs.location == cmd->fs.location) { + /* disable screener regs for the flow entry */ + fs = &(item->fs); + netdev_dbg(netdev, + "Deleting flow filter entry,type=%u,queue=%u,loc=%u,src=%08X,dst=%08X,ps=%u,pd=%u\n", + fs->flow_type, (int)fs->ring_cookie, fs->location, + htonl(fs->h_u.tcp_ip4_spec.ip4src), + htonl(fs->h_u.tcp_ip4_spec.ip4dst), + be16_to_cpu(fs->h_u.tcp_ip4_spec.psrc), + be16_to_cpu(fs->h_u.tcp_ip4_spec.pdst)); + + gem_writel_n(bp, SCRT2, fs->location, 0); + + list_del(&item->list); + bp->rx_fs_list.count--; + spin_unlock_irqrestore(&bp->rx_fs_lock, flags); + kfree(item); + return 0; + } + } + + spin_unlock_irqrestore(&bp->rx_fs_lock, flags); + return -EINVAL; +} + +static int gem_get_flow_entry(struct net_device *netdev, + struct ethtool_rxnfc *cmd) +{ + struct macb *bp = netdev_priv(netdev); + struct ethtool_rx_fs_item *item; + + list_for_each_entry(item, &bp->rx_fs_list.list, list) { + if (item->fs.location == cmd->fs.location) { + memcpy(&cmd->fs, &item->fs, sizeof(cmd->fs)); + return 0; + } + } + return -EINVAL; +} + +static int gem_get_all_flow_entries(struct net_device *netdev, + struct ethtool_rxnfc *cmd, u32 *rule_locs) +{ + struct macb *bp = netdev_priv(netdev); + struct ethtool_rx_fs_item *item; + uint32_t cnt = 0; + + list_for_each_entry(item, &bp->rx_fs_list.list, list) { + if (cnt == cmd->rule_cnt) + return -EMSGSIZE; + rule_locs[cnt] = item->fs.location; + cnt++; + } + cmd->data = bp->max_tuples; + cmd->rule_cnt = cnt; + + return 0; +} + +static int gem_get_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd, + u32 *rule_locs) +{ + struct macb *bp = netdev_priv(netdev); + int ret = 0; + + switch (cmd->cmd) { + case ETHTOOL_GRXRINGS: + cmd->data = bp->num_queues; + break; + case ETHTOOL_GRXCLSRLCNT: + cmd->rule_cnt = bp->rx_fs_list.count; + break; + case ETHTOOL_GRXCLSRULE: + ret = gem_get_flow_entry(netdev, cmd); + break; + case ETHTOOL_GRXCLSRLALL: + ret = gem_get_all_flow_entries(netdev, cmd, rule_locs); + break; + default: + netdev_err(netdev, + "Command parameter %d is not supported\n", cmd->cmd); + ret = -EOPNOTSUPP; + } + + return ret; +} + +static int gem_set_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd) +{ + struct macb *bp = netdev_priv(netdev); + int ret; + + if (!(netdev->hw_features & NETIF_F_NTUPLE)) + return -EOPNOTSUPP; + + switch (cmd->cmd) { + case ETHTOOL_SRXCLSRLINS: + if ((cmd->fs.location >= bp->max_tuples) + || (cmd->fs.ring_cookie >= bp->num_queues)) { + ret = -EINVAL; + break; + } + ret = gem_add_flow_filter(netdev, cmd); + break; + case ETHTOOL_SRXCLSRLDEL: + ret = gem_del_flow_filter(netdev, cmd); + break; + default: + netdev_err(netdev, + "Command parameter %d is not supported\n", cmd->cmd); + ret = -EOPNOTSUPP; + } + + return ret; +} + +static const struct ethtool_ops macb_ethtool_ops = { + .get_regs_len = macb_get_regs_len, + .get_regs = macb_get_regs, + .get_link = ethtool_op_get_link, + .get_ts_info = ethtool_op_get_ts_info, + .get_wol = macb_get_wol, + .set_wol = macb_set_wol, + .get_link_ksettings = macb_get_link_ksettings, + .set_link_ksettings = macb_set_link_ksettings, + .get_ringparam = macb_get_ringparam, + .set_ringparam = macb_set_ringparam, +}; + +static const struct ethtool_ops gem_ethtool_ops = { + .get_regs_len = macb_get_regs_len, + .get_regs = macb_get_regs, + .get_wol = macb_get_wol, + .set_wol = macb_set_wol, + .get_link = ethtool_op_get_link, + .get_ts_info = macb_get_ts_info, + .get_ethtool_stats = gem_get_ethtool_stats, + .get_strings = gem_get_ethtool_strings, + .get_sset_count = gem_get_sset_count, + .get_link_ksettings = macb_get_link_ksettings, + .set_link_ksettings = macb_set_link_ksettings, + .get_ringparam = macb_get_ringparam, + .set_ringparam = macb_set_ringparam, + .get_rxnfc = gem_get_rxnfc, + .set_rxnfc = gem_set_rxnfc, +}; + +static int macb_ioctl(struct net_device *dev, struct ifreq *rq, int cmd) +{ + struct macb *bp = netdev_priv(dev); + + if (!netif_running(dev)) + return -EINVAL; + + if (bp->ptp_info) { + switch (cmd) { + case SIOCSHWTSTAMP: + return bp->ptp_info->set_hwtst(dev, rq, cmd); + case SIOCGHWTSTAMP: + return bp->ptp_info->get_hwtst(dev, rq); + } + } + + return phylink_mii_ioctl(bp->phylink, rq, cmd); +} + +static inline void macb_set_txcsum_feature(struct macb *bp, + netdev_features_t features) +{ + u32 val; + + if (!macb_is_gem(bp)) + return; + + val = gem_readl(bp, DMACFG); + if (features & NETIF_F_HW_CSUM) + val |= GEM_BIT(TXCOEN); + else + val &= ~GEM_BIT(TXCOEN); + + gem_writel(bp, DMACFG, val); +} + +static inline void macb_set_rxcsum_feature(struct macb *bp, + netdev_features_t features) +{ + struct net_device *netdev = bp->dev; + u32 val; + + if (!macb_is_gem(bp)) + return; + + val = gem_readl(bp, NCFGR); + if ((features & NETIF_F_RXCSUM) && !(netdev->flags & IFF_PROMISC)) + val |= GEM_BIT(RXCOEN); + else + val &= ~GEM_BIT(RXCOEN); + + gem_writel(bp, NCFGR, val); +} + +static inline void macb_set_rxflow_feature(struct macb *bp, + netdev_features_t features) +{ + if (!macb_is_gem(bp)) + return; + + gem_enable_flow_filters(bp, !!(features & NETIF_F_NTUPLE)); +} + +static int macb_set_features(struct net_device *netdev, + netdev_features_t features) +{ + struct macb *bp = netdev_priv(netdev); + netdev_features_t changed = features ^ netdev->features; + + /* TX checksum offload */ + if (changed & NETIF_F_HW_CSUM) + macb_set_txcsum_feature(bp, features); + + /* RX checksum offload */ + if (changed & NETIF_F_RXCSUM) + macb_set_rxcsum_feature(bp, features); + + /* RX Flow Filters */ + if (changed & NETIF_F_NTUPLE) + macb_set_rxflow_feature(bp, features); + + return 0; +} + +static void macb_restore_features(struct macb *bp) +{ + struct net_device *netdev = bp->dev; + netdev_features_t features = netdev->features; + struct ethtool_rx_fs_item *item; + + /* TX checksum offload */ + macb_set_txcsum_feature(bp, features); + + /* RX checksum offload */ + macb_set_rxcsum_feature(bp, features); + + /* RX Flow Filters */ + list_for_each_entry(item, &bp->rx_fs_list.list, list) + gem_prog_cmp_regs(bp, &item->fs); + + macb_set_rxflow_feature(bp, features); +} + +static const struct net_device_ops macb_netdev_ops = { + .ndo_open = macb_open, + .ndo_stop = macb_close, + .ndo_start_xmit = macb_start_xmit, + .ndo_set_rx_mode = macb_set_rx_mode, + .ndo_get_stats = macb_get_stats, + .ndo_eth_ioctl = macb_ioctl, + .ndo_validate_addr = eth_validate_addr, + .ndo_change_mtu = macb_change_mtu, + .ndo_set_mac_address = macb_set_mac_addr, +#ifdef CONFIG_NET_POLL_CONTROLLER + .ndo_poll_controller = macb_poll_controller, +#endif + .ndo_set_features = macb_set_features, + .ndo_features_check = macb_features_check, + .ndo_vlan_rx_add_vid = ncsi_vlan_rx_add_vid, + .ndo_vlan_rx_kill_vid = ncsi_vlan_rx_kill_vid, +}; + +/* Configure peripheral capabilities according to device tree + * and integration options used + */ +static void macb_configure_caps(struct macb *bp, + const struct macb_config *dt_conf) +{ + u32 dcfg; + + if (dt_conf) + bp->caps = dt_conf->caps; + + if (hw_is_gem(bp->regs, bp->native_io)) { + bp->caps |= MACB_CAPS_MACB_IS_GEM; + + dcfg = gem_readl(bp, DCFG1); + if (GEM_BFEXT(IRQCOR, dcfg) == 0) + bp->caps |= MACB_CAPS_ISR_CLEAR_ON_WRITE; + if (GEM_BFEXT(NO_PCS, dcfg) == 0) + bp->caps |= MACB_CAPS_PCS; + dcfg = gem_readl(bp, DCFG12); + if (GEM_BFEXT(HIGH_SPEED, dcfg) == 1) + bp->caps |= MACB_CAPS_HIGH_SPEED; + dcfg = gem_readl(bp, DCFG2); + if ((dcfg & (GEM_BIT(RX_PKT_BUFF) | GEM_BIT(TX_PKT_BUFF))) == 0) + bp->caps |= MACB_CAPS_FIFO_MODE; + if (gem_has_ptp(bp)) { + if (!GEM_BFEXT(TSU, gem_readl(bp, DCFG5))) + dev_err(&bp->pdev->dev, + "GEM doesn't support hardware ptp.\n"); + else { +#ifdef CONFIG_MACB_USE_HWSTAMP + bp->hw_dma_cap |= HW_DMA_CAP_PTP; + bp->ptp_info = &gem_ptp_info; +#endif + } + } + } + + dev_dbg(&bp->pdev->dev, "Cadence caps 0x%08x\n", bp->caps); +} + +static void macb_probe_queues(void __iomem *mem, + bool native_io, + unsigned int *queue_mask, + unsigned int *num_queues) +{ + *queue_mask = 0x1; + *num_queues = 1; + + /* is it macb or gem ? + * + * We need to read directly from the hardware here because + * we are early in the probe process and don't have the + * MACB_CAPS_MACB_IS_GEM flag positioned + */ + if (!hw_is_gem(mem, native_io)) + return; + + /* bit 0 is never set but queue 0 always exists */ + *queue_mask |= readl_relaxed(mem + GEM_DCFG6) & 0xff; + *num_queues = hweight32(*queue_mask); +} + +static void macb_clks_disable(struct clk *pclk, struct clk *hclk, struct clk *tx_clk, + struct clk *rx_clk, struct clk *tsu_clk) +{ + struct clk_bulk_data clks[] = { + { .clk = tsu_clk, }, + { .clk = rx_clk, }, + { .clk = pclk, }, + { .clk = hclk, }, + { .clk = tx_clk }, + }; + + clk_bulk_disable_unprepare(ARRAY_SIZE(clks), clks); +} + +static int macb_clk_init(struct platform_device *pdev, struct clk **pclk, + struct clk **hclk, struct clk **tx_clk, + struct clk **rx_clk, struct clk **tsu_clk) +{ + struct macb_platform_data *pdata; + int err; + + pdata = dev_get_platdata(&pdev->dev); + if (pdata) { + *pclk = pdata->pclk; + *hclk = pdata->hclk; + } else { + *pclk = devm_clk_get(&pdev->dev, "pclk"); + *hclk = devm_clk_get(&pdev->dev, "hclk"); + } + + if (IS_ERR_OR_NULL(*pclk)) + return dev_err_probe(&pdev->dev, + IS_ERR(*pclk) ? PTR_ERR(*pclk) : -ENODEV, + "failed to get pclk\n"); + + if (IS_ERR_OR_NULL(*hclk)) + return dev_err_probe(&pdev->dev, + IS_ERR(*hclk) ? PTR_ERR(*hclk) : -ENODEV, + "failed to get hclk\n"); + + *tx_clk = devm_clk_get_optional(&pdev->dev, "tx_clk"); + if (IS_ERR(*tx_clk)) + return PTR_ERR(*tx_clk); + + *rx_clk = devm_clk_get_optional(&pdev->dev, "rx_clk"); + if (IS_ERR(*rx_clk)) + return PTR_ERR(*rx_clk); + + *tsu_clk = devm_clk_get_optional(&pdev->dev, "tsu_clk"); + if (IS_ERR(*tsu_clk)) + return PTR_ERR(*tsu_clk); + + err = clk_prepare_enable(*pclk); + if (err) { + dev_err(&pdev->dev, "failed to enable pclk (%d)\n", err); + return err; + } + + err = clk_prepare_enable(*hclk); + if (err) { + dev_err(&pdev->dev, "failed to enable hclk (%d)\n", err); + goto err_disable_pclk; + } + + err = clk_prepare_enable(*tx_clk); + if (err) { + dev_err(&pdev->dev, "failed to enable tx_clk (%d)\n", err); + goto err_disable_hclk; + } + + err = clk_prepare_enable(*rx_clk); + if (err) { + dev_err(&pdev->dev, "failed to enable rx_clk (%d)\n", err); + goto err_disable_txclk; + } + + err = clk_prepare_enable(*tsu_clk); + if (err) { + dev_err(&pdev->dev, "failed to enable tsu_clk (%d)\n", err); + goto err_disable_rxclk; + } + + return 0; + +err_disable_rxclk: + clk_disable_unprepare(*rx_clk); + +err_disable_txclk: + clk_disable_unprepare(*tx_clk); + +err_disable_hclk: + clk_disable_unprepare(*hclk); + +err_disable_pclk: + clk_disable_unprepare(*pclk); + + return err; +} + +static int phytium_clk_init(struct platform_device *pdev, struct clk **pclk, + struct clk **hclk, struct clk **tx_clk, + struct clk **rx_clk, struct clk **tsu_clk) +{ + struct macb_platform_data *pdata; + struct device_node *np = pdev->dev.of_node; + int err; + + pdata = dev_get_platdata(&pdev->dev); + if (pdata) { + *pclk = pdata->pclk; + *hclk = pdata->hclk; + } else { + if (has_acpi_companion(&pdev->dev)) { + *pclk = NULL; + *hclk = NULL; + } else if (np) { + *pclk = devm_clk_get(&pdev->dev, "pclk"); + *hclk = devm_clk_get(&pdev->dev, "hclk"); + } + } + + if (IS_ERR(*pclk)) + return dev_err_probe(&pdev->dev, + IS_ERR(*pclk) ? PTR_ERR(*pclk) : -ENODEV, + "failed to get pclk\n"); + + if (IS_ERR(*hclk)) + return dev_err_probe(&pdev->dev, + IS_ERR(*hclk) ? PTR_ERR(*hclk) : -ENODEV, + "failed to get hclk\n"); + + *tx_clk = devm_clk_get_optional(&pdev->dev, "tx_clk"); + if (IS_ERR(*tx_clk)) + return PTR_ERR(*tx_clk); + + *rx_clk = devm_clk_get_optional(&pdev->dev, "rx_clk"); + if (IS_ERR(*rx_clk)) + return PTR_ERR(*rx_clk); + + *tsu_clk = devm_clk_get_optional(&pdev->dev, "tsu_clk"); + if (IS_ERR(*tsu_clk)) + return PTR_ERR(*tsu_clk); + + err = clk_prepare_enable(*pclk); + if (err) { + dev_err(&pdev->dev, "failed to enable pclk (%d)\n", err); + return err; + } + + err = clk_prepare_enable(*hclk); + if (err) { + dev_err(&pdev->dev, "failed to enable hclk (%d)\n", err); + goto err_disable_pclk; + } + + err = clk_prepare_enable(*tx_clk); + if (err) { + dev_err(&pdev->dev, "failed to enable tx_clk (%d)\n", err); + goto err_disable_hclk; + } + + err = clk_prepare_enable(*rx_clk); + if (err) { + dev_err(&pdev->dev, "failed to enable rx_clk (%d)\n", err); + goto err_disable_txclk; + } + + err = clk_prepare_enable(*tsu_clk); + if (err) { + dev_err(&pdev->dev, "failed to enable tsu_clk (%d)\n", err); + goto err_disable_rxclk; + } + + return 0; + +err_disable_rxclk: + clk_disable_unprepare(*rx_clk); + +err_disable_txclk: + clk_disable_unprepare(*tx_clk); + +err_disable_hclk: + clk_disable_unprepare(*hclk); + +err_disable_pclk: + clk_disable_unprepare(*pclk); + + return err; +} + +static int macb_init(struct platform_device *pdev) +{ + struct net_device *dev = platform_get_drvdata(pdev); + unsigned int hw_q, q; + struct macb *bp = netdev_priv(dev); + struct macb_queue *queue; + int err; + u32 val, reg; + + bp->tx_ring_size = DEFAULT_TX_RING_SIZE; + bp->rx_ring_size = DEFAULT_RX_RING_SIZE; + + /* set the queue register mapping once for all: queue0 has a special + * register mapping but we don't want to test the queue index then + * compute the corresponding register offset at run time. + */ + for (hw_q = 0, q = 0; hw_q < MACB_MAX_QUEUES; ++hw_q) { + if (!(bp->queue_mask & (1 << hw_q))) + continue; + + queue = &bp->queues[q]; + queue->bp = bp; + spin_lock_init(&queue->tx_ptr_lock); + netif_napi_add(dev, &queue->napi_rx, macb_rx_poll); + netif_napi_add(dev, &queue->napi_tx, macb_tx_poll); + if (hw_q) { + queue->ISR = GEM_ISR(hw_q - 1); + queue->IER = GEM_IER(hw_q - 1); + queue->IDR = GEM_IDR(hw_q - 1); + queue->IMR = GEM_IMR(hw_q - 1); + queue->TBQP = GEM_TBQP(hw_q - 1); + queue->RBQP = GEM_RBQP(hw_q - 1); + queue->RBQS = GEM_RBQS(hw_q - 1); +#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT + if (bp->hw_dma_cap & HW_DMA_CAP_64B) { + queue->TBQPH = GEM_TBQPH(hw_q - 1); + queue->RBQPH = GEM_RBQPH(hw_q - 1); + } +#endif + } else { + /* queue0 uses legacy registers */ + queue->ISR = MACB_ISR; + queue->IER = MACB_IER; + queue->IDR = MACB_IDR; + queue->IMR = MACB_IMR; + queue->TBQP = MACB_TBQP; + queue->RBQP = MACB_RBQP; +#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT + if (bp->hw_dma_cap & HW_DMA_CAP_64B) { + queue->TBQPH = MACB_TBQPH; + queue->RBQPH = MACB_RBQPH; + } +#endif + } + + /* get irq: here we use the linux queue index, not the hardware + * queue index. the queue irq definitions in the device tree + * must remove the optional gaps that could exist in the + * hardware queue mask. + */ + queue->irq = platform_get_irq(pdev, q); + err = devm_request_irq(&pdev->dev, queue->irq, macb_interrupt, + IRQF_SHARED, dev->name, queue); + if (err) { + dev_err(&pdev->dev, + "Unable to request IRQ %d (error %d)\n", + queue->irq, err); + return err; + } + + INIT_WORK(&queue->tx_error_task, macb_tx_error_task); + q++; + } + + dev->netdev_ops = &macb_netdev_ops; + + /* setup appropriated routines according to adapter type */ + if (macb_is_gem(bp)) { + bp->max_tx_length = GEM_MAX_TX_LEN; + bp->macbgem_ops.mog_alloc_rx_buffers = gem_alloc_rx_buffers; + bp->macbgem_ops.mog_free_rx_buffers = gem_free_rx_buffers; + bp->macbgem_ops.mog_init_rings = gem_init_rings; + bp->macbgem_ops.mog_rx = gem_rx; + dev->ethtool_ops = &gem_ethtool_ops; + } else { + bp->max_tx_length = MACB_MAX_TX_LEN; + bp->macbgem_ops.mog_alloc_rx_buffers = macb_alloc_rx_buffers; + bp->macbgem_ops.mog_free_rx_buffers = macb_free_rx_buffers; + bp->macbgem_ops.mog_init_rings = macb_init_rings; + bp->macbgem_ops.mog_rx = macb_rx; + dev->ethtool_ops = &macb_ethtool_ops; + } + + dev->priv_flags |= IFF_LIVE_ADDR_CHANGE; + + /* Set features */ + dev->hw_features = NETIF_F_SG; + + /* Check LSO capability */ + if (GEM_BFEXT(PBUF_LSO, gem_readl(bp, DCFG6))) + dev->hw_features |= MACB_NETIF_LSO; + + /* Checksum offload is only available on gem with packet buffer */ + if (macb_is_gem(bp) && !(bp->caps & MACB_CAPS_FIFO_MODE)) + dev->hw_features |= NETIF_F_HW_CSUM | NETIF_F_RXCSUM; + if (bp->caps & MACB_CAPS_SG_DISABLED) + dev->hw_features &= ~NETIF_F_SG; + dev->features = dev->hw_features; + + /* Check RX Flow Filters support. + * Max Rx flows set by availability of screeners & compare regs: + * each 4-tuple define requires 1 T2 screener reg + 3 compare regs + */ + reg = gem_readl(bp, DCFG8); + bp->max_tuples = min((GEM_BFEXT(SCR2CMP, reg) / 3), + GEM_BFEXT(T2SCR, reg)); + INIT_LIST_HEAD(&bp->rx_fs_list.list); + if (bp->max_tuples > 0) { + /* also needs one ethtype match to check IPv4 */ + if (GEM_BFEXT(SCR2ETH, reg) > 0) { + /* program this reg now */ + reg = 0; + reg = GEM_BFINS(ETHTCMP, (uint16_t)ETH_P_IP, reg); + gem_writel_n(bp, ETHT, SCRT2_ETHT, reg); + /* Filtering is supported in hw but don't enable it in kernel now */ + dev->hw_features |= NETIF_F_NTUPLE; + /* init Rx flow definitions */ + bp->rx_fs_list.count = 0; + spin_lock_init(&bp->rx_fs_lock); + } else + bp->max_tuples = 0; + } + + if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) { + val = 0; + if (phy_interface_mode_is_rgmii(bp->phy_interface)) + val = bp->usrio->rgmii; + else if (bp->phy_interface == PHY_INTERFACE_MODE_RMII && + (bp->caps & MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII)) + val = bp->usrio->rmii; + else if (!(bp->caps & MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII)) + val = bp->usrio->mii; + + if (bp->caps & MACB_CAPS_USRIO_HAS_CLKEN) + val |= bp->usrio->refclk; + + macb_or_gem_writel(bp, USRIO, val); + } + + /* Set MII management clock divider */ + val = macb_mdc_clk_div(bp); + val |= macb_dbw(bp); + if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) + val |= GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL); + macb_writel(bp, NCFGR, val); + + return 0; +} + +static const struct macb_usrio_config macb_default_usrio = { + .mii = MACB_BIT(MII), + .rmii = MACB_BIT(RMII), + .rgmii = GEM_BIT(RGMII), + .refclk = MACB_BIT(CLKEN), +}; + +static const struct macb_config phytium_gem1p0_config = { + .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | + MACB_CAPS_JUMBO | + MACB_CAPS_GEM_HAS_PTP | + MACB_CAPS_BD_RD_PREFETCH | + MACB_CAPS_SEL_CLK, + .dma_burst_length = 16, + .clk_init = phytium_clk_init, + .init = macb_init, + .jumbo_max_len = 10240, + .sel_clk_hw = phytium_gem1p0_sel_clk, + .usrio = &macb_default_usrio, +}; + +/* 1518 rounded up */ +#define AT91ETHER_MAX_RBUFF_SZ 0x600 +/* max number of receive buffers */ +#define AT91ETHER_MAX_RX_DESCR 9 + +static struct sifive_fu540_macb_mgmt *mgmt; + +static int at91ether_alloc_coherent(struct macb *lp) +{ + struct macb_queue *q = &lp->queues[0]; + + q->rx_ring = dma_alloc_coherent(&lp->pdev->dev, + (AT91ETHER_MAX_RX_DESCR * + macb_dma_desc_get_size(lp)), + &q->rx_ring_dma, GFP_KERNEL); + if (!q->rx_ring) + return -ENOMEM; + + q->rx_buffers = dma_alloc_coherent(&lp->pdev->dev, + AT91ETHER_MAX_RX_DESCR * + AT91ETHER_MAX_RBUFF_SZ, + &q->rx_buffers_dma, GFP_KERNEL); + if (!q->rx_buffers) { + dma_free_coherent(&lp->pdev->dev, + AT91ETHER_MAX_RX_DESCR * + macb_dma_desc_get_size(lp), + q->rx_ring, q->rx_ring_dma); + q->rx_ring = NULL; + return -ENOMEM; + } + + return 0; +} + +static void at91ether_free_coherent(struct macb *lp) +{ + struct macb_queue *q = &lp->queues[0]; + + if (q->rx_ring) { + dma_free_coherent(&lp->pdev->dev, + AT91ETHER_MAX_RX_DESCR * + macb_dma_desc_get_size(lp), + q->rx_ring, q->rx_ring_dma); + q->rx_ring = NULL; + } + + if (q->rx_buffers) { + dma_free_coherent(&lp->pdev->dev, + AT91ETHER_MAX_RX_DESCR * + AT91ETHER_MAX_RBUFF_SZ, + q->rx_buffers, q->rx_buffers_dma); + q->rx_buffers = NULL; + } +} + +/* Initialize and start the Receiver and Transmit subsystems */ +static int at91ether_start(struct macb *lp) +{ + struct macb_queue *q = &lp->queues[0]; + struct macb_dma_desc *desc; + dma_addr_t addr; + u32 ctl; + int i, ret; + + ret = at91ether_alloc_coherent(lp); + if (ret) + return ret; + + addr = q->rx_buffers_dma; + for (i = 0; i < AT91ETHER_MAX_RX_DESCR; i++) { + desc = macb_rx_desc(q, i); + macb_set_addr(lp, desc, addr); + desc->ctrl = 0; + addr += AT91ETHER_MAX_RBUFF_SZ; + } + + /* Set the Wrap bit on the last descriptor */ + desc->addr |= MACB_BIT(RX_WRAP); + + /* Reset buffer index */ + q->rx_tail = 0; + + /* Program address of descriptor list in Rx Buffer Queue register */ + macb_writel(lp, RBQP, q->rx_ring_dma); + + /* Enable Receive and Transmit */ + ctl = macb_readl(lp, NCR); + macb_writel(lp, NCR, ctl | MACB_BIT(RE) | MACB_BIT(TE)); + + /* Enable MAC interrupts */ + macb_writel(lp, IER, MACB_BIT(RCOMP) | + MACB_BIT(RXUBR) | + MACB_BIT(ISR_TUND) | + MACB_BIT(ISR_RLE) | + MACB_BIT(TCOMP) | + MACB_BIT(ISR_ROVR) | + MACB_BIT(HRESP)); + + return 0; +} + +static void at91ether_stop(struct macb *lp) +{ + u32 ctl; + + /* Disable MAC interrupts */ + macb_writel(lp, IDR, MACB_BIT(RCOMP) | + MACB_BIT(RXUBR) | + MACB_BIT(ISR_TUND) | + MACB_BIT(ISR_RLE) | + MACB_BIT(TCOMP) | + MACB_BIT(ISR_ROVR) | + MACB_BIT(HRESP)); + + /* Disable Receiver and Transmitter */ + ctl = macb_readl(lp, NCR); + macb_writel(lp, NCR, ctl & ~(MACB_BIT(TE) | MACB_BIT(RE))); + + /* Free resources. */ + at91ether_free_coherent(lp); +} + +/* Open the ethernet interface */ +static int at91ether_open(struct net_device *dev) +{ + struct macb *lp = netdev_priv(dev); + u32 ctl; + int ret; + + ret = pm_runtime_resume_and_get(&lp->pdev->dev); + if (ret < 0) + return ret; + + /* Clear internal statistics */ + ctl = macb_readl(lp, NCR); + macb_writel(lp, NCR, ctl | MACB_BIT(CLRSTAT)); + + macb_set_hwaddr(lp); + + ret = at91ether_start(lp); + if (ret) + goto pm_exit; + + ret = macb_phylink_connect(lp); + if (ret) + goto stop; + + netif_start_queue(dev); + + return 0; + +stop: + at91ether_stop(lp); +pm_exit: + pm_runtime_put_sync(&lp->pdev->dev); + return ret; +} + +/* Close the interface */ +static int at91ether_close(struct net_device *dev) +{ + struct macb *lp = netdev_priv(dev); + + netif_stop_queue(dev); + + phylink_stop(lp->phylink); + phylink_disconnect_phy(lp->phylink); + + at91ether_stop(lp); + + return pm_runtime_put(&lp->pdev->dev); +} + +/* Transmit packet */ +static netdev_tx_t at91ether_start_xmit(struct sk_buff *skb, + struct net_device *dev) +{ + struct macb *lp = netdev_priv(dev); + + if (macb_readl(lp, TSR) & MACB_BIT(RM9200_BNQ)) { + int desc = 0; + + netif_stop_queue(dev); + + /* Store packet information (to free when Tx completed) */ + lp->rm9200_txq[desc].skb = skb; + lp->rm9200_txq[desc].size = skb->len; + lp->rm9200_txq[desc].mapping = dma_map_single(&lp->pdev->dev, skb->data, + skb->len, DMA_TO_DEVICE); + if (dma_mapping_error(&lp->pdev->dev, lp->rm9200_txq[desc].mapping)) { + dev_kfree_skb_any(skb); + dev->stats.tx_dropped++; + netdev_err(dev, "%s: DMA mapping error\n", __func__); + return NETDEV_TX_OK; + } + + /* Set address of the data in the Transmit Address register */ + macb_writel(lp, TAR, lp->rm9200_txq[desc].mapping); + /* Set length of the packet in the Transmit Control register */ + macb_writel(lp, TCR, skb->len); + + } else { + netdev_err(dev, "%s called, but device is busy!\n", __func__); + return NETDEV_TX_BUSY; + } + + return NETDEV_TX_OK; +} + +/* Extract received frame from buffer descriptors and sent to upper layers. + * (Called from interrupt context) + */ +static void at91ether_rx(struct net_device *dev) +{ + struct macb *lp = netdev_priv(dev); + struct macb_queue *q = &lp->queues[0]; + struct macb_dma_desc *desc; + unsigned char *p_recv; + struct sk_buff *skb; + unsigned int pktlen; + + desc = macb_rx_desc(q, q->rx_tail); + while (desc->addr & MACB_BIT(RX_USED)) { + p_recv = q->rx_buffers + q->rx_tail * AT91ETHER_MAX_RBUFF_SZ; + pktlen = MACB_BF(RX_FRMLEN, desc->ctrl); + skb = netdev_alloc_skb(dev, pktlen + 2); + if (skb) { + skb_reserve(skb, 2); + skb_put_data(skb, p_recv, pktlen); + + skb->protocol = eth_type_trans(skb, dev); + dev->stats.rx_packets++; + dev->stats.rx_bytes += pktlen; + netif_rx(skb); + } else { + dev->stats.rx_dropped++; + } + + if (desc->ctrl & MACB_BIT(RX_MHASH_MATCH)) + dev->stats.multicast++; + + /* reset ownership bit */ + desc->addr &= ~MACB_BIT(RX_USED); + + /* wrap after last buffer */ + if (q->rx_tail == AT91ETHER_MAX_RX_DESCR - 1) + q->rx_tail = 0; + else + q->rx_tail++; + + desc = macb_rx_desc(q, q->rx_tail); + } +} + +/* MAC interrupt handler */ +static irqreturn_t at91ether_interrupt(int irq, void *dev_id) +{ + struct net_device *dev = dev_id; + struct macb *lp = netdev_priv(dev); + u32 intstatus, ctl; + unsigned int desc; + + /* MAC Interrupt Status register indicates what interrupts are pending. + * It is automatically cleared once read. + */ + intstatus = macb_readl(lp, ISR); + + /* Receive complete */ + if (intstatus & MACB_BIT(RCOMP)) + at91ether_rx(dev); + + /* Transmit complete */ + if (intstatus & MACB_BIT(TCOMP)) { + /* The TCOM bit is set even if the transmission failed */ + if (intstatus & (MACB_BIT(ISR_TUND) | MACB_BIT(ISR_RLE))) + dev->stats.tx_errors++; + + desc = 0; + if (lp->rm9200_txq[desc].skb) { + dev_consume_skb_irq(lp->rm9200_txq[desc].skb); + lp->rm9200_txq[desc].skb = NULL; + dma_unmap_single(&lp->pdev->dev, lp->rm9200_txq[desc].mapping, + lp->rm9200_txq[desc].size, DMA_TO_DEVICE); + dev->stats.tx_packets++; + dev->stats.tx_bytes += lp->rm9200_txq[desc].size; + } + netif_wake_queue(dev); + } + + /* Work-around for EMAC Errata section 41.3.1 */ + if (intstatus & MACB_BIT(RXUBR)) { + ctl = macb_readl(lp, NCR); + macb_writel(lp, NCR, ctl & ~MACB_BIT(RE)); + wmb(); + macb_writel(lp, NCR, ctl | MACB_BIT(RE)); + } + + if (intstatus & MACB_BIT(ISR_ROVR)) + netdev_err(dev, "ROVR error\n"); + + return IRQ_HANDLED; +} + +#ifdef CONFIG_NET_POLL_CONTROLLER +static void at91ether_poll_controller(struct net_device *dev) +{ + unsigned long flags; + + local_irq_save(flags); + at91ether_interrupt(dev->irq, dev); + local_irq_restore(flags); +} +#endif + +static const struct net_device_ops at91ether_netdev_ops = { + .ndo_open = at91ether_open, + .ndo_stop = at91ether_close, + .ndo_start_xmit = at91ether_start_xmit, + .ndo_get_stats = macb_get_stats, + .ndo_set_rx_mode = macb_set_rx_mode, + .ndo_set_mac_address = eth_mac_addr, + .ndo_eth_ioctl = macb_ioctl, + .ndo_validate_addr = eth_validate_addr, +#ifdef CONFIG_NET_POLL_CONTROLLER + .ndo_poll_controller = at91ether_poll_controller, +#endif +}; + +static int at91ether_clk_init(struct platform_device *pdev, struct clk **pclk, + struct clk **hclk, struct clk **tx_clk, + struct clk **rx_clk, struct clk **tsu_clk) +{ + int err; + + *hclk = NULL; + *tx_clk = NULL; + *rx_clk = NULL; + *tsu_clk = NULL; + + *pclk = devm_clk_get(&pdev->dev, "ether_clk"); + if (IS_ERR(*pclk)) + return PTR_ERR(*pclk); + + err = clk_prepare_enable(*pclk); + if (err) { + dev_err(&pdev->dev, "failed to enable pclk (%d)\n", err); + return err; + } + + return 0; +} + +static int at91ether_init(struct platform_device *pdev) +{ + struct net_device *dev = platform_get_drvdata(pdev); + struct macb *bp = netdev_priv(dev); + int err; + + bp->queues[0].bp = bp; + + dev->netdev_ops = &at91ether_netdev_ops; + dev->ethtool_ops = &macb_ethtool_ops; + + err = devm_request_irq(&pdev->dev, dev->irq, at91ether_interrupt, + 0, dev->name, dev); + if (err) + return err; + + macb_writel(bp, NCR, 0); + + macb_writel(bp, NCFGR, MACB_BF(CLK, MACB_CLK_DIV32) | MACB_BIT(BIG)); + + return 0; +} + +static unsigned long fu540_macb_tx_recalc_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + return mgmt->rate; +} + +static long fu540_macb_tx_round_rate(struct clk_hw *hw, unsigned long rate, + unsigned long *parent_rate) +{ + if (WARN_ON(rate < 2500000)) + return 2500000; + else if (rate == 2500000) + return 2500000; + else if (WARN_ON(rate < 13750000)) + return 2500000; + else if (WARN_ON(rate < 25000000)) + return 25000000; + else if (rate == 25000000) + return 25000000; + else if (WARN_ON(rate < 75000000)) + return 25000000; + else if (WARN_ON(rate < 125000000)) + return 125000000; + else if (rate == 125000000) + return 125000000; + + WARN_ON(rate > 125000000); + + return 125000000; +} + +static int fu540_macb_tx_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate) +{ + rate = fu540_macb_tx_round_rate(hw, rate, &parent_rate); + if (rate != 125000000) + iowrite32(1, mgmt->reg); + else + iowrite32(0, mgmt->reg); + mgmt->rate = rate; + + return 0; +} + +static const struct clk_ops fu540_c000_ops = { + .recalc_rate = fu540_macb_tx_recalc_rate, + .round_rate = fu540_macb_tx_round_rate, + .set_rate = fu540_macb_tx_set_rate, +}; + +static int fu540_c000_clk_init(struct platform_device *pdev, struct clk **pclk, + struct clk **hclk, struct clk **tx_clk, + struct clk **rx_clk, struct clk **tsu_clk) +{ + struct clk_init_data init; + int err = 0; + + err = macb_clk_init(pdev, pclk, hclk, tx_clk, rx_clk, tsu_clk); + if (err) + return err; + + mgmt = devm_kzalloc(&pdev->dev, sizeof(*mgmt), GFP_KERNEL); + if (!mgmt) { + err = -ENOMEM; + goto err_disable_clks; + } + + init.name = "sifive-gemgxl-mgmt"; + init.ops = &fu540_c000_ops; + init.flags = 0; + init.num_parents = 0; + + mgmt->rate = 0; + mgmt->hw.init = &init; + + *tx_clk = devm_clk_register(&pdev->dev, &mgmt->hw); + if (IS_ERR(*tx_clk)) { + err = PTR_ERR(*tx_clk); + goto err_disable_clks; + } + + err = clk_prepare_enable(*tx_clk); + if (err) { + dev_err(&pdev->dev, "failed to enable tx_clk (%u)\n", err); + *tx_clk = NULL; + goto err_disable_clks; + } else { + dev_info(&pdev->dev, "Registered clk switch '%s'\n", init.name); + } + + return 0; + +err_disable_clks: + macb_clks_disable(*pclk, *hclk, *tx_clk, *rx_clk, *tsu_clk); + + return err; +} + +static int fu540_c000_init(struct platform_device *pdev) +{ + mgmt->reg = devm_platform_ioremap_resource(pdev, 1); + if (IS_ERR(mgmt->reg)) + return PTR_ERR(mgmt->reg); + + return macb_init(pdev); +} + +static int init_reset_optional(struct platform_device *pdev) +{ + struct net_device *dev = platform_get_drvdata(pdev); + struct macb *bp = netdev_priv(dev); + int ret; + + if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) { + /* Ensure PHY device used in SGMII mode is ready */ + bp->sgmii_phy = devm_phy_optional_get(&pdev->dev, NULL); + + if (IS_ERR(bp->sgmii_phy)) + return dev_err_probe(&pdev->dev, PTR_ERR(bp->sgmii_phy), + "failed to get SGMII PHY\n"); + + ret = phy_init(bp->sgmii_phy); + if (ret) + return dev_err_probe(&pdev->dev, ret, + "failed to init SGMII PHY\n"); + + ret = zynqmp_pm_is_function_supported(PM_IOCTL, IOCTL_SET_GEM_CONFIG); + if (!ret) { + u32 pm_info[2]; + + ret = of_property_read_u32_array(pdev->dev.of_node, "power-domains", + pm_info, ARRAY_SIZE(pm_info)); + if (ret) { + dev_err(&pdev->dev, "Failed to read power management information\n"); + goto err_out_phy_exit; + } + ret = zynqmp_pm_set_gem_config(pm_info[1], GEM_CONFIG_FIXED, 0); + if (ret) + goto err_out_phy_exit; + + ret = zynqmp_pm_set_gem_config(pm_info[1], GEM_CONFIG_SGMII_MODE, 1); + if (ret) + goto err_out_phy_exit; + } + + } + + /* Fully reset controller at hardware level if mapped in device tree */ + ret = device_reset_optional(&pdev->dev); + if (ret) { + phy_exit(bp->sgmii_phy); + return dev_err_probe(&pdev->dev, ret, "failed to reset controller"); + } + + ret = macb_init(pdev); + +err_out_phy_exit: + if (ret) + phy_exit(bp->sgmii_phy); + + return ret; +} + +static const struct macb_usrio_config sama7g5_usrio = { + .mii = 0, + .rmii = 1, + .rgmii = 2, + .refclk = BIT(2), + .hdfctlen = BIT(6), +}; + +static const struct macb_config fu540_c000_config = { + .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_JUMBO | + MACB_CAPS_GEM_HAS_PTP, + .dma_burst_length = 16, + .clk_init = fu540_c000_clk_init, + .init = fu540_c000_init, + .jumbo_max_len = 10240, + .usrio = &macb_default_usrio, +}; + +static const struct macb_config at91sam9260_config = { + .caps = MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, + .clk_init = macb_clk_init, + .init = macb_init, + .usrio = &macb_default_usrio, +}; + +static const struct macb_config sama5d3macb_config = { + .caps = MACB_CAPS_SG_DISABLED | + MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, + .clk_init = macb_clk_init, + .init = macb_init, + .usrio = &macb_default_usrio, +}; + +static const struct macb_config pc302gem_config = { + .caps = MACB_CAPS_SG_DISABLED | MACB_CAPS_GIGABIT_MODE_AVAILABLE, + .dma_burst_length = 16, + .clk_init = macb_clk_init, + .init = macb_init, + .usrio = &macb_default_usrio, +}; + +static const struct macb_config sama5d2_config = { + .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, + .dma_burst_length = 16, + .clk_init = macb_clk_init, + .init = macb_init, + .usrio = &macb_default_usrio, +}; + +static const struct macb_config sama5d29_config = { + .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | MACB_CAPS_GEM_HAS_PTP, + .dma_burst_length = 16, + .clk_init = macb_clk_init, + .init = macb_init, + .usrio = &macb_default_usrio, +}; + +static const struct macb_config sama5d3_config = { + .caps = MACB_CAPS_SG_DISABLED | MACB_CAPS_GIGABIT_MODE_AVAILABLE | + MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | MACB_CAPS_JUMBO, + .dma_burst_length = 16, + .clk_init = macb_clk_init, + .init = macb_init, + .jumbo_max_len = 10240, + .usrio = &macb_default_usrio, +}; + +static const struct macb_config sama5d4_config = { + .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, + .dma_burst_length = 4, + .clk_init = macb_clk_init, + .init = macb_init, + .usrio = &macb_default_usrio, +}; + +static const struct macb_config emac_config = { + .caps = MACB_CAPS_NEEDS_RSTONUBR | MACB_CAPS_MACB_IS_EMAC, + .clk_init = at91ether_clk_init, + .init = at91ether_init, + .usrio = &macb_default_usrio, +}; + +static const struct macb_config np4_config = { + .caps = MACB_CAPS_USRIO_DISABLED, + .clk_init = macb_clk_init, + .init = macb_init, + .usrio = &macb_default_usrio, +}; + +static const struct macb_config zynqmp_config = { + .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | + MACB_CAPS_JUMBO | + MACB_CAPS_GEM_HAS_PTP | MACB_CAPS_BD_RD_PREFETCH, + .dma_burst_length = 16, + .clk_init = macb_clk_init, + .init = init_reset_optional, + .jumbo_max_len = 10240, + .usrio = &macb_default_usrio, +}; + +static const struct macb_config zynq_config = { + .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_NO_GIGABIT_HALF | + MACB_CAPS_NEEDS_RSTONUBR, + .dma_burst_length = 16, + .clk_init = macb_clk_init, + .init = macb_init, + .usrio = &macb_default_usrio, +}; + +static const struct macb_config mpfs_config = { + .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | + MACB_CAPS_JUMBO | + MACB_CAPS_GEM_HAS_PTP, + .dma_burst_length = 16, + .clk_init = macb_clk_init, + .init = init_reset_optional, + .usrio = &macb_default_usrio, + .max_tx_length = 4040, /* Cadence Erratum 1686 */ + .jumbo_max_len = 4040, +}; + +static const struct macb_config sama7g5_gem_config = { + .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_CLK_HW_CHG | + MACB_CAPS_MIIONRGMII | MACB_CAPS_GEM_HAS_PTP, + .dma_burst_length = 16, + .clk_init = macb_clk_init, + .init = macb_init, + .usrio = &sama7g5_usrio, +}; + +static const struct macb_config sama7g5_emac_config = { + .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | + MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_MIIONRGMII | + MACB_CAPS_GEM_HAS_PTP, + .dma_burst_length = 16, + .clk_init = macb_clk_init, + .init = macb_init, + .usrio = &sama7g5_usrio, +}; + +static const struct macb_config versal_config = { + .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_JUMBO | + MACB_CAPS_GEM_HAS_PTP | MACB_CAPS_BD_RD_PREFETCH | MACB_CAPS_NEED_TSUCLK, + .dma_burst_length = 16, + .clk_init = macb_clk_init, + .init = init_reset_optional, + .jumbo_max_len = 10240, + .usrio = &macb_default_usrio, +}; + +static const struct macb_config phytium_gem2p0_config = { + .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | + MACB_CAPS_JUMBO | + MACB_CAPS_GEM_HAS_PTP | + MACB_CAPS_BD_RD_PREFETCH | + MACB_CAPS_SEL_CLK, + .dma_burst_length = 16, + .clk_init = phytium_clk_init, + .init = macb_init, + .jumbo_max_len = 10240, + .sel_clk_hw = phytium_gem2p0_sel_clk, + .usrio = &macb_default_usrio, +}; + +#if defined(CONFIG_OF) +static const struct of_device_id macb_dt_ids[] = { + { .compatible = "cdns,at91sam9260-macb", .data = &at91sam9260_config }, + { .compatible = "cdns,macb" }, + { .compatible = "cdns,np4-macb", .data = &np4_config }, + { .compatible = "cdns,pc302-gem", .data = &pc302gem_config }, + { .compatible = "cdns,gem", .data = &pc302gem_config }, + { .compatible = "cdns,sam9x60-macb", .data = &at91sam9260_config }, + { .compatible = "atmel,sama5d2-gem", .data = &sama5d2_config }, + { .compatible = "atmel,sama5d29-gem", .data = &sama5d29_config }, + { .compatible = "atmel,sama5d3-gem", .data = &sama5d3_config }, + { .compatible = "atmel,sama5d3-macb", .data = &sama5d3macb_config }, + { .compatible = "atmel,sama5d4-gem", .data = &sama5d4_config }, + { .compatible = "cdns,at91rm9200-emac", .data = &emac_config }, + { .compatible = "cdns,emac", .data = &emac_config }, + { .compatible = "cdns,zynqmp-gem", .data = &zynqmp_config}, /* deprecated */ + { .compatible = "cdns,zynq-gem", .data = &zynq_config }, /* deprecated */ + { .compatible = "sifive,fu540-c000-gem", .data = &fu540_c000_config }, + { .compatible = "microchip,mpfs-macb", .data = &mpfs_config }, + { .compatible = "microchip,sama7g5-gem", .data = &sama7g5_gem_config }, + { .compatible = "microchip,sama7g5-emac", .data = &sama7g5_emac_config }, + { .compatible = "xlnx,zynqmp-gem", .data = &zynqmp_config}, + { .compatible = "xlnx,zynq-gem", .data = &zynq_config }, + { .compatible = "xlnx,versal-gem", .data = &versal_config}, + { .compatible = "cdns,phytium-gem-1.0", .data = &phytium_gem1p0_config }, + { .compatible = "cdns,phytium-gem-2.0", .data = &phytium_gem2p0_config }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(of, macb_dt_ids); +#endif /* CONFIG_OF */ + +#ifdef CONFIG_ACPI +static const struct acpi_device_id macb_acpi_ids[] = { + { .id = "PHYT0036", .driver_data = (kernel_ulong_t)&phytium_gem1p0_config }, + { } +}; + +MODULE_DEVICE_TABLE(acpi, macb_acpi_ids); +#else +#define macb_acpi_ids NULL +#endif + +static const struct macb_config default_gem_config = { + .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | + MACB_CAPS_JUMBO | + MACB_CAPS_GEM_HAS_PTP, + .dma_burst_length = 16, + .clk_init = macb_clk_init, + .init = macb_init, + .usrio = &macb_default_usrio, + .jumbo_max_len = 10240, +}; + +static void gem_ncsi_handler(struct ncsi_dev *nd) +{ + if (unlikely(nd->state != ncsi_dev_state_functional)) + return; + + netdev_dbg(nd->dev, "NCSI interface %s\n", + nd->link_up ? "up" : "down"); +} + +static int macb_get_phy_mode(struct platform_device *pdev) +{ + const char *pm; + int err, i; + + err = device_property_read_string(&pdev->dev, "phy-mode", &pm); + if (err < 0) + return err; + + for (i = 0; i < PHY_INTERFACE_MODE_MAX; i++) { + if (!strcasecmp(pm, phy_modes(i))) + return i; + } + + return -ENODEV; +} + +static int macb_probe(struct platform_device *pdev) +{ + const struct macb_config *macb_config = &default_gem_config; + int (*clk_init)(struct platform_device *, struct clk **, + struct clk **, struct clk **, struct clk **, + struct clk **) = macb_config->clk_init; + int (*init)(struct platform_device *) = macb_config->init; + struct device_node *np = pdev->dev.of_node; + struct clk *pclk, *hclk = NULL, *tx_clk = NULL, *rx_clk = NULL; + struct clk *tsu_clk = NULL; + unsigned int queue_mask, num_queues; + bool native_io; + struct net_device *dev; + struct resource *regs; + u32 wtrmrk_rst_val; + void __iomem *mem; + struct macb *bp; + int err, val; + + mem = devm_platform_get_and_ioremap_resource(pdev, 0, ®s); + if (IS_ERR(mem)) + return PTR_ERR(mem); + + if (np) { + const struct of_device_id *match; + + match = of_match_node(macb_dt_ids, np); + if (match && match->data) { + macb_config = match->data; + clk_init = macb_config->clk_init; + init = macb_config->init; + } + } else if (has_acpi_companion(&pdev->dev)) { + const struct acpi_device_id *match; + + match = acpi_match_device(macb_acpi_ids, &pdev->dev); + if (match && match->driver_data) { + macb_config = (void *)match->driver_data; + clk_init = macb_config->clk_init; + init = macb_config->init; + } + } + + err = clk_init(pdev, &pclk, &hclk, &tx_clk, &rx_clk, &tsu_clk); + if (err) + return err; + + pm_runtime_set_autosuspend_delay(&pdev->dev, MACB_PM_TIMEOUT); + pm_runtime_use_autosuspend(&pdev->dev); + pm_runtime_get_noresume(&pdev->dev); + pm_runtime_set_active(&pdev->dev); + pm_runtime_enable(&pdev->dev); + native_io = hw_is_native_io(mem); + + macb_probe_queues(mem, native_io, &queue_mask, &num_queues); + dev = alloc_etherdev_mq(sizeof(*bp), num_queues); + if (!dev) { + err = -ENOMEM; + goto err_disable_clocks; + } + + dev->base_addr = regs->start; + + SET_NETDEV_DEV(dev, &pdev->dev); + + bp = netdev_priv(dev); + bp->pdev = pdev; + bp->dev = dev; + bp->regs = mem; + bp->native_io = native_io; + if (native_io) { + bp->macb_reg_readl = hw_readl_native; + bp->macb_reg_writel = hw_writel_native; + } else { + bp->macb_reg_readl = hw_readl; + bp->macb_reg_writel = hw_writel; + } + bp->num_queues = num_queues; + bp->queue_mask = queue_mask; + if (macb_config) + bp->dma_burst_length = macb_config->dma_burst_length; + bp->pclk = pclk; + bp->hclk = hclk; + bp->tx_clk = tx_clk; + bp->rx_clk = rx_clk; + bp->tsu_clk = tsu_clk; + if (macb_config) + bp->jumbo_max_len = macb_config->jumbo_max_len; + + if (macb_config) + bp->sel_clk_hw = macb_config->sel_clk_hw; + + if (!hw_is_gem(bp->regs, bp->native_io)) + bp->max_tx_length = MACB_MAX_TX_LEN; + else if (macb_config->max_tx_length) + bp->max_tx_length = macb_config->max_tx_length; + else + bp->max_tx_length = GEM_MAX_TX_LEN; + + bp->wol = 0; + if (device_property_read_bool(&pdev->dev, "magic-packet")) + bp->wol |= MACB_WOL_HAS_MAGIC_PACKET; + device_set_wakeup_capable(&pdev->dev, bp->wol & MACB_WOL_HAS_MAGIC_PACKET); + + bp->usrio = macb_config->usrio; + + /* By default we set to partial store and forward mode for zynqmp. + * Disable if not set in devicetree. + */ + if (GEM_BFEXT(PBUF_CUTTHRU, gem_readl(bp, DCFG6))) { + err = of_property_read_u32(bp->pdev->dev.of_node, + "cdns,rx-watermark", + &bp->rx_watermark); + + if (!err) { + /* Disable partial store and forward in case of error or + * invalid watermark value + */ + wtrmrk_rst_val = (1 << (GEM_BFEXT(RX_PBUF_ADDR, gem_readl(bp, DCFG2)))) - 1; + if (bp->rx_watermark > wtrmrk_rst_val || !bp->rx_watermark) { + dev_info(&bp->pdev->dev, "Invalid watermark value\n"); + bp->rx_watermark = 0; + } + } + } + spin_lock_init(&bp->lock); + spin_lock_init(&bp->stats_lock); + + /* setup capabilities */ + macb_configure_caps(bp, macb_config); + +#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT + if (GEM_BFEXT(DAW64, gem_readl(bp, DCFG6))) { + err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(44)); + if (err) { + dev_err(&pdev->dev, "failed to set DMA mask\n"); + goto err_out_free_netdev; + } + bp->hw_dma_cap |= HW_DMA_CAP_64B; + } +#endif + platform_set_drvdata(pdev, dev); + + dev->irq = platform_get_irq(pdev, 0); + if (dev->irq < 0) { + err = dev->irq; + goto err_out_free_netdev; + } + + /* MTU range: 68 - 1500 or 10240 */ + dev->min_mtu = GEM_MTU_MIN_SIZE; + if ((bp->caps & MACB_CAPS_JUMBO) && bp->jumbo_max_len) + dev->max_mtu = bp->jumbo_max_len - ETH_HLEN - ETH_FCS_LEN; + else + dev->max_mtu = ETH_DATA_LEN; + + if (bp->caps & MACB_CAPS_BD_RD_PREFETCH) { + val = GEM_BFEXT(RXBD_RDBUFF, gem_readl(bp, DCFG10)); + if (val) + bp->rx_bd_rd_prefetch = (2 << (val - 1)) * + macb_dma_desc_get_size(bp); + + val = GEM_BFEXT(TXBD_RDBUFF, gem_readl(bp, DCFG10)); + if (val) + bp->tx_bd_rd_prefetch = (2 << (val - 1)) * + macb_dma_desc_get_size(bp); + } + + bp->rx_intr_mask = MACB_RX_INT_FLAGS; + if (bp->caps & MACB_CAPS_NEEDS_RSTONUBR) + bp->rx_intr_mask |= MACB_BIT(RXUBR); + + err = of_get_ethdev_address(np, bp->dev); + if (err == -EPROBE_DEFER) + goto err_out_free_netdev; + else if (err) + macb_get_hwaddr(bp); + + err = macb_get_phy_mode(pdev); + if (err < 0) + bp->phy_interface = PHY_INTERFACE_MODE_MII; + else + bp->phy_interface = err; + + + /* IP specific init */ + err = init(pdev); + if (err) + goto err_out_free_netdev; + + err = macb_mii_init(bp); + if (err) + goto err_out_phy_exit; + + if (device_property_read_bool(&pdev->dev, "use-ncsi")) { + if (!IS_ENABLED(CONFIG_NET_NCSI)) { + dev_err(&pdev->dev, "NCSI stack not enabled\n"); + goto err_out_free_netdev; + } + dev_notice(&pdev->dev, "Using NCSI interface\n"); + bp->use_ncsi = 1; + bp->ndev = ncsi_register_dev(dev, gem_ncsi_handler); + if (!bp->ndev) + goto err_out_free_netdev; + } else { + bp->use_ncsi = 0; + } + + netif_carrier_off(dev); + + err = register_netdev(dev); + if (err) { + dev_err(&pdev->dev, "Cannot register net device, aborting.\n"); + goto err_out_unregister_mdio; + } + + tasklet_setup(&bp->hresp_err_tasklet, macb_hresp_error_task); + + netdev_info(dev, "Cadence %s rev 0x%08x at 0x%08lx irq %d (%pM)\n", + macb_is_gem(bp) ? "GEM" : "MACB", macb_readl(bp, MID), + dev->base_addr, dev->irq, dev->dev_addr); + + pm_runtime_mark_last_busy(&bp->pdev->dev); + pm_runtime_put_autosuspend(&bp->pdev->dev); + + return 0; + +err_out_unregister_mdio: + mdiobus_unregister(bp->mii_bus); + mdiobus_free(bp->mii_bus); + +err_out_phy_exit: + phy_exit(bp->sgmii_phy); + +err_out_free_netdev: + free_netdev(dev); + +err_disable_clocks: + macb_clks_disable(pclk, hclk, tx_clk, rx_clk, tsu_clk); + pm_runtime_disable(&pdev->dev); + pm_runtime_set_suspended(&pdev->dev); + pm_runtime_dont_use_autosuspend(&pdev->dev); + + return err; +} + +static int macb_remove(struct platform_device *pdev) +{ + struct net_device *dev; + struct macb *bp; + + dev = platform_get_drvdata(pdev); + + if (dev) { + bp = netdev_priv(dev); + unregister_netdev(dev); + phy_exit(bp->sgmii_phy); + unregister_netdev(dev); + mdiobus_unregister(bp->mii_bus); + mdiobus_free(bp->mii_bus); + + tasklet_kill(&bp->hresp_err_tasklet); + pm_runtime_disable(&pdev->dev); + pm_runtime_dont_use_autosuspend(&pdev->dev); + if (!pm_runtime_suspended(&pdev->dev)) { + if (__clk_is_enabled(bp->pclk)) + clk_disable_unprepare(bp->pclk); + if (__clk_is_enabled(bp->hclk)) + clk_disable_unprepare(bp->hclk); + if (__clk_is_enabled(bp->tx_clk)) + clk_disable_unprepare(bp->tx_clk); + if (__clk_is_enabled(bp->rx_clk)) + clk_disable_unprepare(bp->rx_clk); + if (__clk_is_enabled(bp->tsu_clk)) + clk_disable_unprepare(bp->tsu_clk); + pm_runtime_set_suspended(&pdev->dev); + } + phylink_destroy(bp->phylink); + free_netdev(dev); + } + + return 0; +} + +static int __maybe_unused macb_suspend(struct device *dev) +{ + struct net_device *netdev = dev_get_drvdata(dev); + struct macb *bp = netdev_priv(netdev); + struct macb_queue *queue; + unsigned long flags; + unsigned int q; + int err; + u32 tmp; + + if (!device_may_wakeup(&bp->dev->dev)) + phy_exit(bp->sgmii_phy); + + if (!netif_running(netdev)) + return 0; + + if (bp->wol & MACB_WOL_ENABLED) { + spin_lock_irqsave(&bp->lock, flags); + + /* Disable Tx and Rx engines before disabling the queues, + * this is mandatory as per the IP spec sheet + */ + tmp = macb_readl(bp, NCR); + macb_writel(bp, NCR, tmp & ~(MACB_BIT(TE) | MACB_BIT(RE))); + for (q = 0, queue = bp->queues; q < bp->num_queues; + ++q, ++queue) { + /* Disable RX queues */ + if (bp->caps & MACB_CAPS_QUEUE_DISABLE) { + queue_writel(queue, RBQP, MACB_BIT(QUEUE_DISABLE)); + } else { + /* Tie off RX queues */ + queue_writel(queue, RBQP, + lower_32_bits(bp->rx_ring_tieoff_dma)); +#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT + queue_writel(queue, RBQPH, + upper_32_bits(bp->rx_ring_tieoff_dma)); +#endif + } + /* Disable all interrupts */ + queue_writel(queue, IDR, -1); + queue_readl(queue, ISR); + if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) + queue_writel(queue, ISR, -1); + } + /* Enable Receive engine */ + macb_writel(bp, NCR, tmp | MACB_BIT(RE)); + /* Flush all status bits */ + macb_writel(bp, TSR, -1); + macb_writel(bp, RSR, -1); + + spin_unlock_irqrestore(&bp->lock, flags); + + /* Change interrupt handler and + * Enable WoL IRQ on queue 0 + */ + devm_free_irq(dev, bp->queues[0].irq, bp->queues); + if (macb_is_gem(bp)) { + err = devm_request_irq(dev, bp->queues[0].irq, gem_wol_interrupt, + IRQF_SHARED, netdev->name, bp->queues); + if (err) { + dev_err(dev, + "Unable to request IRQ %d (error %d)\n", + bp->queues[0].irq, err); + return err; + } + spin_lock_irqsave(&bp->lock, flags); + queue_writel(bp->queues, IER, GEM_BIT(WOL)); + gem_writel(bp, WOL, MACB_BIT(MAG)); + spin_unlock_irqrestore(&bp->lock, flags); + } else { + err = devm_request_irq(dev, bp->queues[0].irq, macb_wol_interrupt, + IRQF_SHARED, netdev->name, bp->queues); + if (err) { + dev_err(dev, + "Unable to request IRQ %d (error %d)\n", + bp->queues[0].irq, err); + return err; + } + spin_lock_irqsave(&bp->lock, flags); + queue_writel(bp->queues, IER, MACB_BIT(WOL)); + macb_writel(bp, WOL, MACB_BIT(MAG)); + spin_unlock_irqrestore(&bp->lock, flags); + } + + enable_irq_wake(bp->queues[0].irq); + } + + netif_device_detach(netdev); + for (q = 0, queue = bp->queues; q < bp->num_queues; + ++q, ++queue) { + napi_disable(&queue->napi_rx); + napi_disable(&queue->napi_tx); + } + + if (!(bp->wol & MACB_WOL_ENABLED)) { + rtnl_lock(); + phylink_stop(bp->phylink); + rtnl_unlock(); + spin_lock_irqsave(&bp->lock, flags); + macb_reset_hw(bp); + spin_unlock_irqrestore(&bp->lock, flags); + } + + if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) + bp->pm_data.usrio = macb_or_gem_readl(bp, USRIO); + + if (netdev->hw_features & NETIF_F_NTUPLE) + bp->pm_data.scrt2 = gem_readl_n(bp, ETHT, SCRT2_ETHT); + + if (bp->ptp_info) + bp->ptp_info->ptp_remove(netdev); + if (!device_may_wakeup(dev)) + pm_runtime_force_suspend(dev); + + return 0; +} + +static int __maybe_unused macb_resume(struct device *dev) +{ + struct net_device *netdev = dev_get_drvdata(dev); + struct macb *bp = netdev_priv(netdev); + struct macb_queue *queue; + unsigned long flags; + unsigned int q; + int err; + + if (!device_may_wakeup(&bp->dev->dev)) + phy_init(bp->sgmii_phy); + + if (!netif_running(netdev)) + return 0; + + if (!device_may_wakeup(dev)) + pm_runtime_force_resume(dev); + + if (bp->wol & MACB_WOL_ENABLED) { + spin_lock_irqsave(&bp->lock, flags); + /* Disable WoL */ + if (macb_is_gem(bp)) { + queue_writel(bp->queues, IDR, GEM_BIT(WOL)); + gem_writel(bp, WOL, 0); + } else { + queue_writel(bp->queues, IDR, MACB_BIT(WOL)); + macb_writel(bp, WOL, 0); + } + /* Clear ISR on queue 0 */ + queue_readl(bp->queues, ISR); + if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) + queue_writel(bp->queues, ISR, -1); + spin_unlock_irqrestore(&bp->lock, flags); + + /* Replace interrupt handler on queue 0 */ + devm_free_irq(dev, bp->queues[0].irq, bp->queues); + err = devm_request_irq(dev, bp->queues[0].irq, macb_interrupt, + IRQF_SHARED, netdev->name, bp->queues); + if (err) { + dev_err(dev, + "Unable to request IRQ %d (error %d)\n", + bp->queues[0].irq, err); + return err; + } + + disable_irq_wake(bp->queues[0].irq); + + /* Now make sure we disable phy before moving + * to common restore path + */ + rtnl_lock(); + phylink_stop(bp->phylink); + rtnl_unlock(); + } + + if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC)) + macb_init_buffers(bp); + + for (q = 0, queue = bp->queues; q < bp->num_queues; + ++q, ++queue) { + if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC)) { + if (macb_is_gem(bp)) + gem_init_rx_ring(queue); + else + macb_init_rx_ring(queue); + } + + napi_enable(&queue->napi_rx); + napi_enable(&queue->napi_tx); + } + + if (netdev->hw_features & NETIF_F_NTUPLE) + gem_writel_n(bp, ETHT, SCRT2_ETHT, bp->pm_data.scrt2); + + if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) + macb_or_gem_writel(bp, USRIO, bp->pm_data.usrio); + + macb_writel(bp, NCR, MACB_BIT(MPE)); + macb_init_hw(bp); + macb_set_rx_mode(netdev); + macb_restore_features(bp); + rtnl_lock(); + phylink_start(bp->phylink); + rtnl_unlock(); + + netif_device_attach(netdev); + if (bp->ptp_info) + bp->ptp_info->ptp_init(netdev); + + return 0; +} + +static int __maybe_unused macb_runtime_suspend(struct device *dev) +{ + struct net_device *netdev = dev_get_drvdata(dev); + struct macb *bp = netdev_priv(netdev); + + if (!(device_may_wakeup(dev))) { + if (__clk_is_enabled(bp->pclk)) + clk_disable_unprepare(bp->pclk); + if (__clk_is_enabled(bp->hclk)) + clk_disable_unprepare(bp->hclk); + if (__clk_is_enabled(bp->tx_clk)) + clk_disable_unprepare(bp->tx_clk); + if (__clk_is_enabled(bp->rx_clk)) + clk_disable_unprepare(bp->rx_clk); + if (__clk_is_enabled(bp->tsu_clk)) + clk_disable_unprepare(bp->tsu_clk); + } else if (!(bp->caps & MACB_CAPS_NEED_TSUCLK)) { + if (__clk_is_enabled(bp->tsu_clk)) + clk_disable_unprepare(bp->tsu_clk); + } + + return 0; +} + +static int __maybe_unused macb_runtime_resume(struct device *dev) +{ + struct net_device *netdev = dev_get_drvdata(dev); + struct macb *bp = netdev_priv(netdev); + + if (!(device_may_wakeup(dev))) { + clk_prepare_enable(bp->pclk); + clk_prepare_enable(bp->hclk); + clk_prepare_enable(bp->tx_clk); + clk_prepare_enable(bp->rx_clk); + clk_prepare_enable(bp->tsu_clk); + } else if (!(bp->caps & MACB_CAPS_NEED_TSUCLK)) { + clk_prepare_enable(bp->tsu_clk); + } + + return 0; +} + +static const struct dev_pm_ops macb_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(macb_suspend, macb_resume) + SET_RUNTIME_PM_OPS(macb_runtime_suspend, macb_runtime_resume, NULL) +}; + +static struct platform_driver macb_driver = { + .probe = macb_probe, + .remove = macb_remove, + .driver = { + .name = "macb", + .of_match_table = of_match_ptr(macb_dt_ids), + .acpi_match_table = macb_acpi_ids, + .pm = &macb_pm_ops, + }, +}; + +module_platform_driver(macb_driver); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Cadence MACB/GEM Ethernet driver"); +MODULE_AUTHOR("Haavard Skinnemoen (Atmel)"); +MODULE_ALIAS("platform:macb"); From 7732d2328fc2720c9f757cacbcedc5ba71bed7b3 Mon Sep 17 00:00:00 2001 From: Ma Mingrui Date: Sat, 9 May 2026 11:54:39 +0800 Subject: [PATCH 07/13] net: pci-ep-vnet: add Phytium PCIe EP VNet driver Add support for the Phytium PCIe endpoint virtual Ethernet (EP VNet) driver. This driver enables the endpoint side of a PCIe-based virtual Ethernet link, typically paired with a corresponding root complex (RC) driver. Signed-off-by: Ma Mingrui Signed-off-by: Li Tongfeng Signed-off-by: Wang Yinfeng --- drivers/net/ethernet/Kconfig | 3 +- drivers/net/ethernet/Makefile | 3 +- drivers/net/ethernet/phytium-pci-vnet/Kconfig | 25 + .../net/ethernet/phytium-pci-vnet/Makefile | 6 + .../phytium-pci-vnet/pci-ep-net/Kconfig | 19 + .../phytium-pci-vnet/pci-ep-net/Makefile | 6 + .../pci-ep-net/pci_epf_vnet.c | 1572 +++++++++++++++++ .../pci-ep-net/pci_epf_vnet.h | 282 +++ drivers/net/ethernet/phytium/phytmac_v1.c | 14 + 9 files changed, 1928 insertions(+), 2 deletions(-) create mode 100644 drivers/net/ethernet/phytium-pci-vnet/Kconfig create mode 100644 drivers/net/ethernet/phytium-pci-vnet/Makefile create mode 100644 drivers/net/ethernet/phytium-pci-vnet/pci-ep-net/Kconfig create mode 100644 drivers/net/ethernet/phytium-pci-vnet/pci-ep-net/Makefile create mode 100644 drivers/net/ethernet/phytium-pci-vnet/pci-ep-net/pci_epf_vnet.c create mode 100644 drivers/net/ethernet/phytium-pci-vnet/pci-ep-net/pci_epf_vnet.h diff --git a/drivers/net/ethernet/Kconfig b/drivers/net/ethernet/Kconfig index 0bf3c92c88bba..157d1f59aee40 100644 --- a/drivers/net/ethernet/Kconfig +++ b/drivers/net/ethernet/Kconfig @@ -166,6 +166,8 @@ config ETHOC source "drivers/net/ethernet/packetengines/Kconfig" source "drivers/net/ethernet/pasemi/Kconfig" source "drivers/net/ethernet/pensando/Kconfig" +source "drivers/net/ethernet/phytium/Kconfig" +source "drivers/net/ethernet/phytium-pci-vnet/Kconfig" source "drivers/net/ethernet/qlogic/Kconfig" source "drivers/net/ethernet/brocade/Kconfig" source "drivers/net/ethernet/qualcomm/Kconfig" @@ -195,7 +197,6 @@ source "drivers/net/ethernet/wangxun/Kconfig" source "drivers/net/ethernet/wiznet/Kconfig" source "drivers/net/ethernet/xilinx/Kconfig" source "drivers/net/ethernet/xircom/Kconfig" -source "drivers/net/ethernet/phytium/Kconfig" source "drivers/net/ethernet/guangruntong/Kconfig" source "drivers/net/ethernet/bzwx/Kconfig" source "drivers/net/ethernet/linkdata/Kconfig" diff --git a/drivers/net/ethernet/Makefile b/drivers/net/ethernet/Makefile index d2e95e407e0fe..e8a6674b2cb4b 100644 --- a/drivers/net/ethernet/Makefile +++ b/drivers/net/ethernet/Makefile @@ -78,6 +78,8 @@ obj-$(CONFIG_NET_VENDOR_OKI) += oki-semi/ obj-$(CONFIG_ETHOC) += ethoc.o obj-$(CONFIG_NET_VENDOR_PACKET_ENGINES) += packetengines/ obj-$(CONFIG_NET_VENDOR_PASEMI) += pasemi/ +obj-$(CONFIG_NET_VENDOR_PHYTIUM) += phytium/ +obj-$(CONFIG_VNET_VENDOR_PHYTIUM) += phytium-pci-vnet/ obj-$(CONFIG_NET_VENDOR_QLOGIC) += qlogic/ obj-$(CONFIG_NET_VENDOR_QUALCOMM) += qualcomm/ obj-$(CONFIG_NET_VENDOR_REALTEK) += realtek/ @@ -107,7 +109,6 @@ obj-$(CONFIG_NET_VENDOR_XILINX) += xilinx/ obj-$(CONFIG_NET_VENDOR_XIRCOM) += xircom/ obj-$(CONFIG_NET_VENDOR_SYNOPSYS) += synopsys/ obj-$(CONFIG_NET_VENDOR_PENSANDO) += pensando/ -obj-$(CONFIG_NET_VENDOR_PHYTIUM) += phytium/ obj-$(CONFIG_NET_VENDOR_GRT) += guangruntong/ obj-$(CONFIG_NET_VENDOR_BZWX) += bzwx/ obj-$(CONFIG_NET_VENDOR_LINKDATA) += linkdata/ diff --git a/drivers/net/ethernet/phytium-pci-vnet/Kconfig b/drivers/net/ethernet/phytium-pci-vnet/Kconfig new file mode 100644 index 0000000000000..fe992cb721bb3 --- /dev/null +++ b/drivers/net/ethernet/phytium-pci-vnet/Kconfig @@ -0,0 +1,25 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Phytium PCI virtual network device configuration +# + +menu "Phytium PCI Virtual Ethernet Drivers" +config VNET_VENDOR_PHYTIUM + bool "Phytium devices" + depends on ARCH_PHYTIUM + depends on HAS_IOMEM + default y + help + If you have a network (Ethernet) card belonging to this class, say Y. + + Note that the answer to this question doesn't directly affect the + kernel: saying N will just cause the configurator to skip all the + remaining Cadence network card questions. If you say Y, you will be + asked for your specific card in the following questions. + +if VNET_VENDOR_PHYTIUM + +source "drivers/net/ethernet/phytium-pci-vnet/pci-ep-net/Kconfig" + +endif #VNET_VENDOR_PHYTIUM +endmenu diff --git a/drivers/net/ethernet/phytium-pci-vnet/Makefile b/drivers/net/ethernet/phytium-pci-vnet/Makefile new file mode 100644 index 0000000000000..cd1ea18348653 --- /dev/null +++ b/drivers/net/ethernet/phytium-pci-vnet/Makefile @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0 +# Copyright(c) 2022 - 2025 Phytium Technology Co., Ltd. +# +# Makefile for the Phytium PCI virtual network device drivers. + +obj-$(CONFIG_PHYTIUM_PCI_EPF_VNET) += pci-ep-net/ diff --git a/drivers/net/ethernet/phytium-pci-vnet/pci-ep-net/Kconfig b/drivers/net/ethernet/phytium-pci-vnet/pci-ep-net/Kconfig new file mode 100644 index 0000000000000..16a458c91c6d6 --- /dev/null +++ b/drivers/net/ethernet/phytium-pci-vnet/pci-ep-net/Kconfig @@ -0,0 +1,19 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Phytium PCI EP virtual network device configuration +# + +config PHYTIUM_PCI_EPF_VNET + tristate "Phytium PCI Endpoint Virtual Ethernet Device" + depends on PCI_ENDPOINT + depends on PCIE_PHYTIUM_EP + help + This option enables support for the Phytium PCI Endpoint Virtual Ethernet function + driver. It allows the device to appear as a network interface to a PCIe root complex. + + Typically used in conjunction with a Phytium PCIe Root Complex host that supports + the corresponding RC virtual network function. + + If you want this endpoint to provide virtual networking over PCIe, say Y or M. + + If unsure, say N. diff --git a/drivers/net/ethernet/phytium-pci-vnet/pci-ep-net/Makefile b/drivers/net/ethernet/phytium-pci-vnet/pci-ep-net/Makefile new file mode 100644 index 0000000000000..755dd26dceb9d --- /dev/null +++ b/drivers/net/ethernet/phytium-pci-vnet/pci-ep-net/Makefile @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0 +# Copyright(c) 2022 - 2025 Phytium Technology Co., Ltd. +# +# Makefile for the Phytium network device drivers. + +obj-$(CONFIG_PHYTIUM_PCI_EPF_VNET) += pci_epf_vnet.o diff --git a/drivers/net/ethernet/phytium-pci-vnet/pci-ep-net/pci_epf_vnet.c b/drivers/net/ethernet/phytium-pci-vnet/pci-ep-net/pci_epf_vnet.c new file mode 100644 index 0000000000000..8ca505d4c76ba --- /dev/null +++ b/drivers/net/ethernet/phytium-pci-vnet/pci-ep-net/pci_epf_vnet.c @@ -0,0 +1,1572 @@ +// SPDX-License-Identifier: GPL-2.0 +/** + * Virtual network driver based on the pcie interface in EP + * + * Copyright (C) 2023 Phytium Corporation + * Author: netgroup@phytium.com.cn + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "pci_epf_vnet.h" + +char dev_addr[6] = {0x04, 0x05, 0x06, 0x00, 0x00, 0x16}; + +static int debug = 3; +module_param(debug, int, 0); + +static struct cdev_mdata epf_cdev_mdata; +static struct epf_ioctl_data ioctl_data; + +static struct sk_buff *ep_build_skb(struct ep_rx_buffer *rx_buffer, + unsigned int size) +{ + struct sk_buff *skb; + unsigned int truesize; + void *va; + +#if (PAGE_SIZE < 8192) + truesize = EP_RX_PAGE_SIZE / 2; +#else + truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) + + SKB_DATA_ALIGN(EP_SKB_PAD + size); +#endif + + va = page_address(rx_buffer->page) + rx_buffer->page_offset; + /* prefetch first cache line of first page */ + prefetch(va); + + /* build an skb around the page buffer */ + skb = build_skb(va - EP_SKB_PAD, truesize); + if (unlikely(!skb)) + return NULL; + + /* update pointers within the skb to store the data */ + skb_reserve(skb, EP_SKB_PAD); + __skb_put(skb, size); + + /* update buffer offset */ +#if (PAGE_SIZE < 8192) + rx_buffer->page_offset ^= truesize; +#else + rx_buffer->page_offset += truesize; +#endif + + return skb; +} + +static inline int mul32(int size) +{ + int ret = 0; + + if (size % 32) + ret = size + (32 - size % 32); + else + ret = size; + + return ret; +} + +static inline struct device *vnet_to_dev(struct pci_epf_vnet *vnet) +{ + return &vnet->epf->dev; +} + +static inline unsigned int vnet_tx_ring_wrap(struct pci_epf_vnet *vnet, + unsigned int index) +{ + return index & (vnet->tx_ring_size - 1); +} + +static inline struct pci_ep_dma_desc *vnet_tx_desc(struct pci_epf_vnet *vnet, + unsigned int index) +{ + struct pci_ep_queue *queue = vnet->queue; + struct ep_queue *tx_queue = &queue->ep_tx_queue; + /* Obtain the index of descriptors */ + index = vnet_tx_ring_wrap(vnet, index); + return &tx_queue->dma_desc_base[index]; +} + +static inline unsigned int vnet_rx_ring_wrap(struct pci_epf_vnet *vnet, + unsigned int index) +{ + return index & (vnet->rx_ring_size - 1); +} + +static inline struct pci_ep_dma_desc *vnet_rx_desc(struct pci_epf_vnet *vnet, + unsigned int index) +{ + struct pci_ep_queue *queue = vnet->queue; + struct ep_queue *rx_queue = &queue->ep_rx_queue; + /* Obtain the index of descriptors */ + index = vnet_rx_ring_wrap(vnet, index); + return &rx_queue->dma_desc_base[index]; +} + +static inline int pci_epf_vnet_ep2rc_dma(struct pci_epf_vnet *vnet, + dma_addr_t buffer_dma_addr, + u64 addr, u32 data_size, + enum dma_data_direction dir) +{ + int ret; + struct pci_epc *epc = vnet->epf->epc; + + dma_sync_single_for_device(epc->dev.parent, buffer_dma_addr, data_size, + dir); + + ret = pci_epc_start_dma(epc, vnet->epf->func_no, buffer_dma_addr, addr, + data_size, DMA_WRITE); + if (ret) { + dev_err(&epc->dev, "ep2rc pci_epc_start_dma fail! ret %d\n", + ret); + return -EIO; + } + + return 0; +} + +static inline int pci_epf_vnet_rc2ep_dma(struct pci_epf_vnet *vnet, + dma_addr_t buffer_dma_addr, + u64 addr, u32 data_size) +{ + int ret; + struct pci_epc *epc = vnet->epf->epc; + + ret = pci_epc_start_dma(epc, vnet->epf->func_no, buffer_dma_addr, addr, + data_size, DMA_READ); + if (ret) { + dev_err(&epc->dev, "rc2ep pci_epc_start_dma fail! ret %d\n", + ret); + return -EIO; + } + + return 0; +} + +static inline bool vnet_tx_desc_queue_full(struct pci_epf_vnet *vnet) +{ + u32 tail, head; + struct pci_ep_queue *queue = vnet->queue; + struct ep_queue *tx_queue = &queue->ep_tx_queue; + + head = READ_ONCE(tx_queue->head); + tail = READ_ONCE(tx_queue->tail); + + if (head == (tail + 1) % vnet->tx_ring_size) + return true; + else + return false; +} + +static inline int vnet_get_tx_desc_tail_idx(struct pci_epf_vnet *vnet) +{ + u32 tail; + struct pci_ep_queue *queue = vnet->queue; + struct ep_queue *tx_queue = &queue->ep_tx_queue; + + tail = READ_ONCE(tx_queue->tail); + return tail; +} + +static inline int vnet_adj_tx_desc_tail_idx(struct pci_epf_vnet *vnet) +{ + struct pci_ep_queue *queue = vnet->queue; + struct ep_queue *tx_queue = &queue->ep_tx_queue; + u32 tail = READ_ONCE(tx_queue->tail); + + /* Move tail pointer */ + tail = (tail + 1) % vnet->tx_ring_size; + WRITE_ONCE(tx_queue->tail, tail); + + return 0; +} + +static bool ep_alloc_mapped_page(struct pci_epf_vnet *vnet, + struct ep_rx_buffer *rx_buffer_info) +{ + dma_addr_t paddr; + struct pci_epf *epf = vnet->epf; + struct pci_epc *epc = epf->epc; + struct page *page = rx_buffer_info->page; + + /* since we are recycling buffers we should seldom need to alloc */ + if (likely(page)) + return true; + + page = __dev_alloc_pages(EP_GFP_FLAGS, EP_RX_PAGE_ORDER); + if (unlikely(!page)) { + netdev_err(vnet->netdev, "rx alloc page failed\n"); + rx_buffer_info->page = NULL; + return false; + } + + paddr = dma_map_page_attrs(epc->dev.parent, page, 0, + EP_RX_PAGE_SIZE, + DMA_FROM_DEVICE, EP_DMA_ATTR); + if (dma_mapping_error(epc->dev.parent, paddr)) { + __free_pages(page, EP_RX_PAGE_ORDER); + rx_buffer_info->page = NULL; + return false; + } + + rx_buffer_info->addr = paddr; + rx_buffer_info->page = page; + rx_buffer_info->pagecnt_bias = 1; + rx_buffer_info->page_offset = EP_SKB_PAD; + + return true; +} + +static void vnet_rx_refill(struct pci_epf_vnet *vnet, bool init) +{ + struct pci_epf *epf = vnet->epf; + struct pci_epc *epc = epf->epc; + unsigned int entry, space; + struct ep_rx_buffer *rx_buffer_info; + + if (init) + space = vnet->rx_ring_size - 1; + else + space = CIRC_SPACE(vnet->rx_refill_start, + READ_ONCE(vnet->queue->ep_rx_queue.head), + vnet->rx_ring_size); + while (space > 0) { + entry = vnet_rx_ring_wrap(vnet, vnet->rx_refill_start); + + rx_buffer_info = &vnet->rx_buffer_info[entry]; + if (!ep_alloc_mapped_page(vnet, rx_buffer_info)) + break; + /* sync the buffer for use by the device */ + dma_sync_single_range_for_device(epc->dev.parent, + rx_buffer_info->addr, + rx_buffer_info->page_offset, + vnet->rx_buffer_len, + DMA_FROM_DEVICE); + vnet->rx_refill_start = (vnet->rx_refill_start + 1) % + vnet->rx_ring_size; + space--; + } + + vnet->rx_next_to_alloc = vnet->rx_refill_start; +} + +static int vnet_tx_fill(struct pci_epf_vnet *vnet, u32 pktnum) +{ + struct page *page; + dma_addr_t paddr; + struct pci_epf *epf = vnet->epf; + struct pci_epc *epc = epf->epc; + int count; + void *buffer_virt; + + for (count = 0; count < pktnum; count++) { + page = __dev_alloc_pages(EP_GFP_FLAGS, EP_TX_PAGE_ORDER); + if (unlikely(!page)) { + netdev_err(vnet->netdev, "tx alloc page failed\n"); + break; + } + + buffer_virt = page_to_virt(page); + + paddr = dma_map_page_attrs(epc->dev.parent, page, 0, + EP_TX_PAGE_SIZE, + DMA_FROM_DEVICE, EP_DMA_ATTR); + if (dma_mapping_error(epc->dev.parent, paddr)) { + netdev_err(vnet->netdev, "tx map page failed\n"); + break; + } + + vnet->tx_buffer_info[count].page = page; + vnet->tx_buffer_info[count].addr = paddr; + vnet->tx_buffer_info[count].vaddr = buffer_virt; + } + + return count; +} + +static void vnet_free_rx_skb_ring(struct pci_epf_vnet *vnet, int count) +{ + int i; + struct pci_epf *epf = vnet->epf; + struct pci_epc *epc = epf->epc; + struct ep_rx_buffer *rx_buffer_info; + + for (i = 0; i < count; i++) { + rx_buffer_info = &vnet->rx_buffer_info[i]; + if (!rx_buffer_info->page) + continue; + + dma_unmap_page_attrs(epc->dev.parent, rx_buffer_info->addr, + EP_RX_PAGE_SIZE, DMA_FROM_DEVICE, EP_DMA_ATTR); + + __page_frag_cache_drain(rx_buffer_info->page, + rx_buffer_info->pagecnt_bias); + } + + vfree(vnet->rx_buffer_info); + vnet->rx_buffer_info = NULL; +} + +static inline int ep_calc_rx_buf_len(void) +{ +#if (PAGE_SIZE < 8192) + return rounddown(EP_MAX_FRAME_BUILD_SKB, RX_BUFFER_MULTIPLE); +#endif + return rounddown(EP_RXBUFFER_2048, RX_BUFFER_MULTIPLE); +} + +static int vnet_alloc_rx_skb_ring(struct pci_epf_vnet *vnet) +{ + int size; + + vnet->rx_buffer_len = ep_calc_rx_buf_len(); + + /* Allocate pointer array for rx buffers */ + size = vnet->rx_ring_size * sizeof(struct ep_rx_buffer); + vnet->rx_buffer_info = vzalloc(size); + if (!vnet->rx_buffer_info) { + netdev_err(vnet->netdev, + "%s:Unable to allocate buffer memory\n", __func__); + return -ENOMEM; + } + + netdev_dbg(vnet->netdev, + "Allocated %d RX struct buffer entries at %p\n", + vnet->rx_ring_size, vnet->rx_skb); + + /* Pre allocated rx skb */ + vnet_rx_refill(vnet, true); + + return 0; +} + +static void vnet_free_tx_skb_ring(struct pci_epf_vnet *vnet, int count) +{ + int i; + struct pci_epf *epf = vnet->epf; + struct pci_epc *epc = epf->epc; + struct ep_tx_buffer *tx_buffer_info; + + for (i = 0; i < count; i++) { + tx_buffer_info = &vnet->tx_buffer_info[i]; + if (!tx_buffer_info->page) + continue; + + dma_unmap_page_attrs(epc->dev.parent, tx_buffer_info->addr, + EP_RX_PAGE_SIZE, DMA_FROM_DEVICE, EP_DMA_ATTR); + + __page_frag_cache_drain(tx_buffer_info->page, + tx_buffer_info->pagecnt_bias); + } + + vfree(vnet->tx_buffer_info); + vnet->tx_buffer_info = NULL; +} + +static int vnet_alloc_tx_skb_ring(struct pci_epf_vnet *vnet) +{ + int size; + int count; + + size = vnet->tx_ring_size * sizeof(struct ep_tx_buffer); + vnet->tx_buffer_info = vzalloc(size); + if (!vnet->tx_buffer_info) { + netdev_err(vnet->netdev, + "%s:Unable to allocate buffer memory\n", __func__); + return -ENOMEM; + } + + netdev_dbg(vnet->netdev, + "Allocated %d TX struct buffer entries at %p\n", + vnet->tx_ring_size, vnet->tx_buffer_info); + + /* Pre allocated tx buffer */ + count = vnet_tx_fill(vnet, vnet->tx_ring_size); + if (count != vnet->tx_ring_size) { + netdev_err(vnet->netdev, + "%s:There is that page allocation failed!\n", + __func__); + vnet_free_tx_skb_ring(vnet, count); + return -ENOMEM; + } + + return 0; +} + +static int vnet_down(struct net_device *netdev) +{ + struct pci_epf_vnet *vnet = netdev_priv(netdev); + + napi_disable(&vnet->rx_napi); + + netif_stop_queue(netdev); + + netif_carrier_off(netdev); + + vnet_free_rx_skb_ring(vnet, vnet->rx_ring_size); + vnet_free_tx_skb_ring(vnet, vnet->tx_ring_size); + + return 0; +} + +static int vnet_up(struct net_device *netdev) +{ + struct pci_epf_vnet *vnet = netdev_priv(netdev); + struct ep_queue *rx_queue; + struct ep_queue *tx_queue; + int err; + + rx_queue = &vnet->queue->ep_rx_queue; + tx_queue = &vnet->queue->ep_tx_queue; + + err = vnet_alloc_rx_skb_ring(vnet); + if (err) { + netdev_err(vnet->netdev, "%s: failed to alloc rx buffer!!\n", + __func__); + return err; + } + + err = vnet_alloc_tx_skb_ring(vnet); + if (err) { + netdev_err(vnet->netdev, "%s: failed to alloc tx buffer!!\n", + __func__); + return err; + } + + netdev->flags |= IFF_RUNNING | IFF_UP; + + netif_wake_queue(netdev); + + napi_enable(&vnet->rx_napi); + + return 0; +} + +static void vnet_get_drvinfo(struct net_device *netdev, + struct ethtool_drvinfo *drvinfo) +{ + strscpy(drvinfo->driver, DRV_MODULE_NAME, sizeof(drvinfo->driver)); + strscpy(drvinfo->version, DRV_VERSION, sizeof(drvinfo->version)); +} + +static u32 vnet_get_msglevel(struct net_device *netdev) +{ + struct pci_epf_vnet *vnet = netdev_priv(netdev); + + return vnet->msg_enable; +} + +static void vnet_set_msglevel(struct net_device *netdev, u32 value) +{ + struct pci_epf_vnet *vnet = netdev_priv(netdev); + + vnet->msg_enable = value; +} + +static u32 vnet_get_link(struct net_device *netdev) +{ + struct pci_epf_vnet *vnet = netdev_priv(netdev); + struct pci_ep_queue *queue = vnet->queue; + struct ep_queue *tx_queue = &queue->ep_tx_queue; + + return tx_queue->flags & PCI_EP_VNET_FLAGS_BIT(RC_LINK_ACTIVE); +} + +static const char vnet_gstrings_stats[][ETH_GSTRING_LEN] = { + "rx_packets", "tx_packets", "rx_bytes", + "tx_bytes", "rx_errors", "tx_errors", + "rx_dropped", "tx_dropped", "multicast", + "collisions", "rx_length_errors", "rx_over_errors", + "rx_crc_errors", "rx_frame_errors", "rx_fifo_errors", + "rx_missed_errors", "tx_aborted_errors", "tx_carrier_errors", + "tx_fifo_errors", "tx_heartbeat_errors", "tx_window_errors", +}; + +#define VNET_NET_STATS_LEN ARRAY_SIZE(vnet_gstrings_stats) + +static int vnet_get_sset_count(struct net_device *netdev, int sset) +{ + switch (sset) { + case ETH_SS_STATS: + return VNET_NET_STATS_LEN; + default: + return -EOPNOTSUPP; + } +} + +static void vnet_get_ethtool_stats(struct net_device *netdev, + struct ethtool_stats *stats, u64 *data) +{ + int i; + + for (i = 0; i < VNET_NET_STATS_LEN; i++) + data[i] = ((unsigned long *)&netdev->stats)[i]; +} + +static void vnet_get_strings(struct net_device *netdev, u32 stringset, u8 *data) +{ + switch (stringset) { + case ETH_SS_STATS: + memcpy(data, *vnet_gstrings_stats, sizeof(vnet_gstrings_stats)); + break; + } +} + +static const struct ethtool_ops vnet_ethtool_ops = { + .get_drvinfo = vnet_get_drvinfo, + .get_msglevel = vnet_get_msglevel, + .set_msglevel = vnet_set_msglevel, + .get_link = vnet_get_link, + .get_strings = vnet_get_strings, + .get_ethtool_stats = vnet_get_ethtool_stats, + .get_sset_count = vnet_get_sset_count, +}; + +static void pci_ep_vnet_get_defaults(struct pci_epf_vnet *vnet) +{ + struct param_range rxd_size = { .min = VNET_MIN_RXD, + .max = VNET_MAX_RXD, + .count = VNET_DEFAULT_RXD }; + struct param_range txd_size = { .min = VNET_MIN_TXD, + .max = VNET_MAX_TXD, + .count = VNET_DEFAULT_TXD }; + + vnet->rx_refill_start = 0; + vnet->rx_refill_num = 0; + + /* RX buffers initialization */ + vnet->rx_buffer_size = VNET_RXBUFFER_2048; + + vnet->rx_ring_size = rxd_size.count; + vnet->tx_ring_size = txd_size.count; + + vnet->rxd_size = rxd_size; + vnet->txd_size = txd_size; +} + +static bool ep_can_reuse_rx_page(struct ep_rx_buffer *rx_buffer) +{ + unsigned int pagecnt_bias = rx_buffer->pagecnt_bias; + struct page *page = rx_buffer->page; + + /* avoid re-using remote and pfmemalloc pages */ + if (!dev_page_is_reusable(page)) + return false; + +#if (PAGE_SIZE < 8192) + /* if we are only owner of page we can reuse it */ + if (unlikely((page_ref_count(page) - pagecnt_bias) > 1)) + return false; +#else +#define EP_LAST_OFFSET \ + (SKB_WITH_OVERHEAD(PAGE_SIZE) - EP_RXBUFFER_2048) + + if (rx_buffer->page_offset > EP_LAST_OFFSET) + return false; +#endif + + /* If we have drained the page fragment pool we need to update + * the pagecnt_bias and page count so that we fully restock the + * number of references the driver holds. + */ + if (unlikely(!pagecnt_bias)) { + page_ref_add(page, USHRT_MAX); + rx_buffer->pagecnt_bias = USHRT_MAX; + } + + return true; +} + +static void ep_reuse_rx_page(struct pci_epf_vnet *vnet, + struct ep_rx_buffer *old_buff) +{ + struct ep_rx_buffer *new_buff; + u16 nta = vnet->rx_next_to_alloc; + + new_buff = &vnet->rx_buffer_info[nta & (vnet->rx_ring_size - 1)]; + + /* update, and store next to alloc */ + nta++; + vnet->rx_next_to_alloc = (nta < vnet->rx_ring_size) ? nta : 0; + + /* Transfer page from old buffer to new buffer. + * Move each member individually to avoid possible store + * forwarding stalls. + */ + new_buff->addr = old_buff->addr; + new_buff->page = old_buff->page; + new_buff->page_offset = old_buff->page_offset; + new_buff->pagecnt_bias = old_buff->pagecnt_bias; +} + +static void ep_put_rx_buffer(struct pci_epf_vnet *vnet, + struct ep_rx_buffer *rx_buffer_info) +{ + struct pci_epf *epf = vnet->epf; + struct pci_epc *epc = epf->epc; + + if (ep_can_reuse_rx_page(rx_buffer_info)) { + /* hand second half of page back to the ring */ + ep_reuse_rx_page(vnet, rx_buffer_info); + } else { + dma_unmap_page_attrs(epc->dev.parent, rx_buffer_info->addr, + EP_RX_PAGE_SIZE, DMA_FROM_DEVICE, EP_DMA_ATTR); + + __page_frag_cache_drain(rx_buffer_info->page, + rx_buffer_info->pagecnt_bias); + } + + /* clear contents of rx_buffer */ + rx_buffer_info->page = NULL; +} + +static int pci_epf_vnet_rx(struct pci_epf_vnet *vnet, int budget) +{ + struct pci_epf *epf = vnet->epf; + struct pci_epc *epc = epf->epc; + struct ep_queue *rx_queue = &vnet->queue->ep_rx_queue; + u64 src_addr, dst_addr; + struct sk_buff *skb; + unsigned int len; + struct pci_ep_dma_desc *desc; + int count; + u32 desc_crc, pkt_crc; + void *pkt_data; + unsigned int entry; + int ret = 0; + int timeout; + struct ep_rx_buffer *rx_buffer_info; + u32 rx_head, rx_tail; + + for (count = 0; count < budget; count++) { + rx_head = READ_ONCE(rx_queue->head); + rx_tail = READ_ONCE(rx_queue->tail); + /* Obtain descriptor index */ + entry = vnet_rx_ring_wrap(vnet, rx_head); + + /* Obtain descriptors for bar space */ + desc = vnet_rx_desc(vnet, entry); + + /* No package to receive */ + if (rx_head == rx_tail) + break; + + /* Make hw descriptor updates visible to CPU */ + dma_rmb(); + /* Determine if it is a valid package */ + if (!(desc->ctrl & PCI_EP_DMA_DESC_CTRL_FIELD_MASK(USED))) + break; + + rx_buffer_info = &vnet->rx_buffer_info[entry & (vnet->rx_ring_size - 1)]; + if (!rx_buffer_info->page) { + netdev_err(vnet->netdev, "%s:The page used is null!\n", __func__); + break; + } + + rx_buffer_info->pagecnt_bias--; + + /* Obtain packet length from descriptor */ + len = PCI_EP_DMA_DESC_FIELD_GET(LEN, desc->ctrl); + + /* Obtain the DMA address of the package in RC from the descriptor */ + src_addr = desc->addr; + dst_addr = rx_buffer_info->addr + rx_buffer_info->page_offset; + + spin_lock(&vnet->rx_lock); + + /* Start DMA transmission */ + ret = pci_epf_vnet_rc2ep_dma(vnet, dst_addr, + src_addr, mul32(len)); + if (ret) { + spin_unlock(&vnet->rx_lock); + netdev_err(vnet->netdev, + "dma transfer failure(err %d)\n", ret); + vnet->netdev->stats.rx_dropped++; + rx_buffer_info->pagecnt_bias++; + ep_put_rx_buffer(vnet, rx_buffer_info); + + desc->ctrl |= PCI_EP_DMA_DESC_CTRL_PKT_ERR; + desc->ctrl &= ~PCI_EP_DMA_DESC_CTRL_PKT_USED; + dma_wmb(); + goto next_cycle; + } + + timeout = (mul32(len) / DIVISOR + 1) * 10 * 8; + while (!(pci_epc_dma_status(epc, epf->func_no, DMA_READ) & + DMA_STATUS_DONE)) { + timeout -= 5; + udelay(1); + if (timeout <= 0) { + spin_unlock(&vnet->rx_lock); + netdev_err(vnet->netdev, + "%s:dma transfer timeout!\n", + __func__); + vnet->netdev->stats.rx_dropped++; + rx_buffer_info->pagecnt_bias++; + ep_put_rx_buffer(vnet, rx_buffer_info); + + desc->ctrl |= PCI_EP_DMA_DESC_CTRL_PKT_ERR; + desc->ctrl &= ~PCI_EP_DMA_DESC_CTRL_PKT_USED; + dma_wmb(); + goto next_cycle; + } + } + + spin_unlock(&vnet->rx_lock); + + dma_sync_single_range_for_cpu(epc->dev.parent, + rx_buffer_info->addr, + rx_buffer_info->page_offset, + len, DMA_FROM_DEVICE); + + pkt_data = page_address(rx_buffer_info->page) + + rx_buffer_info->page_offset - EP_SKB_PAD; + pkt_crc = crc32_le(~0, (unsigned char *)pkt_data, len); + desc_crc = PCI_EP_DMA_DESC_FIELD_GET(CHECKSUM, desc->ctrl); + + /* Compare the CRC of RC and EP */ + if (pkt_crc != desc_crc) { + vnet->netdev->stats.rx_errors++; + rx_buffer_info->pagecnt_bias++; + ep_put_rx_buffer(vnet, rx_buffer_info); + + desc->ctrl |= PCI_EP_DMA_DESC_CTRL_PKT_ERR; + desc->ctrl &= ~PCI_EP_DMA_DESC_CTRL_PKT_USED; + dma_wmb(); + goto next_cycle; + } + + skb = ep_build_skb(rx_buffer_info, len); + if (unlikely(!skb)) { + netdev_err(vnet->netdev, "rx build skb failed\n"); + vnet->netdev->stats.rx_dropped++; + rx_buffer_info->pagecnt_bias++; + break; + } + + ep_put_rx_buffer(vnet, rx_buffer_info); + + /* Modify the Ctrl key in the descriptor */ + desc->ctrl &= ~PCI_EP_DMA_DESC_CTRL_PKT_USED; + dma_wmb(); + + skb->protocol = eth_type_trans(skb, vnet->netdev); + skb->ip_summed = CHECKSUM_UNNECESSARY; + + /* Upload protocol stack */ + netif_receive_skb(skb); + + vnet->netdev->stats.rx_packets++; + vnet->netdev->stats.rx_bytes += skb->len; + +next_cycle: + /* Move the SKB position in the ring backwards */ + rx_head = (rx_head + 1) % vnet->rx_ring_size; + WRITE_ONCE(rx_queue->head, rx_head); + } + + vnet_rx_refill(vnet, false); + + return count; +} + +int pci_epf_vnet_clean_ep2rc_irq(struct pci_epf_vnet *vnet, int budget) +{ + int ret = 0; + + ret = pci_epf_vnet_rx(vnet, budget); + return ret; +} + +void pci_epf_vnet_set_irq(struct phytium_pcie_ep *priv, struct pci_epf_vnet *vnet) +{ + u32 bar1_lo_addr, bar1_hi_addr; + struct pci_epf *epf = vnet->epf; + + /* Use BAR1 as the area to trigger the EP interrupt */ + bar1_lo_addr = readl(priv->reg_base + + PHYTIUM_PCIE_FUNC_BASE(epf->func_no) + + PHYTIUM_PCI_WIN0_TRSL_ADDR0(BAR_1)); + bar1_hi_addr = readl(priv->reg_base + + PHYTIUM_PCIE_FUNC_BASE(epf->func_no) + + PHYTIUM_PCI_WIN0_TRSL_ADDR1(BAR_1)); + + writel(bar1_lo_addr, priv->hpb_base + PHYTIUM_HPB_REG_MSI64_LO_ADDR); + writel(bar1_hi_addr, priv->hpb_base + PHYTIUM_HPB_REG_MSI64_HI_ADDR); + + /* Enable Interrupt */ + writel(HPB_ERR_EVT_LOG_EN, priv->hpb_base + + PHYTIUM_HPB_REG_ERR_EVT_LOG_EN); + writel(HPB_ERR_EVT_INT_EN, priv->hpb_base + + PHYTIUM_HPB_REG_ERR_EVT_INT_EN); + writel(HPB_MSI_EN, priv->hpb_base + PHYTIUM_HPB_REG_MSI_EN); + writel(HPB_NS_MSI_SPI_EN, priv->hpb_base + + PHYTIUM_HPB_REG_NS_MSI_SPI_EN); +} + +void pci_epf_vnet_enable_irq(struct phytium_pcie_ep *priv) +{ + /* Enable interrupt reporting */ + writel(HPB_ERR_EVT_INT_EN, priv->hpb_base + + PHYTIUM_HPB_REG_ERR_EVT_INT_EN); +} + +void pci_epf_vnet_disable_irq(struct phytium_pcie_ep *priv) +{ + /* Close interrupt reporting */ + writel(HPB_ERR_EVT_INT_CLR, priv->hpb_base + + PHYTIUM_HPB_REG_ERR_EVT_INT_EN); +} + +static int pci_epf_vnet_rx_poll(struct napi_struct *napi, int budget) +{ + int work_done; + struct pci_epf_vnet *vnet = container_of(napi, + struct pci_epf_vnet, + rx_napi); + struct pci_epf *epf = vnet->epf; + struct pci_epc *epc = epf->epc; + struct phytium_pcie_ep *priv = epc_get_drvdata(epc); + + work_done = vnet->clean_rc2ep(vnet, budget); + if (work_done < budget) { + napi_complete_done(napi, work_done); + /* Enable interrupt */ + pci_epf_vnet_enable_irq(priv); + } + return work_done; +} + +static irqreturn_t pci_epf_vnet_rx_irqhandler(int irq, void *data) +{ + struct pci_epf_vnet *vnet = (struct pci_epf_vnet *)data; + struct pci_epf *epf = vnet->epf; + struct pci_epc *epc = epf->epc; + struct phytium_pcie_ep *priv = epc_get_drvdata(epc); + + if (napi_schedule_prep(&vnet->rx_napi)) { + /* Disable interrupt */ + pci_epf_vnet_disable_irq(priv); + __napi_schedule(&vnet->rx_napi); + } + + readl(priv->hpb_base + PHYTIUM_HPB_REG_NS_MSI_DATA); + return IRQ_HANDLED; +} + +static netdev_tx_t vnet_start_xmit(struct sk_buff *skb, + struct net_device *netdev) +{ + u32 tail; + u32 crc32; + int timeout; + int len; + char *data; + dma_addr_t src_addr, dst_addr = 0; + int ret; + unsigned long tx_lock_flags; + struct pci_ep_dma_desc *desc; + u64 ctrl = 0; + u32 tx_flags; + + struct pci_epf_vnet *vnet = netdev_priv(netdev); + struct pci_ep_queue *queue = vnet->queue; + struct pci_epf *epf = vnet->epf; + struct pci_epc *epc = epf->epc; + struct ep_queue *tx_queue = &queue->ep_tx_queue; + + data = skb->data; + len = skb->len; + + /* Check if RC is online */ + tx_flags = READ_ONCE(tx_queue->flags); + if (!(tx_flags & PCI_EP_VNET_FLAGS_BIT(RC_LINK_ACTIVE))) + return NETDEV_TX_BUSY; + + /* Calculate the CRC of the package */ + crc32 = crc32_le(~0, data, len); + + spin_lock_irqsave(&vnet->tx_lock, tx_lock_flags); + + /* Check if the bar space descriptor ring is full */ + if (vnet_tx_desc_queue_full(vnet)) { + spin_unlock_irqrestore(&vnet->tx_lock, tx_lock_flags); + netdev_err(vnet->netdev, "tx desc queue is full!!!\n"); + return NETDEV_TX_BUSY; + } + + /* Obtain the tail index of the bar descriptor ring */ + tail = vnet_get_tx_desc_tail_idx(vnet); + + desc = vnet_tx_desc(vnet, tail); + + /* Obtain the DMA address of the buffer on the RC end */ + dst_addr = desc->addr; + if (!dst_addr) { + spin_unlock_irqrestore(&vnet->tx_lock, tx_lock_flags); + return NETDEV_TX_BUSY; + } + + memcpy(vnet->tx_buffer_info[tail].vaddr, data, len); + + src_addr = vnet->tx_buffer_info[tail].addr; + + ctrl |= PCI_EP_DMA_DESC_CTRL_PKT_USED | + PCI_EP_DMA_DESC_FIELD_SET(LEN, len) | + PCI_EP_DMA_DESC_FIELD_SET(CHECKSUM, (u64)crc32); + + /* Start DMA transmission */ + ret = pci_epf_vnet_ep2rc_dma(vnet, src_addr, dst_addr, mul32(len), + DMA_TO_DEVICE); + if (ret) { + spin_unlock_irqrestore(&vnet->tx_lock, tx_lock_flags); + dev_kfree_skb_any(skb); + /* Packet loss statistics */ + vnet->netdev->stats.tx_dropped++; + netdev_err(vnet->netdev, "dma failed(%d)!\n", ret); + return NETDEV_TX_OK; + } + + timeout = (mul32(len) / DIVISOR + 1) * 10 * 8; + while (!(pci_epc_dma_status(epc, epf->func_no, DMA_WRITE) & + DMA_STATUS_DONE)) { + timeout -= 5; + udelay(1); + if (timeout <= 0) { + netdev_err(vnet->netdev, "%s:dma transfer timeout!\n", __func__); + spin_unlock_irqrestore(&vnet->tx_lock, tx_lock_flags); + dev_kfree_skb_any(skb); + vnet->netdev->stats.tx_dropped++; + return NETDEV_TX_OK; + } + } + + /* Move the tail pointer of the ep2rc descriptor */ + vnet_adj_tx_desc_tail_idx(vnet); + + /* Modify the Ctrl key in the descriptor */ + desc->ctrl = ctrl; + + /* Make descriptor updates visible to device */ + dma_wmb(); + + spin_unlock_irqrestore(&vnet->tx_lock, tx_lock_flags); + + vnet->netdev->stats.tx_packets++; + vnet->netdev->stats.tx_bytes += skb->len; + + dev_kfree_skb_any(skb); + + return NETDEV_TX_OK; +} + +static void pci_epf_vnet_link_detect_handle(struct timer_list *timer) +{ + struct pci_epf_vnet *vnet = container_of(timer, struct pci_epf_vnet, + link_detect_timer); + struct ep_queue *tx_queue = &vnet->queue->ep_tx_queue; + struct net_device *netdev = vnet->netdev; + u32 tx_flags = READ_ONCE(tx_queue->flags); + + /* Link detection */ + if ((tx_flags & PCI_EP_VNET_FLAGS_BIT(RC_LINK_ACTIVE)) && + !(netif_carrier_ok(netdev))) { + netif_carrier_on(netdev); + } else if (!(tx_flags & PCI_EP_VNET_FLAGS_BIT(RC_LINK_ACTIVE)) && + (netif_carrier_ok(netdev))) { + netif_carrier_off(netdev); + } + + mod_timer(timer, jiffies + msecs_to_jiffies(LINK_DETECT_PERIOD)); +} + +static int vnet_close(struct net_device *netdev) +{ + struct pci_epf_vnet *vnet = netdev_priv(netdev); + struct ep_queue *rx_queue; + struct ep_queue *tx_queue; + struct pci_epf *epf = vnet->epf; + struct pci_epc *epc = epf->epc; + struct phytium_pcie_ep *priv = epc_get_drvdata(epc); + u32 rx_flags; + + rx_queue = &vnet->queue->ep_rx_queue; + tx_queue = &vnet->queue->ep_tx_queue; + + dma_unmap_page_attrs(epc->dev.parent, epf_cdev_mdata.phys_addr, + MEM_MAX, DMA_FROM_DEVICE, EP_DMA_ATTR); + + pci_epf_vnet_disable_irq(priv); + /* Set EP end offline */ + rx_flags = READ_ONCE(rx_queue->flags); + rx_flags &= ~PCI_EP_VNET_FLAGS_BIT(EP_LINK_ACTIVE); + WRITE_ONCE(rx_queue->flags, rx_flags); + + del_timer(&vnet->link_detect_timer); + + vnet_down(netdev); + + devm_free_irq(&priv->pdev->dev, vnet->rx_irq, vnet); + + return 0; +} + +static int vnet_open(struct net_device *netdev) +{ + struct pci_epf_vnet *vnet = netdev_priv(netdev); + struct pci_epf *epf = vnet->epf; + struct pci_epc *epc; + struct phytium_pcie_ep *priv; + struct ep_queue *rx_queue; + u32 rx_flags; + int ret; + + /* Prevent opening network card devices before binding EPF and EPC */ + if (!epf->epc) + return -ENXIO; + if (!vnet->bind_success) + return -ENXIO; + + epc = epf->epc; + priv = epc_get_drvdata(epc); + + rx_queue = &vnet->queue->ep_rx_queue; + + /* Obtain interrupt number */ + vnet->rx_irq = platform_get_irq(priv->pdev, 1); + if (vnet->rx_irq < 0) { + dev_err(&priv->pdev->dev, + "Failed to obtain interrupt from device tree!\n"); + return vnet->rx_irq; + } + + /* Apply for interruption */ + ret = devm_request_irq(&priv->pdev->dev, vnet->rx_irq, + pci_epf_vnet_rx_irqhandler, IRQF_SHARED, + "pci_ep_vnet", vnet); + if (ret < 0) { + dev_err(&priv->pdev->dev, + "The EP interruption application failed\n"); + return ret; + } + + netif_carrier_off(netdev); + + ret = vnet_up(netdev); + if (ret) { + netdev_err(netdev, "pci_ep_vnet: Cannot open interface, aborting"); + return ret; + } + + timer_setup(&vnet->link_detect_timer, pci_epf_vnet_link_detect_handle, 0); + add_timer(&vnet->link_detect_timer); + + /* Set EP end online */ + rx_flags = READ_ONCE(rx_queue->flags); + rx_flags |= PCI_EP_VNET_FLAGS_BIT(EP_LINK_ACTIVE); + WRITE_ONCE(rx_queue->flags, rx_flags); + + pci_epf_vnet_set_irq(priv, vnet); + + epf_cdev_mdata.phys_addr = dma_map_page_attrs(epc->dev.parent, epf_cdev_mdata.page, 0, + MEM_MAX, DMA_FROM_DEVICE, EP_DMA_ATTR); + + return 0; +} + +static int vnet_init(struct net_device *netdev) +{ + struct pci_epf_vnet *vnet = netdev_priv(netdev); + + dev_info(vnet_to_dev(vnet), "virtual net device initialized.\n"); + return 0; +} + +static const struct net_device_ops vnet_ops = { + .ndo_init = vnet_init, + .ndo_open = vnet_open, + .ndo_stop = vnet_close, + .ndo_start_xmit = vnet_start_xmit, +}; + +static const struct pci_epf_device_id pci_epf_vnet_ids[] = { + { + .name = "pci_epf_vnet", + }, + {}, +}; + +static int epf_cdev_mmap(struct file *filp, struct vm_area_struct *vma) +{ + unsigned long pfn = epf_cdev_mdata.phys_addr >> PAGE_SHIFT; + unsigned long size = vma->vm_end - vma->vm_start; + + if (remap_pfn_range(vma, vma->vm_start, pfn, size, vma->vm_page_prot)) + return -EAGAIN; + + return 0; +} + +static int epf_cdev_open(struct inode *inode, struct file *filp) +{ + return 0; +} + +static int epf_read_rc_mem(u64 rc_addr, u64 len) +{ + int ret; + int timeout; + unsigned long lock_flags; + + if (len < 0 || len > MEM_MAX) + return -EINVAL; + + spin_lock_irqsave(&epf_cdev_mdata.cdev_vnet->rx_lock, lock_flags); + + ret = pci_epf_vnet_rc2ep_dma(epf_cdev_mdata.cdev_vnet, epf_cdev_mdata.phys_addr, + rc_addr, len); + if (ret) { + spin_unlock_irqrestore(&epf_cdev_mdata.cdev_vnet->rx_lock, lock_flags); + netdev_err(epf_cdev_mdata.cdev_vnet->netdev, + "dma transfer failure(err %d)\n", ret); + return ret; + } + + timeout = (mul32(len) / DIVISOR + 1) * 20 * 8; + while (!(pci_epc_dma_status(epf_cdev_mdata.cdev_vnet->epf->epc, + epf_cdev_mdata.cdev_vnet->epf->func_no, DMA_READ) & + DMA_STATUS_DONE)) { + timeout -= 5; + udelay(1); + if (timeout <= 0) { + spin_unlock_irqrestore(&epf_cdev_mdata.cdev_vnet->rx_lock, lock_flags); + netdev_err(epf_cdev_mdata.cdev_vnet->netdev, + "%s:dma transfer timeout!\n", + __func__); + return -ETIME; + } + } + + spin_unlock_irqrestore(&epf_cdev_mdata.cdev_vnet->rx_lock, lock_flags); + + return 0; +} + +int measure_ddr(phys_addr_t hostddr, uint8_t *bmcaddr, u32 size) +{ + int ret = 0; + u32 offset = hostddr & (PAGE_SIZE - 1); + u64 new_base = hostddr - offset; + u64 new_len = size + offset; + u64 read_len = 0; + u8 *ori_addr = bmcaddr; + void *vaddr = memremap(epf_cdev_mdata.phys_addr, MEM_MAX, MEMREMAP_WB); + + while (new_len) { + if (new_len >= MEM_MAX) + read_len = MEM_MAX; + else + read_len = new_len; + + ret = epf_read_rc_mem(new_base, read_len); + if (ret) + return ret; + + new_len -= read_len; + new_base += read_len; + + memcpy(ori_addr, (u8 *)vaddr + offset, read_len - offset); + ori_addr += read_len - offset; + + if (offset) + offset = 0; + } + + memunmap(vaddr); + + return 0; +} +EXPORT_SYMBOL(measure_ddr); + +static long epf_cdev_ioctl(struct file *file, unsigned int cmd, unsigned long arg) +{ + switch (cmd) { + case IOCTL_EPF_READ_MEM: + if (copy_from_user(&ioctl_data, (struct epf_ioctl_data __user *)arg, + sizeof(struct epf_ioctl_data))) + return -EFAULT; + + pr_debug("Received ioctl data: phy_addr=%llx, len=%d\n", + ioctl_data.ktext_addr, ioctl_data.ktext_len); + return epf_read_rc_mem(ioctl_data.ktext_addr, ioctl_data.ktext_len); + default: + return -EINVAL; + } + return 0; +} + +static int epf_cdev_release(struct inode *inode, struct file *file) +{ + pr_debug("epf_char_release\n"); + return 0; +} + +static const struct file_operations epf_fops = { + .owner = THIS_MODULE, + .open = epf_cdev_open, + .release = epf_cdev_release, + .unlocked_ioctl = epf_cdev_ioctl, + .mmap = epf_cdev_mmap +}; + +static int pci_epf_vnet_probe(struct pci_epf *epf, + const struct pci_epf_device_id *id) +{ + struct pci_epf_vnet *vnet; + struct net_device *netdev; + struct device *dev = &epf->dev; + int err; + + netdev = devm_alloc_etherdev(dev, sizeof(*vnet)); + if (!netdev) + return -ENOMEM; + + netdev->netdev_ops = &vnet_ops; + netdev->ethtool_ops = &vnet_ethtool_ops; + + dev_addr_set(netdev, dev_addr); + + /* MTU range: 46 - 3982 */ + netdev->min_mtu = ETH_ZLEN - ETH_HLEN; + netdev->max_mtu = MAX_JUMBO_FRAME_SIZE - (ETH_HLEN + ETH_FCS_LEN); + + vnet = netdev_priv(netdev); + + epf->header = &vnet_header; + vnet->epf = epf; + vnet->netdev = netdev; + + vnet->msg_enable = (1 << debug) - 1; + vnet->link_detect_timer.expires = jiffies + msecs_to_jiffies(LINK_DETECT_PERIOD); + vnet->bind_success = false; + + epf_set_drvdata(epf, vnet); + + pci_ep_vnet_get_defaults(vnet); + + spin_lock_init(&vnet->tx_lock); + spin_lock_init(&vnet->rx_lock); + + err = register_netdev(netdev); + if (err) { + dev_err(&epf->dev, "Cannot register net device, aborting.\n"); + goto netdev_reg_failed; + } + + epf_cdev_mdata.cdev_vnet = vnet; + + netif_napi_add(netdev, &vnet->rx_napi, pci_epf_vnet_rx_poll); + vnet->clean_rc2ep = pci_epf_vnet_clean_ep2rc_irq; + + /* cdev register */ + err = alloc_chrdev_region(&epf_cdev_mdata.dev_num, 0, 1, DEVICE_NAME); + if (err < 0) + goto err_unregister_chrdev; + + epf_cdev_mdata.epf_cdev = cdev_alloc(); + if (!epf_cdev_mdata.epf_cdev) { + dev_err(&epf->dev, "Failed to allocate cdev\n"); + unregister_chrdev_region(epf_cdev_mdata.dev_num, 1); + err = -ENOMEM; + goto err_unregister_chrdev; + } + + epf_cdev_mdata.epf_cdev->owner = THIS_MODULE; + + cdev_init(epf_cdev_mdata.epf_cdev, &epf_fops); + if (cdev_add(epf_cdev_mdata.epf_cdev, epf_cdev_mdata.dev_num, 1) == -1) + goto err_cdev_del; + + epf_cdev_mdata.epf_class = class_create("epf_class"); + if (IS_ERR(epf_cdev_mdata.epf_class)) { + err = PTR_ERR(epf_cdev_mdata.epf_class); + goto err_cdev_del; + } + + epf_cdev_mdata.epf_device = device_create(epf_cdev_mdata.epf_class, + NULL, epf_cdev_mdata.dev_num, + NULL, DEVICE_NAME); + if (IS_ERR(epf_cdev_mdata.epf_device)) { + err = PTR_ERR(epf_cdev_mdata.epf_device); + goto err_class_destroy; + } + + epf_cdev_mdata.page = alloc_pages(GFP_KERNEL | __GFP_DMA32, EP_CDEV_PAGE_ORDER); + if (!epf_cdev_mdata.page) { + dev_err(&epf->dev, "failed to allocate %d pages\n", EP_CDEV_PAGE_ORDER); + goto err_device_destroy; + } + + return 0; + +err_device_destroy: + device_destroy(epf_cdev_mdata.epf_class, epf_cdev_mdata.dev_num); +err_class_destroy: + class_destroy(epf_cdev_mdata.epf_class); +err_cdev_del: + cdev_del(epf_cdev_mdata.epf_cdev); +err_unregister_chrdev: + unregister_chrdev_region(epf_cdev_mdata.dev_num, 1); +netdev_reg_failed: + free_netdev(netdev); + return err; +} + +static void pci_epf_vnet_remove(struct pci_epf *epf) +{ + struct pci_epf_vnet *vnet = epf_get_drvdata(epf); + struct net_device *netdev = vnet->netdev; + + netif_napi_del(&vnet->rx_napi); + + unregister_netdev(netdev); + free_netdev(netdev); + + device_destroy(epf_cdev_mdata.epf_class, epf_cdev_mdata.dev_num); + class_destroy(epf_cdev_mdata.epf_class); + cdev_del(epf_cdev_mdata.epf_cdev); + unregister_chrdev_region(epf_cdev_mdata.dev_num, 1); + + __free_pages(epf_cdev_mdata.page, EP_CDEV_PAGE_ORDER); +} + +static int pci_epf_vnet_set_bar(struct pci_epf *epf) +{ + int ret; + struct pci_epf_bar *epf_bar; + struct pci_epc *epc = epf->epc; + struct device *dev = &epf->dev; + struct pci_epf_vnet *vnet = epf_get_drvdata(epf); + enum pci_barno vnet_reg_barno = vnet->vnet_reg_barno; + + /* Set BAR1 */ + epf_bar = &epf->bar[BAR_1]; + + ret = pci_epc_set_bar(epc, epf->func_no, epf->vfunc_no, epf_bar); + if (ret) { + pci_epf_free_space(epf, vnet->reg[vnet_reg_barno], + vnet_reg_barno, PRIMARY_INTERFACE); + dev_err(dev, "Failed to set BAR%d\n", vnet_reg_barno); + } + + /* Set BAR2 */ + epf_bar = &epf->bar[vnet_reg_barno]; + + epf_bar->flags |= upper_32_bits(epf_bar->size) ? + PCI_BASE_ADDRESS_MEM_TYPE_64 : + PCI_BASE_ADDRESS_MEM_TYPE_32; + + ret = pci_epc_set_bar(epc, epf->func_no, epf->vfunc_no, epf_bar); + if (ret) { + pci_epf_free_space(epf, vnet->reg[vnet_reg_barno], + vnet_reg_barno, PRIMARY_INTERFACE); + dev_err(dev, "Failed to set BAR%d\n", vnet_reg_barno); + } + + return 0; +} + +static int pci_epf_vnet_alloc_space(struct pci_epf *epf) +{ + struct pci_epf_vnet *vnet = epf_get_drvdata(epf); + struct device *dev = &epf->dev; + void *base; + enum pci_barno vnet_reg_barno = vnet->vnet_reg_barno; + u64 offset; + size_t alloc_size = 64 * 1024; + + /* Allocate BAR1 space */ + base = pci_epf_alloc_space(epf, alloc_size, BAR_1, + 0, PRIMARY_INTERFACE); + if (!base) { + dev_err(dev, "Failed to allocated register BAR%d space\n", BAR_1); + return -ENOMEM; + } + vnet->reg[BAR_1] = base; + + /* Allocate BAR2 space */ + alloc_size = sizeof(struct pci_ep_queue) % PAGE_SIZE ? + (sizeof(struct pci_ep_queue) / PAGE_SIZE * 4 * PAGE_SIZE) : + (sizeof(struct pci_ep_queue) * 2); + base = pci_epf_alloc_space(epf, alloc_size, vnet_reg_barno, + 0, PRIMARY_INTERFACE); + if (!base) { + dev_err(dev, "Failed to allocated register BAR%d space\n", + vnet_reg_barno); + return -ENOMEM; + } + offset = epf->bar[vnet_reg_barno].phys_addr + & (epf->bar[vnet_reg_barno].size / 2 - 1); + if (offset) + offset = epf->bar[vnet_reg_barno].size / 2 - offset; + epf->bar[vnet_reg_barno].phys_addr = + epf->bar[vnet_reg_barno].phys_addr + offset; + vnet->reg[vnet_reg_barno] = (void *)((u64)base + offset); + + return 0; +} + +static int pci_epf_vnet_bind(struct pci_epf *epf) +{ + int ret; + struct pci_epf_vnet *vnet = epf_get_drvdata(epf); + struct pci_epf_header *header = epf->header; + struct pci_epc *epc = epf->epc; + struct phytium_pcie_ep *priv = epc_get_drvdata(epc); + struct device *dev = &epf->dev; + struct pci_ep_queue *queue; + struct ep_queue *rx_queue; + struct ep_queue *tx_queue; + + if (WARN_ON_ONCE(!epc)) + return -EINVAL; + + vnet->vnet_reg_barno = BAR_2; + + ret = pci_epc_write_header(epc, epf->func_no, epf->vfunc_no, header); + if (ret) { + dev_err(dev, "Configuration header write failed\n"); + return ret; + } + + ret = pci_epf_vnet_alloc_space(epf); + if (ret) + return ret; + + ret = pci_epf_vnet_set_bar(epf); + if (ret) + return ret; + + queue = vnet->reg[vnet->vnet_reg_barno]; + vnet->queue = queue; + rx_queue = &queue->ep_rx_queue; + tx_queue = &queue->ep_tx_queue; + + /* Initialize variables in the bar space */ + WRITE_ONCE(rx_queue->head, 0); + WRITE_ONCE(rx_queue->tail, 0); + WRITE_ONCE(rx_queue->flags, 0); + rx_queue->nb_desc = vnet->rx_ring_size; + + WRITE_ONCE(tx_queue->head, 0); + WRITE_ONCE(tx_queue->tail, 0); + WRITE_ONCE(tx_queue->flags, 0); + tx_queue->nb_desc = vnet->tx_ring_size; + vnet->bind_success = true; + + /* Used to apply for interruption */ + if (!priv->pdev) { + dev_err(dev, "Failed to obtain pdev!\n"); + return -ENODEV; + } + + return 0; +} + +static void pci_epf_vnet_unbind(struct pci_epf *epf) +{ + struct pci_epf_vnet *vnet = epf_get_drvdata(epf); + struct pci_epc *epc = epf->epc; + struct pci_epf_bar *epf_bar; + int bar = vnet->vnet_reg_barno; + + pci_epc_stop(epc); + epf_bar = &epf->bar[bar]; + + if (vnet->reg[bar]) { + pci_epf_free_space(epf, vnet->reg[bar], bar, PRIMARY_INTERFACE); + pci_epc_clear_bar(epc, epf->func_no, epf->vfunc_no, epf_bar); + } + + bar = BAR_1; + epf_bar = &epf->bar[bar]; + + if (vnet->reg[bar]) { + pci_epf_free_space(epf, vnet->reg[bar], bar, PRIMARY_INTERFACE); + pci_epc_clear_bar(epc, epf->func_no, epf->vfunc_no, epf_bar); + } +} + +static struct pci_epf_ops vnet_epf_ops = { + .unbind = pci_epf_vnet_unbind, + .bind = pci_epf_vnet_bind, +}; + +static struct pci_epf_driver pci_epf_vnet = { + .driver.name = "pci_epf_vnet", + .probe = pci_epf_vnet_probe, + .remove = pci_epf_vnet_remove, + .id_table = pci_epf_vnet_ids, + .ops = &vnet_epf_ops, + .owner = THIS_MODULE, +}; + +static int __init pci_epf_vnet_init(void) +{ + int ret; + + ret = pci_epf_register_driver(&pci_epf_vnet); + if (ret) { + pr_err("Failed to register pci epf test driver --> %d\n", ret); + return ret; + } + + return 0; +} +module_init(pci_epf_vnet_init); + +static void __exit pci_epf_vnet_exit(void) +{ + pci_epf_unregister_driver(&pci_epf_vnet); +} +module_exit(pci_epf_vnet_exit); + +MODULE_DESCRIPTION("Virtual network driver based on the pcie interface in EP"); +MODULE_AUTHOR("litongfeng1497@phytium.com.cn"); +MODULE_LICENSE("GPL"); +MODULE_VERSION(DRV_VERSION); diff --git a/drivers/net/ethernet/phytium-pci-vnet/pci-ep-net/pci_epf_vnet.h b/drivers/net/ethernet/phytium-pci-vnet/pci-ep-net/pci_epf_vnet.h new file mode 100644 index 0000000000000..bd9fd14982f63 --- /dev/null +++ b/drivers/net/ethernet/phytium-pci-vnet/pci-ep-net/pci_epf_vnet.h @@ -0,0 +1,282 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * phytium pcie ep vnet Ethernet Controller driver + * + * Copyright (C) 2023-2023 PHYTIUM Corporation + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#ifndef __PCI_EPF_VNET_H__ +#define __PCI_EPF_VNET_H__ + +#include +#include +#include + +#define DRV_MODULE_NAME "pci_epf_vnet" +#define DRV_VERSION "1.0.0" + +/* IRQ type */ +#define IRQ_TYPE_LEGACY 0 +#define IRQ_TYPE_MSI 1 +#define IRQ_TYPE_MSIX 2 + +/* DMA transfer state */ +#define DMA_STATUS_DONE BIT(0) +#define DMA_STATUS_ERROR 0xfffffffe + +/* DMA transmission direction */ +#define DMA_READ 0 +#define DMA_WRITE 1 + +/* Supported Rx Buffer Sizes */ +#define VNET_RXBUFFER_1024 1024 +#define VNET_RXBUFFER_2048 2048 +#define VNET_RXBUFFER_4096 4096 +#define VNET_RXBUFFER_8192 8192 +#define VNET_RXBUFFER_16384 16384 + +#define MAX_JUMBO_FRAME_SIZE 4000 + +/* Ctrl in the bar space descriptor*/ +#define PCI_EP_DMA_DESC_CTRL_PKT_USED_OFFSET 0 /* 1=Valid packet */ + /*0=Invalid packet*/ +#define PCI_EP_DMA_DESC_CTRL_PKT_USED_SIZE 1 +#define PCI_EP_DMA_DESC_CTRL_PKT_ERR_OFFSET 1 /* 1=DMA transfer successful */ + /* 0=DMA transfer failure */ +#define PCI_EP_DMA_DESC_CTRL_PKT_ERR_SIZE 1 +#define PCI_EP_DMA_DESC_CTRL_PKT_ADDR_DIFF_OFFSET 2 /* 1=Move the receiving */ + /* address to higher position */ + /* 0=Move receiving address to lower position */ +#define PCI_EP_DMA_DESC_CTRL_PKT_ADDR_DIFF_SIZE 1 +#define PCI_EP_DMA_DESC_CTRL_PKT_MISALIGN_OFFSET 3 +#define PCI_EP_DMA_DESC_CTRL_PKT_MISALIGN_SIZE 5 +#define PCI_EP_DMA_DESC_CTRL_PKT_RESERVE_OFFSET 8 +#define PCI_EP_DMA_DESC_CTRL_PKT_RESERVE_SIZE 8 +#define PCI_EP_DMA_DESC_CTRL_PKT_LEN_OFFSET 16 /* Packet length */ +#define PCI_EP_DMA_DESC_CTRL_PKT_LEN_SIZE 16 +#define PCI_EP_DMA_DESC_CTRL_PKT_CHECKSUM_OFFSET 32 /* Checksum bit */ +#define PCI_EP_DMA_DESC_CTRL_PKT_CHECKSUM_SIZE 32 + +#define PCI_EP_DMA_DESC_CTRL_PKT_USED BIT(0) +#define PCI_EP_DMA_DESC_CTRL_PKT_ERR BIT(1) +#define PCI_EP_DMA_DESC_CTRL_PKT_ADDR_DIFF BIT(10) + +#define PCI_EP_DMA_DESC_CTRL_FIELD_MASK(name) \ + (((1 << PCI_EP_DMA_DESC_CTRL_PKT_##name##_SIZE) - 1) \ + << PCI_EP_DMA_DESC_CTRL_PKT_##name##_OFFSET) + +#define PCI_EP_DMA_DESC_FIELD_SET(name, value) \ + ((((u64)value) & ((1UL << PCI_EP_DMA_DESC_CTRL_PKT_##name##_SIZE) - 1UL)) \ + << PCI_EP_DMA_DESC_CTRL_PKT_##name##_OFFSET) + +#define PCI_EP_DMA_DESC_FIELD_GET(name, desc) \ + (((desc) >> PCI_EP_DMA_DESC_CTRL_PKT_##name##_OFFSET) \ + & ((1UL << PCI_EP_DMA_DESC_CTRL_PKT_##name##_SIZE) - 1)) + +/* Flags in the bar space */ +#define PCI_EP_VNET_FLAGS_EP_LINK_ACTIVE_OFFSET 0 +#define PCI_EP_VNET_FLAGS_RC_LINK_ACTIVE_OFFSET 1 + +#define PCI_EP_VNET_FLAGS_BIT(name) \ + (1 << PCI_EP_VNET_FLAGS_##name##_OFFSET) + +/* TX/RX descriptor defines */ +#define VNET_DEFAULT_TXD 1024 +#define VNET_MAX_TXD 4096 +#define VNET_MIN_TXD 48 + +#define VNET_DEFAULT_RXD 1024 +#define VNET_MAX_RXD 4096 +#define VNET_MIN_RXD 48 + +#define DIVISOR 0x1000 + +#define MAX_BAR_NUM 6 + +#define DMA_ALIGN_SIZE 16 /* Address alignment value of DMA */ + +#define RX_IRQ 1 + +#define PHYTIUM_PCIE_FUNC_BASE(fn) (((fn) << 14) & GENMASK(16, 14)) +#define PHYTIUM_PCI_WIN0_BASE 0x600 +#define PHYTIUM_PCI_WIN0_TRSL_ADDR0(table) (PHYTIUM_PCI_WIN0_BASE + 0X20 * (table) + 0x8) +#define PHYTIUM_PCI_WIN0_TRSL_ADDR1(table) (PHYTIUM_PCI_WIN0_BASE + 0X20 * (table) + 0xc) + +#define HPB_ERR_EVT_LOG_EN BIT(6) +#define HPB_ERR_EVT_INT_EN BIT(6) +#define HPB_ERR_EVT_INT_CLR 0x0 +#define HPB_MSI_EN 0xF +#define HPB_NS_MSI_SPI_EN 0x1 + +#define PHYTIUM_HPB_REG_ERR_EVT_LOG_EN 0x0C4 +#define PHYTIUM_HPB_REG_ERR_EVT_INT_EN 0x0C8 +#define PHYTIUM_HPB_REG_ERR_EVT_WIC 0x0D0 +#define PHYTIUM_HPB_REG_MSI_EN 0x200 +#define PHYTIUM_HPB_REG_MSI64_HI_ADDR 0x208 +#define PHYTIUM_HPB_REG_MSI64_LO_ADDR 0x20C +#define PHYTIUM_HPB_REG_NS_MSI_SPI_EN 0x608 +#define PHYTIUM_HPB_REG_NS_MSI_DATA 0x60c + +#define LINK_DETECT_PERIOD 500 + +#define EP_SKB_PAD (0) + +#define EP_DMA_ATTR \ + (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING) + +#define EP_GFP_FLAGS \ + (GFP_ATOMIC | __GFP_NOWARN | __GFP_DMA32) + +#define EP_RX_PAGE_ORDER 0 +#define EP_RX_PAGE_SIZE (PAGE_SIZE << EP_RX_PAGE_ORDER) + +#define EP_TX_PAGE_ORDER 0 +#define EP_TX_PAGE_SIZE (PAGE_SIZE << EP_TX_PAGE_ORDER) +#define EP_RXBUFFER_2048 2048 +#define RX_BUFFER_MULTIPLE 64 +#define EP_MAX_FRAME_BUILD_SKB \ + (SKB_WITH_OVERHEAD(EP_RXBUFFER_2048) - EP_SKB_PAD) +struct param_range { + u32 min; + u32 max; + u32 count; +}; + +/* Private structure of EPC */ +struct phytium_pcie_ep { + void __iomem *reg_base; + struct resource *mem_res; + void __iomem *hpb_base; + unsigned int max_regions; + unsigned long ob_region_map; + phys_addr_t *ob_addr; + phys_addr_t irq_phys_addr; + void __iomem *irq_cpu_addr; + unsigned long irq_pci_addr; + u8 irq_pci_fn; + struct pci_epc *epc; + + struct platform_device *pdev; +}; + +/* Shared descriptor */ +struct pci_ep_dma_desc { + u64 addr; + u64 ctrl; +}; + +struct ep_queue { + u32 irq_num; + u32 tail; + u32 head; + u32 nb_desc; + u32 flags; + struct pci_ep_dma_desc dma_desc_base[VNET_MAX_RXD]; +}; + +/* bar space */ +struct pci_ep_queue { + struct ep_queue ep_rx_queue; + struct ep_queue ep_tx_queue; +} __packed; + +struct ep_rx_buffer { + dma_addr_t addr; + struct page *page; + __u16 page_offset; + __u16 pagecnt_bias; +}; + +struct ep_tx_buffer { + dma_addr_t addr; + void *vaddr; + struct page *page; + __u16 page_offset; + __u16 pagecnt_bias; +}; + +/* Structure containing variables used by the shared code */ +struct pci_epf_vnet { + u32 msg_enable ____cacheline_aligned; + + void *reg[MAX_BAR_NUM]; + struct pci_epf *epf; + enum pci_barno vnet_reg_barno; + struct hrtimer timer; + struct timer_list link_detect_timer; + + u32 rx_refill_start; + unsigned int rx_refill_num; + u32 rx_ring_size; /* Shared descriptor size */ + u32 rx_buffer_size; + struct sk_buff **rx_skb; + struct ep_rx_buffer *rx_buffer_info; + int rx_buffer_len; + u32 rx_next_to_alloc; + struct param_range rxd_size; /* Records rx descriptor ring size range */ + + u32 tx_head, tx_tail; + u32 tx_ring_size; + struct param_range txd_size; + struct ep_tx_buffer *tx_buffer_info; + + struct net_device *netdev; + + /* Lock to protect tx */ + spinlock_t tx_lock; + /* Lock to protect rx */ + spinlock_t rx_lock; + + struct napi_struct rx_napi; + + struct pci_ep_queue *queue; + + int (*clean_rc2ep)(struct pci_epf_vnet *vnet, int budget); + int rx_irq; + + bool bind_success; +}; + +enum vnet_state_t { + __VNET_TESTING, + __VNET_RESETTING, + __VNET_DOWN, + __VNET_DISABLED +}; + +static struct pci_epf_header vnet_header = { + .vendorid = PCI_ANY_ID, + .deviceid = PCI_ANY_ID, + .baseclass_code = PCI_CLASS_OTHERS, + .interrupt_pin = PCI_INTERRUPT_INTA, +}; + +struct pci_epf_vnet_data { + enum pci_barno vnet_reg_barno; +}; + +/* dma read module */ +struct epf_ioctl_data { + u64 ktext_addr; + u32 ktext_len; +}; + +struct cdev_mdata { + dev_t dev_num; + struct page *page; + phys_addr_t phys_addr; + struct cdev *epf_cdev; + struct class *epf_class; + struct device *epf_device; + struct pci_epf_vnet *cdev_vnet; +}; + +#define IOCTL_EPF_READ_MEM _IOW('a', 1, struct epf_ioctl_data) +#define EP_CDEV_PAGE_ORDER 7 +#define MEM_MAX (EP_RX_PAGE_SIZE << EP_CDEV_PAGE_ORDER) +#define DEVICE_NAME "epf_cdev" +#endif /* __PCI_EPF_VNET_H__ */ diff --git a/drivers/net/ethernet/phytium/phytmac_v1.c b/drivers/net/ethernet/phytium/phytmac_v1.c index 870f07edc58fc..befb3e83e56e8 100644 --- a/drivers/net/ethernet/phytium/phytmac_v1.c +++ b/drivers/net/ethernet/phytium/phytmac_v1.c @@ -177,6 +177,20 @@ static int phytmac_pcs_software_reset(struct phytmac *pdata, int reset) return 0; } +static int phytmac_pcs_software_reset(struct phytmac *pdata, int reset) +{ + u32 value = PHYTMAC_READ(pdata, PHYTMAC_PCSCTRL); + + if (reset) + value |= PHYTMAC_BIT(PCS_RESET); + else + value &= ~PHYTMAC_BIT(PCS_RESET); + + PHYTMAC_WRITE(pdata, PHYTMAC_PCSCTRL, value); + + return 0; +} + static int phytmac_mac_linkup(struct phytmac *pdata, phy_interface_t interface, int speed, int duplex) { From 12ab0223b893d91137a8df2764e3c18f437e6027 Mon Sep 17 00:00:00 2001 From: Ma Mingrui Date: Sat, 9 May 2026 12:22:00 +0800 Subject: [PATCH 08/13] net: pci-rc-vnet: add Phytium PCIe RC VNet driver Add support for the Phytium PCIe root complex virtual Ethernet (RC VNet) driver. This driver enables the root complex side of a PCIe-based virtual Ethernet link, typically paired with a corresponding endpoint (EP) driver. Signed-off-by: Ma Mingrui Signed-off-by: Li Tongfeng Signed-off-by: Wang Yinfeng --- drivers/net/ethernet/phytium-pci-vnet/Kconfig | 3 +- .../net/ethernet/phytium-pci-vnet/Makefile | 1 + .../phytium-pci-vnet/pci-rc-net/Kconfig | 18 + .../phytium-pci-vnet/pci-rc-net/Makefile | 6 + .../phytium-pci-vnet/pci-rc-net/pci_rc_vnet.c | 1006 +++++++++++++++++ .../phytium-pci-vnet/pci-rc-net/pci_rc_vnet.h | 231 ++++ 6 files changed, 1264 insertions(+), 1 deletion(-) create mode 100644 drivers/net/ethernet/phytium-pci-vnet/pci-rc-net/Kconfig create mode 100644 drivers/net/ethernet/phytium-pci-vnet/pci-rc-net/Makefile create mode 100644 drivers/net/ethernet/phytium-pci-vnet/pci-rc-net/pci_rc_vnet.c create mode 100644 drivers/net/ethernet/phytium-pci-vnet/pci-rc-net/pci_rc_vnet.h diff --git a/drivers/net/ethernet/phytium-pci-vnet/Kconfig b/drivers/net/ethernet/phytium-pci-vnet/Kconfig index fe992cb721bb3..94934b2250df4 100644 --- a/drivers/net/ethernet/phytium-pci-vnet/Kconfig +++ b/drivers/net/ethernet/phytium-pci-vnet/Kconfig @@ -5,7 +5,7 @@ menu "Phytium PCI Virtual Ethernet Drivers" config VNET_VENDOR_PHYTIUM - bool "Phytium devices" + bool "Phytium Virtual Ethernet devices" depends on ARCH_PHYTIUM depends on HAS_IOMEM default y @@ -20,6 +20,7 @@ config VNET_VENDOR_PHYTIUM if VNET_VENDOR_PHYTIUM source "drivers/net/ethernet/phytium-pci-vnet/pci-ep-net/Kconfig" +source "drivers/net/ethernet/phytium-pci-vnet/pci-rc-net/Kconfig" endif #VNET_VENDOR_PHYTIUM endmenu diff --git a/drivers/net/ethernet/phytium-pci-vnet/Makefile b/drivers/net/ethernet/phytium-pci-vnet/Makefile index cd1ea18348653..33d25e17d2d86 100644 --- a/drivers/net/ethernet/phytium-pci-vnet/Makefile +++ b/drivers/net/ethernet/phytium-pci-vnet/Makefile @@ -4,3 +4,4 @@ # Makefile for the Phytium PCI virtual network device drivers. obj-$(CONFIG_PHYTIUM_PCI_EPF_VNET) += pci-ep-net/ +obj-$(CONFIG_PHYTIUM_PCI_RC_VNET) += pci-rc-net/ diff --git a/drivers/net/ethernet/phytium-pci-vnet/pci-rc-net/Kconfig b/drivers/net/ethernet/phytium-pci-vnet/pci-rc-net/Kconfig new file mode 100644 index 0000000000000..a0a78bc37a5d4 --- /dev/null +++ b/drivers/net/ethernet/phytium-pci-vnet/pci-rc-net/Kconfig @@ -0,0 +1,18 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Phytium PCI EP virtual network device configuration +# + +config PHYTIUM_PCI_RC_VNET + tristate "Phytium PCI Root Complex Virtual Ethernet Device" + depends on PCI + help + This driver provides support for a virtual Ethernet interface on the + Phytium PCIe Root Complex side. It is designed to work together with + a matching PCIe Endpoint function (such as PHYTIUM_PCI_EPF_VNET) to + enable Ethernet communication over PCIe links. + + If you want to use the Root Complex side of the virtual network interface + over PCIe, say Y or M. + + If unsure, say N. diff --git a/drivers/net/ethernet/phytium-pci-vnet/pci-rc-net/Makefile b/drivers/net/ethernet/phytium-pci-vnet/pci-rc-net/Makefile new file mode 100644 index 0000000000000..a0fb493975c82 --- /dev/null +++ b/drivers/net/ethernet/phytium-pci-vnet/pci-rc-net/Makefile @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0 +# Copyright(c) 2022 - 2025 Phytium Technology Co., Ltd. +# +# Makefile for the Phytium network device drivers. + +obj-$(CONFIG_PHYTIUM_PCI_RC_VNET) += pci_rc_vnet.o diff --git a/drivers/net/ethernet/phytium-pci-vnet/pci-rc-net/pci_rc_vnet.c b/drivers/net/ethernet/phytium-pci-vnet/pci-rc-net/pci_rc_vnet.c new file mode 100644 index 0000000000000..52239e22fc5af --- /dev/null +++ b/drivers/net/ethernet/phytium-pci-vnet/pci-rc-net/pci_rc_vnet.c @@ -0,0 +1,1006 @@ +// SPDX-License-Identifier: GPL-2.0 +/** + * Virtual network driver based on the pcie interface in RC + * + * Copyright (C) 2023 Phytium Corporation + * Author: netgroup@phytium.com.cn + */ + +#include "pci_rc_vnet.h" + +static int debug = 3; +static const char dev_addr[6] = {0x04, 0x0c, 0x00, 0x00, 0x0d, 0x05}; + +static const char pci_rc_vnet_gstrings_stats[][ETH_GSTRING_LEN] = { + "rx_packets", "tx_packets", "rx_bytes", "tx_bytes", "rx_errors", + "tx_errors", "rx_dropped", "tx_dropped", "multicast", "collisions", + "rx_length_errors", "rx_over_errors", "rx_crc_errors", + "rx_frame_errors", "rx_fifo_errors", "rx_missed_errors", + "tx_aborted_errors", "tx_carrier_errors", "tx_fifo_errors", + "tx_heartbeat_errors", "tx_window_errors", +}; + +#define PCI_RC_VNET_NET_STATS_LEN 21 +#define PCI_RC_VNET_STATS_LEN ARRAY_SIZE(pci_rc_vnet_gstrings_stats) + +static const char pci_rc_vnet_gstrings_test[][ETH_GSTRING_LEN] = { +}; + +#define PCI_RC_VNET_TEST_LEN ARRAY_SIZE(pci_rc_vnet_gstrings_test) + +static inline int mul32(int size) +{ + int ret = 0; + + if (size % 32) + ret = size + (32 - size % 32); + else + ret = size; + + return ret; +} + +static struct sk_buff *rc_build_skb(struct rc_rx_buffer *rx_buffer, + unsigned int size) +{ + struct sk_buff *skb; + unsigned int truesize; + void *va; + +#if (PAGE_SIZE < 8192) + truesize = RC_RX_PAGE_SIZE / 2; +#else + truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) + + SKB_DATA_ALIGN(RC_SKB_PAD + size); +#endif + + va = page_address(rx_buffer->page) + rx_buffer->page_offset; + /* prefetch first cache line of first page */ + prefetch(va); + + /* build an skb around the page buffer */ + skb = build_skb(va - RC_SKB_PAD, truesize); + if (unlikely(!skb)) + return NULL; + + /* update pointers within the skb to store the data */ + skb_reserve(skb, RC_SKB_PAD); + __skb_put(skb, size); + + /* update buffer offset */ +#if (PAGE_SIZE < 8192) + rx_buffer->page_offset ^= truesize; +#else + rx_buffer->page_offset += truesize; +#endif + + return skb; +} + +static int pci_rc_vnet_tx_poll(struct pci_rc_vnet_private *tp, int budget) +{ + int len; + int work_done = 0; + u64 ctrl; + int tx_reclaim_start = 0; + int tx_reclaim_num = 0; + struct pci_rc_vnet_dma_desc *tx_desc_ring; + unsigned long tx_reclaim_lock_flags; + + struct pci_dev *pdev = tp->pci_dev; + struct device *dev = &pdev->dev; + int ring_size = tp->params.tx_ring.count; + + if (!(READ_ONCE(tp->tx_queue->flags) & PCI_RC_VNET_FLAGS_EP_LINK)) { + netdev_info(tp->netdev, "EP Device not online!\n"); + return 0; + } + tx_desc_ring = tp->tx_queue->desc_ring; + spin_lock_irqsave(&tp->tx_reclaim_lock, tx_reclaim_lock_flags); + tx_reclaim_start = tp->tx_reclaim_start; + tx_reclaim_num = tp->tx_reclaim_num; + + /* Ensure ctrl is at least as up-to-date as used */ + dma_rmb(); + ctrl = tx_desc_ring[tx_reclaim_start].ctrl; + while ((work_done < budget) && (work_done < tx_reclaim_num) && + (!(ctrl & PCI_RC_VNET_DMA_DESC_CTRL_FIELD_MASK(USED)))) { + len = PCI_RC_VNET_DMA_DESC_FIELD_GET(PKT_LEN, ctrl); + dma_unmap_single(dev, tp->tx_phys_addr_list[tx_reclaim_start], + mul32(len), DMA_TO_DEVICE); + dev_kfree_skb_any(tp->tx_skbuff[tx_reclaim_start]); + tp->tx_skbuff[tx_reclaim_start] = NULL; + work_done++; + tx_reclaim_start++; + if (tx_reclaim_start == ring_size) + tx_reclaim_start = 0; + + if (ctrl & PCI_RC_VNET_DMA_DESC_CTRL_FIELD_MASK(ERR)) { + tp->netdev->stats.tx_dropped++; + } else { + tp->netdev->stats.tx_packets++; + tp->netdev->stats.tx_bytes += len; + } + dma_rmb(); + ctrl = tx_desc_ring[tx_reclaim_start].ctrl; + } + + tp->tx_reclaim_start = tx_reclaim_start; + tp->tx_reclaim_num -= work_done; + spin_unlock_irqrestore(&tp->tx_reclaim_lock, tx_reclaim_lock_flags); + + if (netif_queue_stopped(tp->netdev) && (tp->tx_queue->nb_desc - tp->tx_reclaim_num >= + PCI_RC_VNET_TX_QUEUE_RESTART_THRESHOLD)) { + netif_wake_queue(tp->netdev); + } + + return work_done; +} + +static netdev_tx_t pci_rc_vnet_start_xmit(struct sk_buff *skb, struct net_device *netdev) +{ + u32 tail, head; + u32 crc32; + u64 ctrl; + int len; + char *data; + unsigned long tx_lock_flags; + unsigned long tx_reclaim_lock_flags; + dma_addr_t phys_addr; + struct pci_rc_vnet_private *tp = netdev_priv(netdev); + struct pci_rc_vnet_tx_queue *tx_queue = tp->tx_queue; + struct pci_dev *pdev = tp->pci_dev; + struct device *dev = &pdev->dev; + int ring_size = tp->params.tx_ring.count; + + if (!(READ_ONCE(tp->tx_queue->flags) & PCI_RC_VNET_FLAGS_EP_LINK)) { + netdev_info(tp->netdev, "EP Device not online!\n"); + return NETDEV_TX_BUSY; + } + + data = skb->data; + len = skb->len; + + if (unlikely(tp->tx_reclaim_num >= PCI_RC_VNET_TX_QUEUE_STOP_THRESHOLD)) { + netif_stop_queue(tp->netdev); + netdev_info(tp->netdev, + "desc queue reclaim is not finished\n"); + return NETDEV_TX_BUSY; + } + + tail = READ_ONCE(tx_queue->tail); + head = READ_ONCE(tx_queue->head); + + crc32 = crc32_le(~0, data, len); + + phys_addr = dma_map_single(dev, data, mul32(len), DMA_TO_DEVICE); + + if (dma_mapping_error(dev, phys_addr)) { + netdev_err(tp->netdev, "tx map failed\n"); + tp->netdev->stats.tx_errors++; + return NETDEV_TX_BUSY; + } + + dma_sync_single_range_for_device(dev, phys_addr, 0, mul32(len), DMA_TO_DEVICE); + + ctrl = PCI_RC_VNET_DMA_DESC_SET(USED, 1) | + PCI_RC_VNET_DMA_DESC_SET(PKT_LEN, len) | + PCI_RC_VNET_DMA_DESC_SET(CHECKSUM, crc32); + + spin_lock_irqsave(&tp->tx_lock, tx_lock_flags); + + if (unlikely(head == (tail + 1) % tx_queue->nb_desc)) { + spin_unlock_irqrestore(&tp->tx_lock, tx_lock_flags); + netif_stop_queue(tp->netdev); + dma_unmap_single(dev, phys_addr, mul32(len), DMA_TO_DEVICE); + netdev_info(tp->netdev, "tx_desc_queue_is_full\n"); + return NETDEV_TX_BUSY; + } + + tx_queue->desc_ring[tail].addr = phys_addr; + tx_queue->desc_ring[tail].ctrl = ctrl; + + /* Make descriptor updates visible to device */ + wmb(); + + tail = (tail + 1) % ring_size; + WRITE_ONCE(tx_queue->tail, tail); + + spin_unlock_irqrestore(&tp->tx_lock, tx_lock_flags); + + spin_lock_irqsave(&tp->tx_reclaim_lock, tx_reclaim_lock_flags); + tp->tx_reclaim_num++; + spin_unlock_irqrestore(&tp->tx_reclaim_lock, tx_reclaim_lock_flags); + + tp->tx_skbuff[tail] = skb; + tp->tx_phys_addr_list[tail] = phys_addr; + + writel(SEND_MSI_IRQ, tp->msi_irq_addr); + + return NETDEV_TX_OK; +} + +static bool rc_alloc_mapped_page(struct pci_rc_vnet_private *tp, + struct rc_rx_buffer *rx_buffer_info) +{ + dma_addr_t paddr; + struct page *page = rx_buffer_info->page; + struct pci_rc_vnet_dma_desc *rx_desc_ring; + + rx_desc_ring = tp->rx_queue->desc_ring; + + /* since we are recycling buffers we should seldom need to alloc */ + if (likely(page)) + return true; + + page = __dev_alloc_pages(RC_GFP_FLAGS, RC_RX_PAGE_ORDER); + if (unlikely(!page)) { + netdev_err(tp->netdev, "rx alloc page failed\n"); + rx_buffer_info->page = NULL; + return false; + } + + paddr = dma_map_page_attrs(&tp->pci_dev->dev, page, 0, + RC_RX_PAGE_SIZE, + DMA_FROM_DEVICE, RC_RX_DMA_ATTR); + if (dma_mapping_error(&tp->pci_dev->dev, paddr)) { + __free_pages(page, RC_RX_PAGE_ORDER); + rx_buffer_info->page = NULL; + return false; + } + + rx_buffer_info->addr = paddr; + rx_buffer_info->page = page; + rx_buffer_info->pagecnt_bias = 1; + rx_buffer_info->page_offset = RC_SKB_PAD; + + return true; +} + +static inline int rc_calc_rx_buf_len(void) +{ +#if (PAGE_SIZE < 8192) + return rounddown(RC_MAX_FRAME_BUILD_SKB, RX_BUFFER_MULTIPLE); +#endif + return rounddown(RC_RXBUFFER_2048, RX_BUFFER_MULTIPLE); +} + +static void pci_rc_vnet_rx_refill(struct pci_rc_vnet_private *tp, bool init) +{ + unsigned int entry, space; + struct rc_rx_buffer *rx_buffer_info; + struct pci_rc_vnet_dma_desc *rx_desc_ring; + int ring_size = tp->params.rx_ring.count; + + rx_desc_ring = tp->rx_queue->desc_ring; + + if (init) + space = ring_size - 1; + else + space = CIRC_SPACE(tp->rx_refill_start, + READ_ONCE(tp->queue->rx_queue.head), + ring_size); + + while (space > 0) { + entry = tp->rx_refill_start & (ring_size - 1); + rx_buffer_info = &tp->rx_buffer_info[entry]; + if (!rc_alloc_mapped_page(tp, rx_buffer_info)) + break; + /* sync the buffer for use by the device */ + dma_sync_single_range_for_device(&tp->pci_dev->dev, rx_buffer_info->addr, + rx_buffer_info->page_offset, + tp->rx_buffer_len, DMA_FROM_DEVICE); + rx_desc_ring[tp->rx_refill_start].addr = + rx_buffer_info->addr + rx_buffer_info->page_offset; + rx_desc_ring[tp->rx_refill_start].ctrl = 0; + + dma_wmb(); + + tp->rx_refill_start = (tp->rx_refill_start + 1) & (ring_size - 1); + space--; + } + + tp->rx_next_to_alloc = tp->rx_refill_start; +} + +static int pci_rc_vnet_rx_buff_alloc(struct pci_rc_vnet_private *tp) +{ + int ring_size = tp->params.rx_ring.count; + struct pci_rc_vnet_dma_desc *desc_ring; + + tp->rx_buffer_len = rc_calc_rx_buf_len(); + desc_ring = tp->rx_queue->desc_ring; + tp->rx_buffer_info = vzalloc(ring_size * sizeof(struct rc_rx_buffer)); + if (!tp->rx_buffer_info) { + netdev_err(tp->netdev, "Unable to allocate rx_buffer_info memory\n"); + return -ENOMEM; + } + + /* Pre allocated rx skb */ + pci_rc_vnet_rx_refill(tp, true); + + return 0; +} + +static void pci_rc_vnet_rx_buff_free(struct pci_rc_vnet_private *tp) +{ + int i; + int ring_size = tp->params.rx_ring.count; + + for (i = 0; i < ring_size; i++) { + if (tp->rx_buffer_info) { + if (tp->rx_buffer_info[i].page) { + dma_sync_single_range_for_cpu(&tp->pci_dev->dev, + tp->rx_buffer_info[i].addr, + 0, RC_RX_PAGE_SIZE, + DMA_FROM_DEVICE); + /* free resources associated with mapping */ + dma_unmap_page_attrs(&tp->pci_dev->dev, + tp->rx_buffer_info[i].addr, + RC_RX_PAGE_SIZE, + DMA_FROM_DEVICE, + RC_RX_DMA_ATTR); + __page_frag_cache_drain(tp->rx_buffer_info[i].page, + tp->rx_buffer_info[i].pagecnt_bias); + } + } + } + + vfree(tp->rx_buffer_info); + tp->rx_buffer_info = NULL; +} + +static int pci_rc_vnet_tx_buff_alloc(struct pci_rc_vnet_private *tp) +{ + int ret = 0, i; + int ring_size = tp->params.tx_ring.count; + dma_addr_t *tx_phys_addr_list; + struct sk_buff **tx_skbuff; + + tx_skbuff = kcalloc(ring_size, sizeof(struct sk_buff *), GFP_KERNEL); + if (!tx_skbuff) { + ret = -ENOMEM; + goto err_failed_alloc_skbuff; + } + for (i = 0; i < ring_size; i++) + tx_skbuff[i] = NULL; + + tp->tx_skbuff = tx_skbuff; + + tx_phys_addr_list = kmalloc_array(ring_size, sizeof(dma_addr_t), GFP_KERNEL); + if (!tx_phys_addr_list) { + ret = -ENOMEM; + goto err_free_skbuff; + } + tp->tx_phys_addr_list = tx_phys_addr_list; + return 0; + +err_free_skbuff: + kfree(tp->tx_skbuff); +err_failed_alloc_skbuff: + return ret; +} + +static void pci_rc_vnet_tx_buff_clean(struct pci_rc_vnet_private *tp) +{ + int len, i; + int ring_size = tp->params.tx_ring.count; + struct pci_dev *pdev = tp->pci_dev; + struct device *dev = &pdev->dev; + unsigned long tx_lock_flag; + unsigned long tx_reclaim_lock_flag; + u32 tx_tail; + + spin_lock_irqsave(&tp->tx_lock, tx_lock_flag); + spin_lock_irqsave(&tp->tx_reclaim_lock, tx_reclaim_lock_flag); + + for (i = 0; i < ring_size; i++) { + if (tp->tx_skbuff[i]) { + len = tp->tx_skbuff[i]->len; + dma_unmap_single(dev, tp->tx_phys_addr_list[i], + mul32(len), DMA_TO_DEVICE); + tp->tx_queue->desc_ring[i].ctrl = 0; + dev_kfree_skb_any(tp->tx_skbuff[i]); + } + } + tp->tx_reclaim_num = 0; + tx_tail = READ_ONCE(tp->tx_queue->tail); + tp->tx_reclaim_start = tx_tail; + WRITE_ONCE(tp->tx_queue->head, tx_tail); + + spin_unlock_irqrestore(&tp->tx_reclaim_lock, tx_reclaim_lock_flag); + spin_unlock_irqrestore(&tp->tx_lock, tx_lock_flag); +} + +static void pci_rc_vnet_tx_buff_free(struct pci_rc_vnet_private *tp) +{ + kfree(tp->tx_skbuff); + tp->tx_skbuff = NULL; + kfree(tp->tx_phys_addr_list); + tp->tx_phys_addr_list = NULL; +} + +static int pci_rc_vnet_up(struct pci_rc_vnet_private *tp) +{ + int err = 0; + + /* Apply for DMA buffer */ + if (pci_rc_vnet_tx_buff_alloc(tp)) { + err = -ENOMEM; + netdev_err(tp->netdev, "Failed to allocate tx buffer\n"); + goto err_failed_alloc_tx_buff; + } + if (pci_rc_vnet_rx_buff_alloc(tp)) { + err = -ENOMEM; + netdev_err(tp->netdev, "Failed to allocate rx buffer\n"); + goto err_free_tx_buff; + } + + tp->netdev->flags |= IFF_UP; + + netif_wake_queue(tp->netdev); + return err; + +err_free_tx_buff: + pci_rc_vnet_tx_buff_free(tp); + +err_failed_alloc_tx_buff: + return err; +} + +static void pci_rc_vnet_down(struct pci_rc_vnet_private *tp) +{ + netif_stop_queue(tp->netdev); + + netif_carrier_off(tp->netdev); + + msleep(200); + pci_rc_vnet_tx_buff_clean(tp); + pci_rc_vnet_tx_buff_free(tp); + pci_rc_vnet_rx_buff_free(tp); +} + +static bool rc_can_reuse_rx_page(struct rc_rx_buffer *rx_buffer) +{ + unsigned int pagecnt_bias = rx_buffer->pagecnt_bias; + struct page *page = rx_buffer->page; + + /* avoid re-using remote and pfmemalloc pages */ + if (!dev_page_is_reusable(page)) + return false; + +#if (PAGE_SIZE < 8192) + /* if we are only owner of page we can reuse it */ + if (unlikely((page_ref_count(page) - pagecnt_bias) > 1)) + return false; +#else +#define RC_LAST_OFFSET \ + (SKB_WITH_OVERHEAD(PAGE_SIZE) - RC_RXBUFFER_2048) + + if (rx_buffer->page_offset > RC_LAST_OFFSET) + return false; +#endif + + /* If we have drained the page fragment pool we need to update + * the pagecnt_bias and page count so that we fully restock the + * number of references the driver holds. + */ + if (unlikely(!pagecnt_bias)) { + page_ref_add(page, USHRT_MAX); + rx_buffer->pagecnt_bias = USHRT_MAX; + } + + return true; +} + +static void rc_reuse_rx_page(struct pci_rc_vnet_private *tp, + struct rc_rx_buffer *old_buff) +{ + struct rc_rx_buffer *new_buff; + u16 nta = tp->rx_next_to_alloc; + int ring_size = tp->params.rx_ring.count; + struct pci_rc_vnet_dma_desc *rx_desc_ring; + + rx_desc_ring = tp->rx_queue->desc_ring; + + new_buff = &tp->rx_buffer_info[nta & (ring_size - 1)]; + + /* update, and store next to alloc */ + nta++; + tp->rx_next_to_alloc = (nta < ring_size) ? nta : 0; + + /* Transfer page from old buffer to new buffer. + * Move each member individually to avoid possible store + * forwarding stalls. + */ + new_buff->addr = old_buff->addr; + new_buff->page = old_buff->page; + new_buff->page_offset = old_buff->page_offset; + new_buff->pagecnt_bias = old_buff->pagecnt_bias; +} + +static void rc_put_rx_buffer(struct pci_rc_vnet_private *tp, + struct rc_rx_buffer *rx_buffer_info) +{ + if (rc_can_reuse_rx_page(rx_buffer_info)) { + /* hand second half of page back to the ring */ + rc_reuse_rx_page(tp, rx_buffer_info); + } else { + dma_unmap_page_attrs(&tp->pci_dev->dev, + rx_buffer_info->addr, + RC_RX_PAGE_SIZE, + DMA_FROM_DEVICE, + RC_RX_DMA_ATTR); + __page_frag_cache_drain(rx_buffer_info->page, + rx_buffer_info->pagecnt_bias); + } + + /* clear contents of rx_buffer */ + rx_buffer_info->page = NULL; +} + +static int pci_rc_vnet_rx(struct pci_rc_vnet_private *tp, u32 head, u32 tail) +{ + u32 desc_crc, pkt_crc; + int used; + u64 ctrl; + int pkt_len; + void *pkt_data; + + struct rc_rx_buffer *rx_buffer_info; + struct pci_rc_vnet_rx_queue *rx_queue; + struct pci_rc_vnet_dma_desc *rx_desc_ring; + struct sk_buff *skb; + int ring_size = tp->params.tx_ring.count; + unsigned int pkt_num, count = 0; + + rx_queue = tp->rx_queue; + rx_desc_ring = rx_queue->desc_ring; + + if (tail >= head) + pkt_num = tail - head; + else + pkt_num = ring_size - head + tail; + + for (count = 0; count < pkt_num; count++) { + /* Ensure ctrl is at least as up-to-date as rxused */ + dma_rmb(); + + ctrl = rx_desc_ring[head].ctrl; + used = PCI_RC_VNET_DMA_DESC_FIELD_GET(USED, ctrl); + + pkt_len = PCI_RC_VNET_DMA_DESC_FIELD_GET(PKT_LEN, ctrl); + + /* No package to receive */ + if (!used) + break; + + /* get rx buffer */ + rx_buffer_info = &tp->rx_buffer_info[head & (ring_size - 1)]; + if (!rx_buffer_info->page) { + netdev_err(tp->netdev, + "%s: The page used is null!\n", __func__); + break; + } + rx_buffer_info->pagecnt_bias--; + + dma_sync_single_range_for_cpu(&tp->pci_dev->dev, rx_buffer_info->addr, + rx_buffer_info->page_offset, + pkt_len, DMA_FROM_DEVICE); + + pkt_data = page_address(rx_buffer_info->page) + + rx_buffer_info->page_offset - RC_SKB_PAD; + pkt_crc = crc32_le(~0, (unsigned char *)pkt_data, pkt_len); + desc_crc = PCI_RC_VNET_DMA_DESC_FIELD_GET(CHECKSUM, ctrl); + if (desc_crc != pkt_crc) { + tp->netdev->stats.rx_errors++; + rx_buffer_info->pagecnt_bias++; + rc_put_rx_buffer(tp, rx_buffer_info); + goto refresh; + } + + skb = rc_build_skb(rx_buffer_info, pkt_len); + if (unlikely(!skb)) { + netdev_err(tp->netdev, "rx build skb failed\n"); + tp->netdev->stats.rx_dropped++; + rx_buffer_info->pagecnt_bias++; + break; + } + + rc_put_rx_buffer(tp, rx_buffer_info); + + skb->protocol = eth_type_trans(skb, tp->netdev); + /* Upload protocol stack */ + netif_receive_skb(skb); + + tp->netdev->stats.rx_packets++; + tp->netdev->stats.rx_bytes += pkt_len; +refresh: + head = (head + 1) % ring_size; + } + + WRITE_ONCE(rx_queue->head, head); + + pci_rc_vnet_rx_refill(tp, false); + + return count; +} + +static void pci_rc_vnet_get_drvinfo(struct net_device *netdev, struct ethtool_drvinfo *info) +{ + struct pci_rc_vnet_private *tp = netdev_priv(netdev); + + strscpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver)); + strscpy(info->version, DRV_VERSION, sizeof(info->version)); + strscpy(info->bus_info, pci_name(tp->pci_dev), + sizeof(info->bus_info)); +} + +static u32 pci_rc_vnet_get_msglevel(struct net_device *netdev) +{ + struct pci_rc_vnet_private *tp = netdev_priv(netdev); + + return tp->msg_enable; +} + +static void pci_rc_vnet_set_msglevel(struct net_device *netdev, u32 value) +{ + struct pci_rc_vnet_private *tp = netdev_priv(netdev); + + tp->msg_enable = value; +} + +static u32 pci_rc_vnet_get_link(struct net_device *netdev) +{ + struct pci_rc_vnet_private *tp = netdev_priv(netdev); + u32 tx_flags; + + tx_flags = READ_ONCE(tp->tx_queue->flags); + return tx_flags & PCI_RC_VNET_FLAGS_EP_LINK; +} + +static void pci_rc_vnet_get_strings(struct net_device *netdev, u32 stringset, u8 *data) +{ + switch (stringset) { + case ETH_SS_TEST: + memcpy(data, *pci_rc_vnet_gstrings_test, sizeof(pci_rc_vnet_gstrings_test)); + break; + case ETH_SS_STATS: + memcpy(data, *pci_rc_vnet_gstrings_stats, sizeof(pci_rc_vnet_gstrings_stats)); + break; + } +} + +static int pci_rc_vnet_get_sset_count(struct net_device *netdev, int sset) +{ + switch (sset) { + case ETH_SS_TEST: + return PCI_RC_VNET_TEST_LEN; + case ETH_SS_STATS: + return PCI_RC_VNET_STATS_LEN; + default: + return -EOPNOTSUPP; + } +} + +static void pci_rc_vnet_get_ethtool_stats(struct net_device *netdev, + struct ethtool_stats *stats, u64 *data) +{ + int i; + + for (i = 0; i < PCI_RC_VNET_NET_STATS_LEN; i++) + data[i] = ((unsigned long *)&netdev->stats)[i]; +} + +static const struct ethtool_ops pci_rc_vnet_ethtool_ops = { + .get_drvinfo = pci_rc_vnet_get_drvinfo, + .get_msglevel = pci_rc_vnet_get_msglevel, + .set_msglevel = pci_rc_vnet_set_msglevel, + .get_link = pci_rc_vnet_get_link, + .get_strings = pci_rc_vnet_get_strings, + .get_ethtool_stats = pci_rc_vnet_get_ethtool_stats, + .get_sset_count = pci_rc_vnet_get_sset_count, +}; + +static int pci_rc_vnet_open(struct net_device *dev) +{ + int err; + struct pci_rc_vnet_private *tp = netdev_priv(dev); + struct pci_rc_vnet_rx_queue *rx_queue; + u32 rx_flags; + + rx_queue = tp->rx_queue; + WRITE_ONCE(rx_queue->head, READ_ONCE(rx_queue->tail)); + + netif_carrier_off(dev); + err = pci_rc_vnet_up(tp); + if (err) + goto err; + + hrtimer_start(&tp->link_detect_timer, ns_to_ktime(PCI_RC_VNET_LINK_DETECT_PERIOD_NS), + HRTIMER_MODE_REL_SOFT); + hrtimer_start(&tp->receive_timer, ns_to_ktime(PCI_RC_VNET_RECEIVE_PERIOD_NS), + HRTIMER_MODE_REL_SOFT); + hrtimer_start(&tp->tx_poll_timer, ns_to_ktime(PCI_RC_VNET_TX_POLL_PERIOD_NS), + HRTIMER_MODE_REL_SOFT); + rx_flags |= PCI_RC_VNET_FLAGS_RC_LINK; + WRITE_ONCE(tp->rx_queue->flags, rx_flags); + + return 0; + +err: + return err; +} + +static int pci_rc_vnet_close(struct net_device *netdev) +{ + struct pci_rc_vnet_private *tp = netdev_priv(netdev); + u32 rx_flags; + + rx_flags = READ_ONCE(tp->rx_queue->flags); + rx_flags &= ~PCI_RC_VNET_FLAGS_RC_LINK; + WRITE_ONCE(tp->rx_queue->flags, rx_flags); + + hrtimer_cancel(&tp->link_detect_timer); + hrtimer_cancel(&tp->receive_timer); + hrtimer_cancel(&tp->tx_poll_timer); + + pci_rc_vnet_down(tp); + + return 0; +} + +static const struct net_device_ops pci_rc_netdev_ops = { + .ndo_open = pci_rc_vnet_open, + .ndo_stop = pci_rc_vnet_close, + .ndo_start_xmit = pci_rc_vnet_start_xmit, +}; + +static void pci_rc_vnet_get_defaults(struct pci_rc_vnet_private *tp) +{ + struct param_range rx_ring = { .min = 64, .max = 2048, .count = 512 }; + struct param_range tx_ring = { .min = 64, .max = 2048, .count = 512 }; + + tp->params.rx_ring = rx_ring; + tp->params.tx_ring = tx_ring; + tp->rx_buffer_len = PCI_RC_VNET_RXBUFFER_2048; + + tp->tx_reclaim_start = READ_ONCE(tp->tx_queue->tail); + tp->tx_reclaim_num = 0; + tp->rx_refill_start = READ_ONCE(tp->rx_queue->head); + tp->rx_next_to_alloc = tp->rx_refill_start; + + tp->params.tx_ring.count = tp->tx_queue->nb_desc; + tp->params.rx_ring.count = tp->rx_queue->nb_desc; +} + +static enum hrtimer_restart link_detect_timer_handle(struct hrtimer *timer) +{ + struct pci_rc_vnet_private *tp = + container_of(timer, struct pci_rc_vnet_private, link_detect_timer); + int ret; + + ret = READ_ONCE(tp->tx_queue->flags) & PCI_RC_VNET_FLAGS_EP_LINK; + if (ret && !(netif_carrier_ok(tp->netdev))) + netif_carrier_on(tp->netdev); + else if (!ret && netif_carrier_ok(tp->netdev)) + netif_carrier_off(tp->netdev); + + hrtimer_forward_now(timer, ns_to_ktime(PCI_RC_VNET_LINK_DETECT_PERIOD_NS)); + return HRTIMER_RESTART; +} + +static enum hrtimer_restart pci_epf_vnet_tx_poll_callback(struct hrtimer *timer) +{ + struct pci_rc_vnet_private *tp = + container_of(timer, struct pci_rc_vnet_private, tx_poll_timer); + int tx_reclaim_start, head, budget; + int ring_size = tp->params.tx_ring.count; + + tx_reclaim_start = READ_ONCE(tp->tx_reclaim_start); + head = READ_ONCE(tp->tx_queue->head); + + if (tx_reclaim_start != head) { + if (tx_reclaim_start < head) + budget = head - tx_reclaim_start; + else + budget = ring_size - tx_reclaim_start + head; + + pci_rc_vnet_tx_poll(tp, budget); + } + + hrtimer_forward_now(timer, ns_to_ktime(PCI_RC_VNET_TX_POLL_PERIOD_NS)); + return HRTIMER_RESTART; +} + +static enum hrtimer_restart pci_epf_vnet_ep2rc_callback(struct hrtimer *timer) +{ + u32 head; + u32 tail; + int ret = 0; + struct pci_rc_vnet_private *tp = + container_of(timer, struct pci_rc_vnet_private, receive_timer); + struct pci_rc_vnet_rx_queue *rx_queue; + + rx_queue = tp->rx_queue; + head = READ_ONCE(rx_queue->head); + tail = READ_ONCE(rx_queue->tail); + + /* Receive packets */ + ret = READ_ONCE(tp->tx_queue->flags) & PCI_RC_VNET_FLAGS_EP_LINK; + if (ret) { + if (head != tail) { + ret = pci_rc_vnet_rx(tp, head, tail); + if (!ret) { + netdev_err(tp->netdev, "data packet not received\n"); + goto next_cycle; + } + } + } + +next_cycle: + hrtimer_forward_now(timer, ns_to_ktime(PCI_RC_VNET_RECEIVE_PERIOD_NS)); + return HRTIMER_RESTART; +} + +static int pci_rc_vnet_probe(struct pci_dev *pdev, const struct pci_device_id *ent) +{ + int err; + int rc; + enum pci_barno bar; + struct net_device *netdev; + resource_size_t mmio_start, mmio_len; + struct pci_rc_vnet_private *tp; + uintptr_t msi_irq_addr; + enum pci_barno bar_index = BAR_2; + + if (pci_is_bridge(pdev)) + return -ENODEV; + + err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); + if (err) { + dev_err(&pdev->dev, "dma_set_mask failed\n"); + return err; + } + + netdev = alloc_etherdev(sizeof(struct pci_rc_vnet_private)); + + if (!netdev) + return -ENOMEM; + + mmio_start = pci_resource_start(pdev, 0); + mmio_len = pci_resource_len(pdev, 0); + + SET_NETDEV_DEV(netdev, &pdev->dev); + + netdev->mem_start = mmio_start; + netdev->mem_end = mmio_start + mmio_len; + + netdev->features &= ~NETIF_F_HW_CSUM; + + dev_addr_set(netdev, dev_addr); + + netdev->netdev_ops = &pci_rc_netdev_ops; + netdev->ethtool_ops = &pci_rc_vnet_ethtool_ops; + + netdev->min_mtu = ETH_ZLEN - ETH_HLEN; + netdev->max_mtu = MAX_JUMBO_FRAME_SIZE - (ETH_HLEN + ETH_FCS_LEN); + + tp = netdev_priv(netdev); + tp->pci_dev = pdev; + tp->netdev = netdev; + tp->msg_enable = (1 << debug) - 1; + + err = pci_enable_device(pdev); + if (err) { + netdev_err(tp->netdev, "Cannot enable PCI device\n"); + goto err_free_device; + } + + err = pci_request_regions(pdev, DRV_MODULE_NAME); + if (err) { + netdev_err(tp->netdev, "Cannot obtain PCI resources\n"); + goto err_disable_pdev; + } + + pci_set_master(pdev); + + for (bar = BAR_0; bar <= BAR_5; bar++) { + if (pci_resource_flags(pdev, bar) & IORESOURCE_MEM) + tp->bar[bar] = pci_ioremap_bar(pdev, bar); + } + + tp->queue = tp->bar[bar_index]; + msi_irq_addr = (uintptr_t)tp->bar[BAR_1]; + tp->msi_irq_addr = (void *)msi_irq_addr; + + tp->tx_queue = &tp->queue->tx_queue; + if (!tp->tx_queue) { + err = -ENOMEM; + netdev_err(tp->netdev, + "Cannot perform PCI tp without BAR%d\n", bar_index); + goto err_iounmap; + } + tp->rx_queue = &tp->queue->rx_queue; + if (!tp->rx_queue) { + err = -ENOMEM; + netdev_err(tp->netdev, + "Cannot perform PCI tp without BAR%d\n", bar_index); + goto err_iounmap; + } + + pci_set_drvdata(pdev, tp); + + /* Register as a network device */ + rc = register_netdev(netdev); + + if (rc) { + netdev_err(tp->netdev, "Unable to register_netdev\n"); + goto err_iounmap; + } + pci_rc_vnet_get_defaults(tp); + + spin_lock_init(&tp->tx_lock); + spin_lock_init(&tp->tx_reclaim_lock); + spin_lock_init(&tp->rx_lock); + + hrtimer_init(&tp->link_detect_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_SOFT); + tp->link_detect_timer.function = link_detect_timer_handle; + + hrtimer_init(&tp->receive_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_SOFT); + tp->receive_timer.function = pci_epf_vnet_ep2rc_callback; + + hrtimer_init(&tp->tx_poll_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_SOFT); + tp->tx_poll_timer.function = pci_epf_vnet_tx_poll_callback; + + return 0; + +err_iounmap: + for (bar = BAR_0; bar <= BAR_5; bar++) { + if (tp->bar[bar]) + pci_iounmap(pdev, tp->bar[bar]); + } + +err_disable_pdev: + pci_disable_device(pdev); + +err_free_device: + free_netdev(netdev); + return err; +} + +static void pci_rc_vnet_remove(struct pci_dev *pdev) +{ + enum pci_barno bar; + struct pci_rc_vnet_private *tp = pci_get_drvdata(pdev); + struct net_device *net_dev = tp->netdev; + + unregister_netdev(net_dev); + + for (bar = BAR_0; bar <= BAR_5; bar++) { + if (tp->bar[bar]) + pci_iounmap(pdev, tp->bar[bar]); + } + + pci_release_regions(pdev); + free_netdev(net_dev); + pci_disable_device(pdev); +} + +static const struct pci_device_id pci_rc_vnet_tbl[] = { + { PCI_DEVICE(0x16c3, 0xedda) }, + { } +}; + +static struct pci_driver pci_rc_vnet_driver = { + .name = DRV_MODULE_NAME, + .id_table = pci_rc_vnet_tbl, + .probe = pci_rc_vnet_probe, + .remove = pci_rc_vnet_remove, +}; + +module_pci_driver(pci_rc_vnet_driver); +MODULE_DESCRIPTION("Virtual network driver based on the pcie interface in RC"); +MODULE_AUTHOR("litongfeng1497@phytium.com.cn"); +MODULE_LICENSE("GPL"); +MODULE_VERSION(DRV_VERSION); diff --git a/drivers/net/ethernet/phytium-pci-vnet/pci-rc-net/pci_rc_vnet.h b/drivers/net/ethernet/phytium-pci-vnet/pci-rc-net/pci_rc_vnet.h new file mode 100644 index 0000000000000..0b640b59bc7f8 --- /dev/null +++ b/drivers/net/ethernet/phytium-pci-vnet/pci-rc-net/pci_rc_vnet.h @@ -0,0 +1,231 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * phytium pcie ep vnet Ethernet Controller driver + * + * Copyright (C) 2023-2025 PHYTIUM Corporation + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#ifndef __PCI_RC_VNET_H +#define __PCI_RC_VNET_H +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define DRV_MODULE_NAME "pci_rc_vnet" +#define DRV_VERSION "1.0.0" + +#define PCI_ENDPOINT_VNET_IRQ_NUMBER 0x00 +#define PCI_ENDPOINT_VNET_QUEUE_TAIL 0x04 +#define PCI_ENDPOINT_VNET_QUEUE_HEAD 0x08 +#define PCI_ENDPOINT_VNET_DMA_DESC_BASE 0x0c + +#define PCI_ENDPOINT_VNET_DMA_DESC_LOWER_ADDR 0x00 +#define PCI_ENDPOINT_VNET_DMA_DESC_UPPER_ADDR 0x04 +#define PCI_ENDPOINT_VNET_DMA_DESC_CTRL 0x08 + +#define PCI_RC_VNET_DMA_DESC_CTRL_PKT_USED_OFFSET 0 +#define PCI_RC_VNET_DMA_DESC_CTRL_PKT_USED_SIZE 1 +#define PCI_RC_VNET_DMA_DESC_CTRL_PKT_ERR_OFFSET 1 +#define PCI_RC_VNET_DMA_DESC_CTRL_PKT_ERR_SIZE 1 +#define PCI_RC_VNET_DMA_DESC_CTRL_PKT_ADDR_DIFF_OFFSET 2 +#define PCI_RC_VNET_DMA_DESC_CTRL_PKT_ADDR_DIFF_SIZE 1 +#define PCI_RC_VNET_DMA_DESC_CTRL_PKT_MISALIGN_OFFSET 3 +#define PCI_RC_VNET_DMA_DESC_CTRL_PKT_MISALIGN_SIZE 5 +#define PCI_RC_VNET_DMA_DESC_CTRL_PKT_RESERVE_OFFSET 8 +#define PCI_RC_VNET_DMA_DESC_CTRL_PKT_RESERVE_SIZE 8 +#define PCI_RC_VNET_DMA_DESC_CTRL_PKT_PKT_LEN_OFFSET 16 +#define PCI_RC_VNET_DMA_DESC_CTRL_PKT_PKT_LEN_SIZE 16 +#define PCI_RC_VNET_DMA_DESC_CTRL_PKT_CHECKSUM_OFFSET 32 +#define PCI_RC_VNET_DMA_DESC_CTRL_PKT_CHECKSUM_SIZE 32 + +#define PCI_RC_VNET_DMA_DESC_CTRL_PKT_USED BIT(0) + +#define PCI_RC_VNET_RXBUFFER_1024 1024 +#define PCI_RC_VNET_RXBUFFER_2048 2048 +#define PCI_RC_VNET_RXBUFFER_4096 4096 +#define PCI_RC_VNET_RXBUFFER_8192 8192 +#define PCI_RC_VNET_RXBUFFER_16384 16384 + +#define PCI_RC_VNET_DMA_BUFF_NUM 4096 +#define PCI_RC_VNET_WAKE_NET_QUEUE_THRESHOLD 10 +#define PCI_RC_VNET_TX_QUEUE_STOP_THRESHOLD (PCI_RC_VNET_DMA_BUFF_NUM - 16) +#define PCI_RC_VNET_TX_QUEUE_RESTART_THRESHOLD 64 + +#define PCI_RC_VNET_WATCHDOG_PERIOD (2 * HZ) +#define PCI_RC_VNET_LINK_DETECT_PERIOD_NS 500000000 +#define PCI_RC_VNET_RECEIVE_PERIOD_NS 5000 +#define PCI_RC_VNET_TX_POLL_PERIOD_NS 5000 +#define PCI_RC_VNET_NAPI_PULL_BUDGET 64 + +#define BAR_NUM 6 +#define TX_IRQ 2 +#define RX_IRQ 1 + +#define PCI_RC_VNET_FLAGS_EP_LINK BIT(0) +#define PCI_RC_VNET_FLAGS_RC_LINK BIT(1) + +#define MAX_JUMBO_FRAME_SIZE 4000 + +#define PCI_RC_VNET_DMA_DESC_CTRL_FIELD_MASK(name) \ + (((1UL << PCI_RC_VNET_DMA_DESC_CTRL_PKT_##name##_SIZE) - 1) \ + << PCI_RC_VNET_DMA_DESC_CTRL_PKT_##name##_OFFSET) + +#define PCI_RC_VNET_DMA_DESC_FIELD_GET(name, desc) \ + (((desc) >> PCI_RC_VNET_DMA_DESC_CTRL_PKT_##name##_OFFSET) \ + & ((1UL << PCI_RC_VNET_DMA_DESC_CTRL_PKT_##name##_SIZE) - 1)) + +#define PCI_RC_VNET_DMA_DESC_SET(name, value) \ + ((((u64)value) & ((1UL << PCI_RC_VNET_DMA_DESC_CTRL_PKT_##name##_SIZE) - 1)) \ + << PCI_RC_VNET_DMA_DESC_CTRL_PKT_##name##_OFFSET) + +#define SEND_MSI_IRQ 0xf +#define RC_SKB_PAD (0) + +#define RC_RX_DMA_ATTR \ + (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING) + +#define RC_GFP_FLAGS \ + (GFP_ATOMIC | __GFP_NOWARN) + +#define RC_RX_PAGE_ORDER 0 +#define RC_RX_PAGE_SIZE (PAGE_SIZE << RC_RX_PAGE_ORDER) + +#define RC_RXBUFFER_2048 2048 +#define RX_BUFFER_MULTIPLE 64 +#define RC_MAX_FRAME_BUILD_SKB \ + (SKB_WITH_OVERHEAD(RC_RXBUFFER_2048) - RC_SKB_PAD) +enum pci_barno { + BAR_0, + BAR_1, + BAR_2, + BAR_3, + BAR_4, + BAR_5, +}; + +struct param_range { + u32 min; + u32 max; + u32 count; +}; + +struct params { + struct param_range rx_ring; + struct param_range tx_ring; +}; + +struct skb_dma { + void *virt_addr; + dma_addr_t phys_addr; +}; + +struct pci_rc_vnet_dma_desc { + u64 addr; + u64 ctrl; +}; + +struct pci_rc_vnet_tx_queue { + u32 irq; + u32 tail; + u32 head; + u32 nb_desc; + u32 flags; + struct pci_rc_vnet_dma_desc desc_ring[PCI_RC_VNET_DMA_BUFF_NUM]; +}; + +struct pci_rc_vnet_rx_queue { + u32 irq; + u32 tail; + u32 head; + u32 nb_desc; + u32 flags; + struct pci_rc_vnet_dma_desc desc_ring[PCI_RC_VNET_DMA_BUFF_NUM]; +}; + +struct rc_rx_buffer { + dma_addr_t addr; + struct page *page; + __u16 page_offset; + __u16 pagecnt_bias; +}; + +/* bar space */ +struct pci_rc_vnet_queue { + struct pci_rc_vnet_tx_queue tx_queue; + struct pci_rc_vnet_rx_queue rx_queue; +} __packed; + +struct pci_rc_vnet_private { + u32 msg_enable ____cacheline_aligned; + int rx_buffer_len; + struct pci_dev *pci_dev; + void __iomem *bar[BAR_NUM]; + + dma_addr_t *tx_phys_addr_list; + struct sk_buff **tx_skbuff; + struct rc_rx_buffer *rx_buffer_info; + u32 rx_next_to_alloc; + + struct pci_rc_vnet_queue *queue; + + struct pci_rc_vnet_tx_queue *tx_queue; + struct pci_rc_vnet_rx_queue *rx_queue; + + struct net_device *netdev; + int irq_type; + int num_irqs; + + /* Lock to protect tx */ + spinlock_t tx_lock; + /* Lock to protect tx_reclaim_num */ + spinlock_t tx_reclaim_lock; + /* Lock to protect rx */ + spinlock_t rx_lock; + + struct params params; + + struct work_struct tx_timeout_task; + + int (*clean_ep2rc)(struct pci_rc_vnet_private *tp, int budget); + + struct napi_struct tx_napi; + struct napi_struct rx_napi; + + int tx_reclaim_start; + int tx_reclaim_num; + + int rx_refill_start; + struct hrtimer link_detect_timer; + struct hrtimer receive_timer; + struct hrtimer tx_poll_timer; + void __iomem *msi_irq_addr; +}; + +#endif //end __PCI_RC_VNET_H From 5017d6cf6dac15caa83b39ebaf9a4efe0d0933ab Mon Sep 17 00:00:00 2001 From: Ma Mingrui Date: Sat, 9 May 2026 12:26:45 +0800 Subject: [PATCH 09/13] net/phytmac: Add SGMII mode support for 2.5G/1G To ensure proper auto-negotiation with the PHY at different link speeds, the MAC requires mode-specific PHY initialization during speed transitions. This patch adds dynamic speed reconfiguration for SGMII mode, allowing the driver to switch between 2.5G/1G/100M/10M as needed. Signed-off-by: Ma Mingrui Signed-off-by: Li Tongfeng Signed-off-by: Wang Yinfeng --- drivers/net/ethernet/cadence/macb_main.c.orig | 6258 ----------------- drivers/net/ethernet/phytium/phytmac.h | 19 +- drivers/net/ethernet/phytium/phytmac_main.c | 35 +- drivers/net/ethernet/phytium/phytmac_v1.c | 49 + drivers/net/ethernet/phytium/phytmac_v1.h | 3 + drivers/net/ethernet/phytium/phytmac_v2.c | 2 + 6 files changed, 106 insertions(+), 6260 deletions(-) delete mode 100644 drivers/net/ethernet/cadence/macb_main.c.orig diff --git a/drivers/net/ethernet/cadence/macb_main.c.orig b/drivers/net/ethernet/cadence/macb_main.c.orig deleted file mode 100644 index e134c4e7cc478..0000000000000 --- a/drivers/net/ethernet/cadence/macb_main.c.orig +++ /dev/null @@ -1,6258 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Cadence MACB/GEM Ethernet Controller driver - * - * Copyright (C) 2004-2006 Atmel Corporation - */ - -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "macb.h" - -/* This structure is only used for MACB on SiFive FU540 devices */ -struct sifive_fu540_macb_mgmt { - void __iomem *reg; - unsigned long rate; - struct clk_hw hw; -}; - -#define MAX_RING_ADDR_ALLOC_TIMES 3 -#define RING_ADDR_INTERVAL 128 - -#define MACB_RX_BUFFER_SIZE 128 -#define RX_BUFFER_MULTIPLE 64 /* bytes */ - -#define DEFAULT_RX_RING_SIZE 512 /* must be power of 2 */ -#define MIN_RX_RING_SIZE 64 -#define MAX_RX_RING_SIZE 8192 -#define RX_RING_BYTES(bp) (macb_dma_desc_get_size(bp) \ - * (bp)->rx_ring_size) - -#define DEFAULT_TX_RING_SIZE 512 /* must be power of 2 */ -#define MIN_TX_RING_SIZE 64 -#define MAX_TX_RING_SIZE 4096 -#define TX_RING_BYTES(bp) (macb_dma_desc_get_size(bp) \ - * (bp)->tx_ring_size) - -/* level of occupied TX descriptors under which we wake up TX process */ -#define MACB_TX_WAKEUP_THRESH(bp) (3 * (bp)->tx_ring_size / 4) - -#define MACB_RX_INT_FLAGS (MACB_BIT(RCOMP) | MACB_BIT(ISR_ROVR)) -#define MACB_TX_ERR_FLAGS (MACB_BIT(ISR_TUND) \ - | MACB_BIT(ISR_RLE) \ - | MACB_BIT(TXERR)) -#define MACB_TX_INT_FLAGS (MACB_TX_ERR_FLAGS | MACB_BIT(TCOMP) \ - | MACB_BIT(TXUBR)) - -/* Max length of transmit frame must be a multiple of 8 bytes */ -#define MACB_TX_LEN_ALIGN 8 -#define MACB_MAX_TX_LEN ((unsigned int)((1 << MACB_TX_FRMLEN_SIZE) - 1) & ~((unsigned int)(MACB_TX_LEN_ALIGN - 1))) -/* Limit maximum TX length as per Cadence TSO errata. This is to avoid a - * false amba_error in TX path from the DMA assuming there is not enough - * space in the SRAM (16KB) even when there is. - */ -#define GEM_MAX_TX_LEN (unsigned int)(0x3FC0) - -#define GEM_MTU_MIN_SIZE ETH_MIN_MTU -#define MACB_NETIF_LSO NETIF_F_TSO - -#define MACB_WOL_HAS_MAGIC_PACKET (0x1 << 0) -#define MACB_WOL_ENABLED (0x1 << 1) - -#define HS_SPEED_100M 0 -#define HS_SPEED_1000M 1 -#define HS_SPEED_2500M 2 -#define HS_SPEED_5000M 3 -#define HS_SPEED_10000M 4 -#define MACB_SERDES_RATE_5G 0 -#define MACB_SERDES_RATE_10G 1 - -/* Graceful stop timeouts in us. We should allow up to - * 1 frame time (10 Mbits/s, full-duplex, ignoring collisions) - */ -#define MACB_HALT_TIMEOUT 14000 -#define MACB_PM_TIMEOUT 100 /* ms */ - -#define MACB_MDIO_TIMEOUT 1000000 /* in usecs */ - -static void macb_tx_unmap(struct macb *bp, - struct macb_tx_skb *tx_skb, - int budget); -static void macb_set_addr(struct macb *bp, struct macb_dma_desc *desc, - dma_addr_t addr); - -/* DMA buffer descriptor might be different size - * depends on hardware configuration: - * - * 1. dma address width 32 bits: - * word 1: 32 bit address of Data Buffer - * word 2: control - * - * 2. dma address width 64 bits: - * word 1: 32 bit address of Data Buffer - * word 2: control - * word 3: upper 32 bit address of Data Buffer - * word 4: unused - * - * 3. dma address width 32 bits with hardware timestamping: - * word 1: 32 bit address of Data Buffer - * word 2: control - * word 3: timestamp word 1 - * word 4: timestamp word 2 - * - * 4. dma address width 64 bits with hardware timestamping: - * word 1: 32 bit address of Data Buffer - * word 2: control - * word 3: upper 32 bit address of Data Buffer - * word 4: unused - * word 5: timestamp word 1 - * word 6: timestamp word 2 - */ -static unsigned int macb_dma_desc_get_size(struct macb *bp) -{ -#ifdef MACB_EXT_DESC - unsigned int desc_size; - - switch (bp->hw_dma_cap) { - case HW_DMA_CAP_64B: - desc_size = sizeof(struct macb_dma_desc) - + sizeof(struct macb_dma_desc_64); - break; - case HW_DMA_CAP_PTP: - desc_size = sizeof(struct macb_dma_desc) - + sizeof(struct macb_dma_desc_ptp); - break; - case HW_DMA_CAP_64B_PTP: - desc_size = sizeof(struct macb_dma_desc) - + sizeof(struct macb_dma_desc_64) - + sizeof(struct macb_dma_desc_ptp); - break; - default: - desc_size = sizeof(struct macb_dma_desc); - } - return desc_size; -#endif - return sizeof(struct macb_dma_desc); -} - -static unsigned int macb_adj_dma_desc_idx(struct macb *bp, unsigned int desc_idx) -{ -#ifdef MACB_EXT_DESC - switch (bp->hw_dma_cap) { - case HW_DMA_CAP_64B: - case HW_DMA_CAP_PTP: - desc_idx <<= 1; - break; - case HW_DMA_CAP_64B_PTP: - desc_idx *= 3; - break; - default: - break; - } -#endif - return desc_idx; -} - -#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT -static struct macb_dma_desc_64 *macb_64b_desc(struct macb *bp, struct macb_dma_desc *desc) -{ - return (struct macb_dma_desc_64 *)((void *)desc - + sizeof(struct macb_dma_desc)); -} -#endif - -/* Ring buffer accessors */ -static unsigned int macb_tx_ring_wrap(struct macb *bp, unsigned int index) -{ - return index & (bp->tx_ring_size - 1); -} - -static struct macb_dma_desc *macb_tx_desc(struct macb_queue *queue, - unsigned int index) -{ - index = macb_tx_ring_wrap(queue->bp, index); - index = macb_adj_dma_desc_idx(queue->bp, index); - return &queue->tx_ring[index]; -} - -static struct macb_tx_skb *macb_tx_skb(struct macb_queue *queue, - unsigned int index) -{ - return &queue->tx_skb[macb_tx_ring_wrap(queue->bp, index)]; -} - -static dma_addr_t macb_tx_dma(struct macb_queue *queue, unsigned int index) -{ - dma_addr_t offset; - - offset = macb_tx_ring_wrap(queue->bp, index) * - macb_dma_desc_get_size(queue->bp); - - return queue->tx_ring_dma + offset; -} - -static unsigned int macb_rx_ring_wrap(struct macb *bp, unsigned int index) -{ - return index & (bp->rx_ring_size - 1); -} - -static struct macb_dma_desc *macb_rx_desc(struct macb_queue *queue, unsigned int index) -{ - index = macb_rx_ring_wrap(queue->bp, index); - index = macb_adj_dma_desc_idx(queue->bp, index); - return &queue->rx_ring[index]; -} - -static void *macb_rx_buffer(struct macb_queue *queue, unsigned int index) -{ - return queue->rx_buffers + queue->bp->rx_buffer_size * - macb_rx_ring_wrap(queue->bp, index); -} - -/* I/O accessors */ -static u32 hw_readl_native(struct macb *bp, int offset) -{ - return __raw_readl(bp->regs + offset); -} - -static void hw_writel_native(struct macb *bp, int offset, u32 value) -{ - __raw_writel(value, bp->regs + offset); -} - -static u32 hw_readl(struct macb *bp, int offset) -{ - return readl_relaxed(bp->regs + offset); -} - -static void hw_writel(struct macb *bp, int offset, u32 value) -{ - writel_relaxed(value, bp->regs + offset); -} - -/* Find the CPU endianness by using the loopback bit of NCR register. When the - * CPU is in big endian we need to program swapped mode for management - * descriptor access. - */ -static bool hw_is_native_io(void __iomem *addr) -{ - u32 value = MACB_BIT(LLB); - - __raw_writel(value, addr + MACB_NCR); - value = __raw_readl(addr + MACB_NCR); - - /* Write 0 back to disable everything */ - __raw_writel(0, addr + MACB_NCR); - - return value == MACB_BIT(LLB); -} - -static bool hw_is_gem(void __iomem *addr, bool native_io) -{ - u32 id; - - if (native_io) - id = __raw_readl(addr + MACB_MID); - else - id = readl_relaxed(addr + MACB_MID); - - return MACB_BFEXT(IDNUM, id) >= 0x2; -} - -static void macb_set_hwaddr(struct macb *bp) -{ - u32 bottom; - u16 top; - - bottom = get_unaligned_le32(bp->dev->dev_addr); - macb_or_gem_writel(bp, SA1B, bottom); - top = get_unaligned_le16(bp->dev->dev_addr + 4); - macb_or_gem_writel(bp, SA1T, top); - - if (gem_has_ptp(bp)) { - gem_writel(bp, RXPTPUNI, bottom); - gem_writel(bp, TXPTPUNI, bottom); - } - - /* Clear unused address register sets */ - macb_or_gem_writel(bp, SA2B, 0); - macb_or_gem_writel(bp, SA2T, 0); - macb_or_gem_writel(bp, SA3B, 0); - macb_or_gem_writel(bp, SA3T, 0); - macb_or_gem_writel(bp, SA4B, 0); - macb_or_gem_writel(bp, SA4T, 0); -} - -static void macb_get_hwaddr(struct macb *bp) -{ - u32 bottom; - u16 top; - u8 addr[6]; - int i; - - /* Check all 4 address register for valid address */ - for (i = 0; i < 4; i++) { - bottom = macb_or_gem_readl(bp, SA1B + i * 8); - top = macb_or_gem_readl(bp, SA1T + i * 8); - - addr[0] = bottom & 0xff; - addr[1] = (bottom >> 8) & 0xff; - addr[2] = (bottom >> 16) & 0xff; - addr[3] = (bottom >> 24) & 0xff; - addr[4] = top & 0xff; - addr[5] = (top >> 8) & 0xff; - - if (is_valid_ether_addr(addr)) { - eth_hw_addr_set(bp->dev, addr); - return; - } - } - - dev_info(&bp->pdev->dev, "invalid hw address, using random\n"); - eth_hw_addr_random(bp->dev); -} - -static int macb_mdio_wait_for_idle(struct macb *bp) -{ - u32 val; - - return readx_poll_timeout(MACB_READ_NSR, bp, val, val & MACB_BIT(IDLE), - 1, MACB_MDIO_TIMEOUT); -} - -static int macb_mdio_read_c22(struct mii_bus *bus, int mii_id, int regnum) -{ - struct macb *bp = bus->priv; - int status; - - status = pm_runtime_resume_and_get(&bp->pdev->dev); - if (status < 0) - goto mdio_pm_exit; - - status = macb_mdio_wait_for_idle(bp); - if (status < 0) - goto mdio_read_exit; - - macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C22_SOF) - | MACB_BF(RW, MACB_MAN_C22_READ) - | MACB_BF(PHYA, mii_id) - | MACB_BF(REGA, regnum) - | MACB_BF(CODE, MACB_MAN_C22_CODE))); - - status = macb_mdio_wait_for_idle(bp); - if (status < 0) - goto mdio_read_exit; - - status = MACB_BFEXT(DATA, macb_readl(bp, MAN)); - -mdio_read_exit: - pm_runtime_mark_last_busy(&bp->pdev->dev); - pm_runtime_put_autosuspend(&bp->pdev->dev); -mdio_pm_exit: - return status; -} - -static int macb_mdio_read_c45(struct mii_bus *bus, int mii_id, int devad, - int regnum) -{ - struct macb *bp = bus->priv; - int status; - - status = pm_runtime_get_sync(&bp->pdev->dev); - if (status < 0) { - pm_runtime_put_noidle(&bp->pdev->dev); - goto mdio_pm_exit; - } - - status = macb_mdio_wait_for_idle(bp); - if (status < 0) - goto mdio_read_exit; - - macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C45_SOF) - | MACB_BF(RW, MACB_MAN_C45_ADDR) - | MACB_BF(PHYA, mii_id) - | MACB_BF(REGA, devad & 0x1F) - | MACB_BF(DATA, regnum & 0xFFFF) - | MACB_BF(CODE, MACB_MAN_C45_CODE))); - - status = macb_mdio_wait_for_idle(bp); - if (status < 0) - goto mdio_read_exit; - - macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C45_SOF) - | MACB_BF(RW, MACB_MAN_C45_READ) - | MACB_BF(PHYA, mii_id) - | MACB_BF(REGA, devad & 0x1F) - | MACB_BF(CODE, MACB_MAN_C45_CODE))); - - status = macb_mdio_wait_for_idle(bp); - if (status < 0) - goto mdio_read_exit; - - status = MACB_BFEXT(DATA, macb_readl(bp, MAN)); - -mdio_read_exit: - pm_runtime_mark_last_busy(&bp->pdev->dev); - pm_runtime_put_autosuspend(&bp->pdev->dev); -mdio_pm_exit: - return status; -} - -static int macb_mdio_write_c22(struct mii_bus *bus, int mii_id, int regnum, - u16 value) -{ - struct macb *bp = bus->priv; - int status; - - status = pm_runtime_resume_and_get(&bp->pdev->dev); - if (status < 0) - goto mdio_pm_exit; - - status = macb_mdio_wait_for_idle(bp); - if (status < 0) - goto mdio_write_exit; - - macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C22_SOF) - | MACB_BF(RW, MACB_MAN_C22_WRITE) - | MACB_BF(PHYA, mii_id) - | MACB_BF(REGA, regnum) - | MACB_BF(CODE, MACB_MAN_C22_CODE) - | MACB_BF(DATA, value))); - - status = macb_mdio_wait_for_idle(bp); - if (status < 0) - goto mdio_write_exit; - -mdio_write_exit: - pm_runtime_mark_last_busy(&bp->pdev->dev); - pm_runtime_put_autosuspend(&bp->pdev->dev); -mdio_pm_exit: - return status; -} - -static int macb_mdio_write_c45(struct mii_bus *bus, int mii_id, - int devad, int regnum, - u16 value) -{ - struct macb *bp = bus->priv; - int status; - - status = pm_runtime_get_sync(&bp->pdev->dev); - if (status < 0) { - pm_runtime_put_noidle(&bp->pdev->dev); - goto mdio_pm_exit; - } - - status = macb_mdio_wait_for_idle(bp); - if (status < 0) - goto mdio_write_exit; - - macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C45_SOF) - | MACB_BF(RW, MACB_MAN_C45_ADDR) - | MACB_BF(PHYA, mii_id) - | MACB_BF(REGA, devad & 0x1F) - | MACB_BF(DATA, regnum & 0xFFFF) - | MACB_BF(CODE, MACB_MAN_C45_CODE))); - - status = macb_mdio_wait_for_idle(bp); - if (status < 0) - goto mdio_write_exit; - - macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C45_SOF) - | MACB_BF(RW, MACB_MAN_C45_WRITE) - | MACB_BF(PHYA, mii_id) - | MACB_BF(REGA, devad & 0x1F) - | MACB_BF(CODE, MACB_MAN_C45_CODE) - | MACB_BF(DATA, value))); - - status = macb_mdio_wait_for_idle(bp); - if (status < 0) - goto mdio_write_exit; - -mdio_write_exit: - pm_runtime_mark_last_busy(&bp->pdev->dev); - pm_runtime_put_autosuspend(&bp->pdev->dev); -mdio_pm_exit: - return status; -} - -static void macb_init_buffers(struct macb *bp) -{ - struct macb_queue *queue; - unsigned int q; - - for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { - queue_writel(queue, RBQP, lower_32_bits(queue->rx_ring_dma)); -#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT - if (bp->hw_dma_cap & HW_DMA_CAP_64B) - queue_writel(queue, RBQPH, - upper_32_bits(queue->rx_ring_dma)); -#endif - queue_writel(queue, TBQP, lower_32_bits(queue->tx_ring_dma)); -#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT - if (bp->hw_dma_cap & HW_DMA_CAP_64B) - queue_writel(queue, TBQPH, - upper_32_bits(queue->tx_ring_dma)); -#endif - } -} - -/** - * macb_set_tx_clk() - Set a clock to a new frequency - * @bp: pointer to struct macb - * @speed: New frequency in Hz - */ -static void macb_set_tx_clk(struct macb *bp, int speed) -{ - long ferr, rate, rate_rounded; - - if (!bp->tx_clk || (bp->caps & MACB_CAPS_CLK_HW_CHG)) - return; - - /* In case of MII the PHY is the clock master */ - if (bp->phy_interface == PHY_INTERFACE_MODE_MII) - return; - - switch (speed) { - case SPEED_10: - rate = 2500000; - break; - case SPEED_100: - rate = 25000000; - break; - case SPEED_1000: - rate = 125000000; - break; - default: - return; - } - - rate_rounded = clk_round_rate(bp->tx_clk, rate); - if (rate_rounded < 0) - return; - - /* RGMII allows 50 ppm frequency error. Test and warn if this limit - * is not satisfied. - */ - ferr = abs(rate_rounded - rate); - ferr = DIV_ROUND_UP(ferr, rate / 100000); - if (ferr > 5) - netdev_warn(bp->dev, - "unable to generate target frequency: %ld Hz\n", - rate); - - if (clk_set_rate(bp->tx_clk, rate_rounded)) - netdev_err(bp->dev, "adjusting tx_clk failed.\n"); -} - -static void macb_usx_pcs_link_up(struct phylink_pcs *pcs, unsigned int mode, - phy_interface_t interface, int speed, - int duplex) -{ - struct macb *bp = container_of(pcs, struct macb, phylink_usx_pcs); - u32 config; - - config = gem_readl(bp, USX_CONTROL); - if (speed == SPEED_10000) { - config = GEM_BFINS(SERDES_RATE, MACB_SERDES_RATE_10G, config); - config = GEM_BFINS(USX_CTRL_SPEED, HS_SPEED_10000M, config); - } else if (speed == SPEED_5000) { - config = GEM_BFINS(SERDES_RATE, MACB_SERDES_RATE_5G, config); - config = GEM_BFINS(USX_CTRL_SPEED, HS_SPEED_5000M, config); - } - - config &= ~(GEM_BIT(TX_SCR_BYPASS) | GEM_BIT(RX_SCR_BYPASS)); - /* reset */ - config &= ~(GEM_BIT(SIGNAL_OK) | GEM_BIT(TX_EN)); - config |= GEM_BIT(RX_SYNC_RESET); - - gem_writel(bp, USX_CONTROL, config); - - /* enable rx and tx */ - config &= ~(GEM_BIT(RX_SYNC_RESET)); - config |= GEM_BIT(SIGNAL_OK) | GEM_BIT(TX_EN); - - gem_writel(bp, USX_CONTROL, config); -} - -static void macb_usx_pcs_get_state(struct phylink_pcs *pcs, - struct phylink_link_state *state) -{ - struct macb *bp = container_of(pcs, struct macb, phylink_usx_pcs); - u32 val; - - if (state->interface == PHY_INTERFACE_MODE_5GBASER) - state->speed = SPEED_5000; - else if (state->interface == PHY_INTERFACE_MODE_10GBASER || - state->interface == PHY_INTERFACE_MODE_USXGMII) - state->speed = bp->speed; - - state->duplex = 1; - state->an_complete = 1; - - val = gem_readl(bp, USX_STATUS); - state->link = !!(val & GEM_BIT(USX_BLOCK_LOCK)); - val = gem_readl(bp, NCFGR); - if (val & GEM_BIT(PAE)) - state->pause = MLO_PAUSE_RX; -} - -static int macb_usx_pcs_config(struct phylink_pcs *pcs, - unsigned int neg_mode, - phy_interface_t interface, - const unsigned long *advertising, - bool permit_pause_to_mac) -{ - struct macb *bp = container_of(pcs, struct macb, phylink_usx_pcs); - - gem_writel(bp, USX_CONTROL, gem_readl(bp, USX_CONTROL) | - GEM_BIT(SIGNAL_OK) | GEM_BIT(TX_EN)); - - return 0; -} - -static void macb_pcs_get_state(struct phylink_pcs *pcs, - struct phylink_link_state *state) -{ - state->link = 0; -} - -static void macb_pcs_an_restart(struct phylink_pcs *pcs) -{ - /* Not supported */ -} - -static int macb_pcs_config(struct phylink_pcs *pcs, - unsigned int neg_mode, - phy_interface_t interface, - const unsigned long *advertising, - bool permit_pause_to_mac) -{ - return 0; -} - -static const struct phylink_pcs_ops macb_phylink_usx_pcs_ops = { - .pcs_get_state = macb_usx_pcs_get_state, - .pcs_config = macb_usx_pcs_config, - .pcs_link_up = macb_usx_pcs_link_up, -}; - -static const struct phylink_pcs_ops macb_phylink_pcs_ops = { - .pcs_get_state = macb_pcs_get_state, - .pcs_an_restart = macb_pcs_an_restart, - .pcs_config = macb_pcs_config, -}; - -static void macb_mac_config(struct phylink_config *config, unsigned int mode, - const struct phylink_link_state *state) -{ - struct net_device *ndev = to_net_dev(config->dev); - struct macb *bp = netdev_priv(ndev); - unsigned long flags; - u32 old_ctrl, ctrl; - u32 old_ncr, ncr; - - spin_lock_irqsave(&bp->lock, flags); - - old_ctrl = ctrl = macb_or_gem_readl(bp, NCFGR); - old_ncr = ncr = macb_or_gem_readl(bp, NCR); - - if (bp->caps & MACB_CAPS_MACB_IS_EMAC) { - if (state->interface == PHY_INTERFACE_MODE_RMII) - ctrl |= MACB_BIT(RM9200_RMII); - } else if (macb_is_gem(bp)) { - ctrl &= ~(GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL)); - ncr &= ~GEM_BIT(ENABLE_HS_MAC); - - if (state->interface == PHY_INTERFACE_MODE_SGMII || - state->interface == PHY_INTERFACE_MODE_2500BASEX) { - ctrl |= GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL); - } else if (state->interface == PHY_INTERFACE_MODE_10GBASER || - state->interface == PHY_INTERFACE_MODE_USXGMII || - state->interface == PHY_INTERFACE_MODE_5GBASER) { - ctrl |= GEM_BIT(PCSSEL); - ncr |= GEM_BIT(ENABLE_HS_MAC); - } else if (bp->caps & MACB_CAPS_MIIONRGMII && - bp->phy_interface == PHY_INTERFACE_MODE_MII) { - ncr |= MACB_BIT(MIIONRGMII); - } - } - - /* Apply the new configuration, if any */ - if (old_ctrl ^ ctrl) - macb_or_gem_writel(bp, NCFGR, ctrl); - - if (old_ncr ^ ncr) - macb_or_gem_writel(bp, NCR, ncr); - - /* Disable AN for SGMII fixed link configuration, enable otherwise. - * Must be written after PCSSEL is set in NCFGR, - * otherwise writes will not take effect. - */ - if (macb_is_gem(bp) && (state->interface == PHY_INTERFACE_MODE_SGMII || - state->interface == PHY_INTERFACE_MODE_2500BASEX)) { - u32 pcsctrl, old_pcsctrl; - - old_pcsctrl = gem_readl(bp, PCSCNTRL); - if (mode == MLO_AN_FIXED) - pcsctrl = old_pcsctrl & ~GEM_BIT(PCSAUTONEG); - else - pcsctrl = old_pcsctrl | GEM_BIT(PCSAUTONEG); - if (old_pcsctrl != pcsctrl) - gem_writel(bp, PCSCNTRL, pcsctrl); - } - - spin_unlock_irqrestore(&bp->lock, flags); -} - -static void macb_mac_link_down(struct phylink_config *config, unsigned int mode, - phy_interface_t interface) -{ - struct net_device *ndev = to_net_dev(config->dev); - struct macb *bp = netdev_priv(ndev); - struct macb_tx_skb *tx_skb; - struct macb_queue *queue; - struct macb_dma_desc *tx_desc = NULL; - unsigned int q; - u32 ctrl; - int i; - - if (bp->use_ncsi) - ncsi_stop_dev(bp->ndev); - - if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC)) - for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) - queue_writel(queue, IDR, - bp->rx_intr_mask | MACB_TX_INT_FLAGS | MACB_BIT(HRESP)); - - /* Disable Rx and Tx */ - ctrl = macb_readl(bp, NCR) & ~(MACB_BIT(RE) | - MACB_BIT(TE)) & ~(MACB_BIT(2PT5G)); - macb_writel(bp, NCR, ctrl); - - /* Tx clean */ - for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { - spin_lock_bh(&queue->tx_ptr_lock); - for (i = 0; i < bp->tx_ring_size; i++) { - tx_skb = macb_tx_skb(queue, i); - /* free unsent skb buffers */ - if (tx_skb) - macb_tx_unmap(bp, tx_skb, 0); - - tx_desc = macb_tx_desc(queue, i); - macb_set_addr(bp, tx_desc, 0); - tx_desc->ctrl &= ~MACB_BIT(TX_USED); - } - spin_unlock_bh(&queue->tx_ptr_lock); - } - - netif_tx_stop_all_queues(ndev); -} - -static void phytium_gem1p0_sel_clk(struct macb *bp, int speed) -{ - if (bp->phy_interface == PHY_INTERFACE_MODE_10GBASER || - bp->phy_interface == PHY_INTERFACE_MODE_USXGMII) { - gem_writel(bp, SRC_SEL_LN, 0x1); /*0x1c04*/ - if (speed == SPEED_5000) { - gem_writel(bp, DIV_SEL0_LN, 0x8); /*0x1c08*/ - gem_writel(bp, DIV_SEL1_LN, 0x2); /*0x1c0c*/ - gem_writel(bp, PMA_XCVR_POWER_STATE, 0x0); /*0x1c10*/ - } else { - gem_writel(bp, DIV_SEL0_LN, 0x4); /*0x1c08*/ - gem_writel(bp, DIV_SEL1_LN, 0x1); /*0x1c0c*/ - gem_writel(bp, PMA_XCVR_POWER_STATE, 0x1); /*0x1c10*/ - } - } else if (bp->phy_interface == PHY_INTERFACE_MODE_5GBASER) { - gem_writel(bp, SRC_SEL_LN, 0x1); /*0x1c04*/ - gem_writel(bp, DIV_SEL0_LN, 0x8); /*0x1c08*/ - gem_writel(bp, DIV_SEL1_LN, 0x2); /*0x1c0c*/ - gem_writel(bp, PMA_XCVR_POWER_STATE, 0x0); /*0x1c10*/ - } else if (bp->phy_interface == PHY_INTERFACE_MODE_2500BASEX) { - gem_writel(bp, SRC_SEL_LN, 0x1); /*0x1c04*/ - gem_writel(bp, DIV_SEL0_LN, 0x1); /*0x1c08*/ - gem_writel(bp, DIV_SEL1_LN, 0x2); /*0x1c0c*/ - gem_writel(bp, PMA_XCVR_POWER_STATE, 0x1); /*0x1c10*/ - gem_writel(bp, TX_CLK_SEL0, 0x0); /*0x1c20*/ - gem_writel(bp, TX_CLK_SEL1, 0x1); /*0x1c24*/ - gem_writel(bp, TX_CLK_SEL2, 0x1); /*0x1c28*/ - gem_writel(bp, TX_CLK_SEL3, 0x1); /*0x1c2c*/ - gem_writel(bp, RX_CLK_SEL0, 0x1); /*0x1c30*/ - gem_writel(bp, RX_CLK_SEL1, 0x0); /*0x1c34*/ - gem_writel(bp, TX_CLK_SEL3_0, 0x0); /*0x1c70*/ - gem_writel(bp, TX_CLK_SEL4_0, 0x0); /*0x1c74*/ - gem_writel(bp, RX_CLK_SEL3_0, 0x0); /*0x1c78*/ - gem_writel(bp, RX_CLK_SEL4_0, 0x0); /*0x1c7c*/ - } else if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) { - if (speed == SPEED_2500) { - gem_writel(bp, DIV_SEL0_LN, 0x1); /*0x1c08*/ - gem_writel(bp, DIV_SEL1_LN, 0x2); /*0x1c0c*/ - gem_writel(bp, PMA_XCVR_POWER_STATE, 0x1); /*0x1c10*/ - gem_writel(bp, TX_CLK_SEL0, 0x0); /*0x1c20*/ - gem_writel(bp, TX_CLK_SEL1, 0x1); /*0x1c24*/ - gem_writel(bp, TX_CLK_SEL2, 0x1); /*0x1c28*/ - gem_writel(bp, TX_CLK_SEL3, 0x1); /*0x1c2c*/ - gem_writel(bp, RX_CLK_SEL0, 0x1); /*0x1c30*/ - gem_writel(bp, RX_CLK_SEL1, 0x0); /*0x1c34*/ - gem_writel(bp, TX_CLK_SEL3_0, 0x0); /*0x1c70*/ - gem_writel(bp, TX_CLK_SEL4_0, 0x0); /*0x1c74*/ - gem_writel(bp, RX_CLK_SEL3_0, 0x0); /*0x1c78*/ - gem_writel(bp, RX_CLK_SEL4_0, 0x0); /*0x1c7c*/ - } else if (speed == SPEED_1000) { - gem_writel(bp, SRC_SEL_LN, 0x1); /*0x1c04*/ - gem_writel(bp, DIV_SEL0_LN, 0x4); /*0x1c08*/ - gem_writel(bp, DIV_SEL1_LN, 0x8); /*0x1c0c*/ - gem_writel(bp, PMA_XCVR_POWER_STATE, 0x1); /*0x1c10*/ - gem_writel(bp, TX_CLK_SEL0, 0x0); /*0x1c20*/ - gem_writel(bp, TX_CLK_SEL1, 0x0); /*0x1c24*/ - gem_writel(bp, TX_CLK_SEL2, 0x0); /*0x1c28*/ - gem_writel(bp, TX_CLK_SEL3, 0x1); /*0x1c2c*/ - gem_writel(bp, RX_CLK_SEL0, 0x1); /*0x1c30*/ - gem_writel(bp, RX_CLK_SEL1, 0x0); /*0x1c34*/ - gem_writel(bp, TX_CLK_SEL3_0, 0x0); /*0x1c70*/ - gem_writel(bp, TX_CLK_SEL4_0, 0x0); /*0x1c74*/ - gem_writel(bp, RX_CLK_SEL3_0, 0x0); /*0x1c78*/ - gem_writel(bp, RX_CLK_SEL4_0, 0x0); /*0x1c7c*/ - } else if (speed == SPEED_100 || speed == SPEED_10) { - gem_writel(bp, SRC_SEL_LN, 0x1); /*0x1c04*/ - gem_writel(bp, DIV_SEL0_LN, 0x4); /*0x1c08*/ - gem_writel(bp, DIV_SEL1_LN, 0x8); /*0x1c0c*/ - gem_writel(bp, PMA_XCVR_POWER_STATE, 0x1); /*0x1c10*/ - gem_writel(bp, TX_CLK_SEL0, 0x0); /*0x1c20*/ - gem_writel(bp, TX_CLK_SEL1, 0x0); /*0x1c24*/ - gem_writel(bp, TX_CLK_SEL2, 0x1); /*0x1c28*/ - gem_writel(bp, TX_CLK_SEL3, 0x1); /*0x1c2c*/ - gem_writel(bp, RX_CLK_SEL0, 0x1); /*0x1c30*/ - gem_writel(bp, RX_CLK_SEL1, 0x0); /*0x1c34*/ - gem_writel(bp, TX_CLK_SEL3_0, 0x1); /*0x1c70*/ - gem_writel(bp, TX_CLK_SEL4_0, 0x0); /*0x1c74*/ - gem_writel(bp, RX_CLK_SEL3_0, 0x0); /*0x1c78*/ - gem_writel(bp, RX_CLK_SEL4_0, 0x1); /*0x1c7c*/ - } - } else if (bp->phy_interface == PHY_INTERFACE_MODE_RGMII || - bp->phy_interface == PHY_INTERFACE_MODE_RGMII_ID) { - if (speed == SPEED_1000) { - gem_writel(bp, MII_SELECT, 0x1); /*0x1c18*/ - gem_writel(bp, SEL_MII_ON_RGMII, 0x0); /*0x1c1c*/ - gem_writel(bp, TX_CLK_SEL0, 0x0); /*0x1c20*/ - gem_writel(bp, TX_CLK_SEL1, 0x1); /*0x1c24*/ - gem_writel(bp, TX_CLK_SEL2, 0x0); /*0x1c28*/ - gem_writel(bp, TX_CLK_SEL3, 0x0); /*0x1c2c*/ - gem_writel(bp, RX_CLK_SEL0, 0x0); /*0x1c30*/ - gem_writel(bp, RX_CLK_SEL1, 0x1); /*0x1c34*/ - gem_writel(bp, CLK_250M_DIV10_DIV100_SEL, 0x0); /*0x1c38*/ - gem_writel(bp, RX_CLK_SEL5, 0x1); /*0x1c48*/ - gem_writel(bp, RGMII_TX_CLK_SEL0, 0x1); /*0x1c80*/ - gem_writel(bp, RGMII_TX_CLK_SEL1, 0x0); /*0x1c84*/ - } else if (speed == SPEED_100) { - gem_writel(bp, MII_SELECT, 0x1); /*0x1c18*/ - gem_writel(bp, SEL_MII_ON_RGMII, 0x0); /*0x1c1c*/ - gem_writel(bp, TX_CLK_SEL0, 0x0); /*0x1c20*/ - gem_writel(bp, TX_CLK_SEL1, 0x1); /*0x1c24*/ - gem_writel(bp, TX_CLK_SEL2, 0x0); /*0x1c28*/ - gem_writel(bp, TX_CLK_SEL3, 0x0); /*0x1c2c*/ - gem_writel(bp, RX_CLK_SEL0, 0x0); /*0x1c30*/ - gem_writel(bp, RX_CLK_SEL1, 0x1); /*0x1c34*/ - gem_writel(bp, CLK_250M_DIV10_DIV100_SEL, 0x0); /*0x1c38*/ - gem_writel(bp, RX_CLK_SEL5, 0x1); /*0x1c48*/ - gem_writel(bp, RGMII_TX_CLK_SEL0, 0x0); /*0x1c80*/ - gem_writel(bp, RGMII_TX_CLK_SEL1, 0x0); /*0x1c84*/ - } else { - gem_writel(bp, MII_SELECT, 0x1); /*0x1c18*/ - gem_writel(bp, SEL_MII_ON_RGMII, 0x0); /*0x1c1c*/ - gem_writel(bp, TX_CLK_SEL0, 0x0); /*0x1c20*/ - gem_writel(bp, TX_CLK_SEL1, 0x1); /*0x1c24*/ - gem_writel(bp, TX_CLK_SEL2, 0x0); /*0x1c28*/ - gem_writel(bp, TX_CLK_SEL3, 0x0); /*0x1c2c*/ - gem_writel(bp, RX_CLK_SEL0, 0x0); /*0x1c30*/ - gem_writel(bp, RX_CLK_SEL1, 0x1); /*0x1c34*/ - gem_writel(bp, CLK_250M_DIV10_DIV100_SEL, 0x1); /*0x1c38*/ - gem_writel(bp, RX_CLK_SEL5, 0x1); /*0x1c48*/ - gem_writel(bp, RGMII_TX_CLK_SEL0, 0x0); /*0x1c80*/ - gem_writel(bp, RGMII_TX_CLK_SEL1, 0x0); /*0x1c84*/ - } - } else if (bp->phy_interface == PHY_INTERFACE_MODE_RMII) { - gem_writel(bp, RX_CLK_SEL5, 0x1); /*0x1c48*/ - } - - if (speed == SPEED_100) - gem_writel(bp, HS_MAC_CONFIG, GEM_BFINS(HS_MAC_SPEED, HS_SPEED_100M, - gem_readl(bp, HS_MAC_CONFIG))); - else if (speed == SPEED_1000) - gem_writel(bp, HS_MAC_CONFIG, GEM_BFINS(HS_MAC_SPEED, HS_SPEED_1000M, - gem_readl(bp, HS_MAC_CONFIG))); - else if (speed == SPEED_2500) - gem_writel(bp, HS_MAC_CONFIG, GEM_BFINS(HS_MAC_SPEED, HS_SPEED_2500M, - gem_readl(bp, HS_MAC_CONFIG))); - else if (speed == SPEED_5000) - gem_writel(bp, HS_MAC_CONFIG, GEM_BFINS(HS_MAC_SPEED, HS_SPEED_5000M, - gem_readl(bp, HS_MAC_CONFIG))); - else if (speed == SPEED_10000) - gem_writel(bp, HS_MAC_CONFIG, GEM_BFINS(HS_MAC_SPEED, HS_SPEED_10000M, - gem_readl(bp, HS_MAC_CONFIG))); -} - -static void phytium_gem2p0_sel_clk(struct macb *bp, int speed) -{ - if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) { - if (speed == SPEED_100 || speed == SPEED_10) { - gem_writel(bp, SRC_SEL_LN, 0x1); /*0x1c04*/ - gem_writel(bp, DIV_SEL1_LN, 0x1); /*0x1c0c*/ - } - } - - if (speed == SPEED_100 || speed == SPEED_10) - gem_writel(bp, HS_MAC_CONFIG, GEM_BFINS(HS_MAC_SPEED, HS_SPEED_100M, - gem_readl(bp, HS_MAC_CONFIG))); - else if (speed == SPEED_1000) - gem_writel(bp, HS_MAC_CONFIG, GEM_BFINS(HS_MAC_SPEED, HS_SPEED_1000M, - gem_readl(bp, HS_MAC_CONFIG))); - else if (speed == SPEED_2500) - gem_writel(bp, HS_MAC_CONFIG, GEM_BFINS(HS_MAC_SPEED, HS_SPEED_2500M, - gem_readl(bp, HS_MAC_CONFIG))); -} - -/* Use juggling algorithm to left rotate tx ring and tx skb array */ -static void gem_shuffle_tx_one_ring(struct macb_queue *queue) -{ - unsigned int head, tail, count, ring_size, desc_size; - struct macb_tx_skb tx_skb, *skb_curr, *skb_next; - struct macb_dma_desc *desc_curr, *desc_next; - unsigned int i, cycles, shift, curr, next; - struct macb *bp = queue->bp; - unsigned char desc[24]; - unsigned long flags; - - desc_size = macb_dma_desc_get_size(bp); - - if (WARN_ON_ONCE(desc_size > ARRAY_SIZE(desc))) - return; - - spin_lock_irqsave(&queue->tx_ptr_lock, flags); - head = queue->tx_head; - tail = queue->tx_tail; - ring_size = bp->tx_ring_size; - count = CIRC_CNT(head, tail, ring_size); - - if (!(tail % ring_size)) - goto unlock; - - if (!count) { - queue->tx_head = 0; - queue->tx_tail = 0; - goto unlock; - } - - shift = tail % ring_size; - cycles = gcd(ring_size, shift); - - for (i = 0; i < cycles; i++) { - memcpy(&desc, macb_tx_desc(queue, i), desc_size); - memcpy(&tx_skb, macb_tx_skb(queue, i), - sizeof(struct macb_tx_skb)); - - curr = i; - next = (curr + shift) % ring_size; - - while (next != i) { - desc_curr = macb_tx_desc(queue, curr); - desc_next = macb_tx_desc(queue, next); - - memcpy(desc_curr, desc_next, desc_size); - - if (next == ring_size - 1) - desc_curr->ctrl &= ~MACB_BIT(TX_WRAP); - if (curr == ring_size - 1) - desc_curr->ctrl |= MACB_BIT(TX_WRAP); - - skb_curr = macb_tx_skb(queue, curr); - skb_next = macb_tx_skb(queue, next); - memcpy(skb_curr, skb_next, sizeof(struct macb_tx_skb)); - - curr = next; - next = (curr + shift) % ring_size; - } - - desc_curr = macb_tx_desc(queue, curr); - memcpy(desc_curr, &desc, desc_size); - if (i == ring_size - 1) - desc_curr->ctrl &= ~MACB_BIT(TX_WRAP); - if (curr == ring_size - 1) - desc_curr->ctrl |= MACB_BIT(TX_WRAP); - memcpy(macb_tx_skb(queue, curr), &tx_skb, - sizeof(struct macb_tx_skb)); - } - - queue->tx_head = count; - queue->tx_tail = 0; - - /* Make descriptor updates visible to hardware */ - wmb(); - -unlock: - spin_unlock_irqrestore(&queue->tx_ptr_lock, flags); -} - -/* Rotate the queue so that the tail is at index 0 */ -static void gem_shuffle_tx_rings(struct macb *bp) -{ - struct macb_queue *queue; - int q; - - for (q = 0, queue = bp->queues; q < bp->num_queues; q++, queue++) - gem_shuffle_tx_one_ring(queue); -} - -static void macb_mac_link_up(struct phylink_config *config, - struct phy_device *phy, - unsigned int mode, phy_interface_t interface, - int speed, int duplex, - bool tx_pause, bool rx_pause) -{ - struct net_device *ndev = to_net_dev(config->dev); - struct macb *bp = netdev_priv(ndev); - struct macb_queue *queue; - unsigned long flags; - unsigned int q; - u32 ctrl; - int err; - - spin_lock_irqsave(&bp->lock, flags); - - if (bp->caps & MACB_CAPS_SEL_CLK) - bp->sel_clk_hw(bp, speed); - - ctrl = macb_or_gem_readl(bp, NCFGR); - - ctrl &= ~(MACB_BIT(SPD) | MACB_BIT(FD)); - - if (speed == SPEED_100) - ctrl |= MACB_BIT(SPD); - - if (duplex) - ctrl |= MACB_BIT(FD); - - if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC)) { - ctrl &= ~MACB_BIT(PAE); - if (macb_is_gem(bp)) { - ctrl &= ~GEM_BIT(GBE); - - if (speed == SPEED_1000 || speed == SPEED_2500) - ctrl |= GEM_BIT(GBE); - } - - if (rx_pause) - ctrl |= MACB_BIT(PAE); - - macb_set_tx_clk(bp, speed); - - for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { - queue_writel(queue, IER, - bp->rx_intr_mask | MACB_TX_INT_FLAGS | MACB_BIT(HRESP)); - } - } - - macb_or_gem_writel(bp, NCFGR, ctrl); - - if (speed == SPEED_2500) { - u32 network_ctrl; - u32 pcsctrl, old_pcsctrl; - - network_ctrl = macb_readl(bp, NCR); - network_ctrl |= MACB_BIT(2PT5G); - macb_writel(bp, NCR, network_ctrl); - - old_pcsctrl = gem_readl(bp, PCSCNTRL); - pcsctrl = old_pcsctrl & ~GEM_BIT(PCSAUTONEG); - if (old_pcsctrl != pcsctrl) - gem_writel(bp, PCSCNTRL, pcsctrl); - } - - if (bp->phy_interface == PHY_INTERFACE_MODE_10GBASER || - bp->phy_interface == PHY_INTERFACE_MODE_USXGMII) { - if (speed == SPEED_5000) - gem_writel(bp, HS_MAC_CONFIG, - GEM_BFINS(HS_MAC_SPEED, HS_SPEED_5000M, - gem_readl(bp, HS_MAC_CONFIG))); - else - gem_writel(bp, HS_MAC_CONFIG, - GEM_BFINS(HS_MAC_SPEED, HS_SPEED_10000M, - gem_readl(bp, HS_MAC_CONFIG))); - } else if (bp->phy_interface == PHY_INTERFACE_MODE_5GBASER) - gem_writel(bp, HS_MAC_CONFIG, - GEM_BFINS(HS_MAC_SPEED, HS_SPEED_5000M, - gem_readl(bp, HS_MAC_CONFIG))); - - spin_unlock_irqrestore(&bp->lock, flags); - - if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC)) { - macb_set_tx_clk(bp, speed); - gem_shuffle_tx_rings(bp); - } - - /* Enable Rx and Tx; Enable PTP unicast */ - ctrl = macb_readl(bp, NCR); - if (gem_has_ptp(bp)) - ctrl |= MACB_BIT(PTPUNI); - - macb_writel(bp, NCR, ctrl | MACB_BIT(RE) | MACB_BIT(TE)); - - if (bp->use_ncsi) { - /* Start the NCSI device */ - err = ncsi_start_dev(bp->ndev); - if (err) { - netdev_err(bp->dev, "Ncsi start dev failed (error %d)\n", err); - return; - } - } - - netif_tx_wake_all_queues(ndev); -} - -static struct phylink_pcs *macb_mac_select_pcs(struct phylink_config *config, - phy_interface_t interface) -{ - struct net_device *ndev = to_net_dev(config->dev); - struct macb *bp = netdev_priv(ndev); - - if (interface == PHY_INTERFACE_MODE_10GBASER || - interface == PHY_INTERFACE_MODE_5GBASER || - interface == PHY_INTERFACE_MODE_USXGMII) { - return &bp->phylink_usx_pcs; - } else if (interface == PHY_INTERFACE_MODE_SGMII || - interface == PHY_INTERFACE_MODE_2500BASEX) { - return &bp->phylink_sgmii_pcs; - } else { - return NULL; - } -} - -static const struct phylink_mac_ops macb_phylink_ops = { - .mac_select_pcs = macb_mac_select_pcs, - .mac_config = macb_mac_config, - .mac_link_down = macb_mac_link_down, - .mac_link_up = macb_mac_link_up, -}; - -static bool macb_phy_handle_exists(struct device_node *dn) -{ - dn = of_parse_phandle(dn, "phy-handle", 0); - of_node_put(dn); - return dn != NULL; -} - -static int macb_phylink_connect(struct macb *bp) -{ - struct device_node *dn = bp->pdev->dev.of_node; - struct net_device *dev = bp->dev; - struct phy_device *phydev; - int ret; - - if (dn) - ret = phylink_of_phy_connect(bp->phylink, dn, 0); - - if (!dn || (ret && !macb_phy_handle_exists(dn))) { - phydev = phy_find_first(bp->mii_bus); - if (!phydev) { - netdev_err(dev, "no PHY found\n"); - return -ENXIO; - } - - /* attach the mac to the phy */ - ret = phylink_connect_phy(bp->phylink, phydev); - } - - if (ret) { - netdev_err(dev, "Could not attach PHY (%d)\n", ret); - return ret; - } - - phylink_start(bp->phylink); - - return 0; -} - -static void macb_get_pcs_fixed_state(struct phylink_config *config, - struct phylink_link_state *state) -{ - struct net_device *ndev = to_net_dev(config->dev); - struct macb *bp = netdev_priv(ndev); - - state->link = (macb_readl(bp, NSR) & MACB_BIT(NSR_LINK)) != 0; -} - -static void macb_get_usx_pcs_fixed_state(struct phylink_config *config, - struct phylink_link_state *state) -{ - u32 val; - struct net_device *ndev = to_net_dev(config->dev); - struct macb *bp = netdev_priv(ndev); - - val = gem_readl(bp, USX_STATUS); - state->link = !!(val & GEM_BIT(USX_BLOCK_LOCK)); - val = gem_readl(bp, NCFGR); - if (val & GEM_BIT(PAE)) - state->pause = MLO_PAUSE_RX; -} - -/* based on au1000_eth. c*/ -static int macb_mii_probe(struct net_device *dev) -{ - struct macb *bp = netdev_priv(dev); - - bp->phylink_sgmii_pcs.ops = &macb_phylink_pcs_ops; - bp->phylink_sgmii_pcs.neg_mode = true; - bp->phylink_usx_pcs.ops = &macb_phylink_usx_pcs_ops; - bp->phylink_usx_pcs.neg_mode = true; - - bp->phylink_config.dev = &dev->dev; - bp->phylink_config.type = PHYLINK_NETDEV; - bp->phylink_config.mac_managed_pm = true; - - if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII || - bp->phy_interface == PHY_INTERFACE_MODE_2500BASEX) { - bp->phylink_config.poll_fixed_state = true; - bp->phylink_config.get_fixed_state = macb_get_pcs_fixed_state; - } else if (bp->phy_interface == PHY_INTERFACE_MODE_USXGMII) { - bp->phylink_config.poll_fixed_state = true; - bp->phylink_config.get_fixed_state = macb_get_usx_pcs_fixed_state; - } - - bp->phylink_config.mac_capabilities = MAC_ASYM_PAUSE | - MAC_10 | MAC_100; - - __set_bit(PHY_INTERFACE_MODE_MII, - bp->phylink_config.supported_interfaces); - __set_bit(PHY_INTERFACE_MODE_RMII, - bp->phylink_config.supported_interfaces); - - /* Determine what modes are supported */ - if (macb_is_gem(bp) && (bp->caps & MACB_CAPS_GIGABIT_MODE_AVAILABLE)) { - bp->phylink_config.mac_capabilities |= MAC_1000FD; - if (!(bp->caps & MACB_CAPS_NO_GIGABIT_HALF)) - bp->phylink_config.mac_capabilities |= MAC_1000HD; - - __set_bit(PHY_INTERFACE_MODE_GMII, - bp->phylink_config.supported_interfaces); - phy_interface_set_rgmii(bp->phylink_config.supported_interfaces); - - if (bp->caps & MACB_CAPS_PCS) - __set_bit(PHY_INTERFACE_MODE_SGMII, - bp->phylink_config.supported_interfaces); - - if (bp->caps & MACB_CAPS_HIGH_SPEED) { - __set_bit(PHY_INTERFACE_MODE_10GBASER, - bp->phylink_config.supported_interfaces); - __set_bit(PHY_INTERFACE_MODE_USXGMII, - bp->phylink_config.supported_interfaces); - bp->phylink_config.mac_capabilities |= MAC_10000FD; - } - } - - bp->phylink = phylink_create(&bp->phylink_config, bp->pdev->dev.fwnode, - bp->phy_interface, &macb_phylink_ops); - if (IS_ERR(bp->phylink)) { - netdev_err(dev, "Could not create a phylink instance (%ld)\n", - PTR_ERR(bp->phylink)); - return PTR_ERR(bp->phylink); - } - - return 0; -} - -static int macb_mdiobus_register(struct macb *bp) -{ - struct device_node *child, *np = bp->pdev->dev.of_node; - - /* If we have a child named mdio, probe it instead of looking for PHYs - * directly under the MAC node - */ - child = of_get_child_by_name(np, "mdio"); - if (child) { - int ret = of_mdiobus_register(bp->mii_bus, child); - - of_node_put(child); - return ret; - } - - /* Only create the PHY from the device tree if at least one PHY is - * described. Otherwise scan the entire MDIO bus. We do this to support - * old device tree that did not follow the best practices and did not - * describe their network PHYs. - */ - for_each_available_child_of_node(np, child) - if (of_mdiobus_child_is_phy(child)) { - /* The loop increments the child refcount, - * decrement it before returning. - */ - of_node_put(child); - - return of_mdiobus_register(bp->mii_bus, np); - } - - return mdiobus_register(bp->mii_bus); -} - -static int macb_mii_init(struct macb *bp) -{ - struct device_node *child, *np = bp->pdev->dev.of_node; - int err = -ENXIO; - - /* With fixed-link, we don't need to register the MDIO bus, - * except if we have a child named "mdio" in the device tree. - * In that case, some devices may be attached to the MACB's MDIO bus. - */ - child = of_get_child_by_name(np, "mdio"); - if (child) - of_node_put(child); - else if (of_phy_is_fixed_link(np)) - return macb_mii_probe(bp->dev); - - /* Enable management port */ - macb_writel(bp, NCR, MACB_BIT(MPE)); - - bp->mii_bus = mdiobus_alloc(); - if (!bp->mii_bus) { - err = -ENOMEM; - goto err_out; - } - - bp->mii_bus->name = "MACB_mii_bus"; - bp->mii_bus->read = &macb_mdio_read_c22; - bp->mii_bus->write = &macb_mdio_write_c22; - bp->mii_bus->read_c45 = &macb_mdio_read_c45; - bp->mii_bus->write_c45 = &macb_mdio_write_c45; - snprintf(bp->mii_bus->id, MII_BUS_ID_SIZE, "%s-%x", - bp->pdev->name, bp->pdev->id); - bp->mii_bus->priv = bp; - bp->mii_bus->parent = &bp->pdev->dev; - - dev_set_drvdata(&bp->dev->dev, bp->mii_bus); - - err = macb_mdiobus_register(bp); - if (err) - goto err_out_free_mdiobus; - - err = macb_mii_probe(bp->dev); - if (err) - goto err_out_unregister_bus; - - return 0; - -err_out_unregister_bus: - mdiobus_unregister(bp->mii_bus); -err_out_free_mdiobus: - mdiobus_free(bp->mii_bus); -err_out: - return err; -} - -static void macb_update_stats(struct macb *bp) -{ - u32 *p = &bp->hw_stats.macb.rx_pause_frames; - u32 *end = &bp->hw_stats.macb.tx_pause_frames + 1; - int offset = MACB_PFR; - - WARN_ON((unsigned long)(end - p - 1) != (MACB_TPF - MACB_PFR) / 4); - - for (; p < end; p++, offset += 4) - *p += bp->macb_reg_readl(bp, offset); -} - -static int macb_halt_tx(struct macb *bp) -{ - u32 status; - - macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(THALT)); - - /* Poll TSR until TGO is cleared or timeout. */ - return read_poll_timeout_atomic(macb_readl, status, - !(status & MACB_BIT(TGO)), - 250, MACB_HALT_TIMEOUT, false, - bp, TSR); -} - -static void macb_tx_unmap(struct macb *bp, struct macb_tx_skb *tx_skb, int budget) -{ - if (tx_skb->mapping) { - if (tx_skb->mapped_as_page) - dma_unmap_page(&bp->pdev->dev, tx_skb->mapping, - tx_skb->size, DMA_TO_DEVICE); - else - dma_unmap_single(&bp->pdev->dev, tx_skb->mapping, - tx_skb->size, DMA_TO_DEVICE); - tx_skb->mapping = 0; - } - - if (tx_skb->skb) { - dev_consume_skb_any(tx_skb->skb); - tx_skb->skb = NULL; - } -} - -static void macb_set_addr(struct macb *bp, struct macb_dma_desc *desc, dma_addr_t addr) -{ -#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT - struct macb_dma_desc_64 *desc_64; - - if (bp->hw_dma_cap & HW_DMA_CAP_64B) { - desc_64 = macb_64b_desc(bp, desc); - desc_64->addrh = upper_32_bits(addr); - /* The low bits of RX address contain the RX_USED bit, clearing - * of which allows packet RX. Make sure the high bits are also - * visible to HW at that point. - */ - dma_wmb(); - } -#endif - desc->addr = lower_32_bits(addr); -} - -static dma_addr_t macb_get_addr(struct macb *bp, struct macb_dma_desc *desc) -{ - dma_addr_t addr = 0; -#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT - struct macb_dma_desc_64 *desc_64; - - if (bp->hw_dma_cap & HW_DMA_CAP_64B) { - desc_64 = macb_64b_desc(bp, desc); - addr = ((u64)(desc_64->addrh) << 32); - } -#endif - addr |= MACB_BF(RX_WADDR, MACB_BFEXT(RX_WADDR, desc->addr)); -#ifdef CONFIG_MACB_USE_HWSTAMP - if (bp->hw_dma_cap & HW_DMA_CAP_PTP) - addr &= ~GEM_BIT(DMA_RXVALID); -#endif - return addr; -} - -static void macb_tx_error_task(struct work_struct *work) -{ - struct macb_queue *queue = container_of(work, struct macb_queue, - tx_error_task); - bool halt_timeout = false; - struct macb *bp = queue->bp; - struct macb_tx_skb *tx_skb; - struct macb_dma_desc *desc; - struct sk_buff *skb; - unsigned int tail; - unsigned long flags; - - netdev_vdbg(bp->dev, "macb_tx_error_task: q = %u, t = %u, h = %u\n", - (unsigned int)(queue - bp->queues), - queue->tx_tail, queue->tx_head); - - /* Prevent the queue NAPI TX poll from running, as it calls - * macb_tx_complete(), which in turn may call netif_wake_subqueue(). - * As explained below, we have to halt the transmission before updating - * TBQP registers so we call netif_tx_stop_all_queues() to notify the - * network engine about the macb/gem being halted. - */ - napi_disable(&queue->napi_tx); - spin_lock_irqsave(&bp->lock, flags); - - /* Make sure nobody is trying to queue up new packets */ - netif_tx_stop_all_queues(bp->dev); - - /* Stop transmission now - * (in case we have just queued new packets) - * macb/gem must be halted to write TBQP register - */ - if (macb_halt_tx(bp)) { - netdev_err(bp->dev, "BUG: halt tx timed out\n"); - macb_writel(bp, NCR, macb_readl(bp, NCR) & (~MACB_BIT(TE))); - halt_timeout = true; - } - - /* Treat frames in TX queue including the ones that caused the error. - * Free transmit buffers in upper layer. - */ - for (tail = queue->tx_tail; tail != queue->tx_head; tail++) { - u32 ctrl; - - desc = macb_tx_desc(queue, tail); - ctrl = desc->ctrl; - tx_skb = macb_tx_skb(queue, tail); - skb = tx_skb->skb; - - if (ctrl & MACB_BIT(TX_USED)) { - /* skb is set for the last buffer of the frame */ - while (!skb) { - macb_tx_unmap(bp, tx_skb, 0); - tail++; - tx_skb = macb_tx_skb(queue, tail); - skb = tx_skb->skb; - } - - /* ctrl still refers to the first buffer descriptor - * since it's the only one written back by the hardware - */ - if (!(ctrl & MACB_BIT(TX_BUF_EXHAUSTED))) { - netdev_vdbg(bp->dev, "txerr skb %u (data %p) TX complete\n", - macb_tx_ring_wrap(bp, tail), - skb->data); - bp->dev->stats.tx_packets++; - queue->stats.tx_packets++; - bp->dev->stats.tx_bytes += skb->len; - queue->stats.tx_bytes += skb->len; - } - } else { - /* "Buffers exhausted mid-frame" errors may only happen - * if the driver is buggy, so complain loudly about - * those. Statistics are updated by hardware. - */ - if (ctrl & MACB_BIT(TX_BUF_EXHAUSTED)) - netdev_err(bp->dev, - "BUG: TX buffers exhausted mid-frame\n"); - - desc->ctrl = ctrl | MACB_BIT(TX_USED); - } - - macb_tx_unmap(bp, tx_skb, 0); - } - - /* Set end of TX queue */ - desc = macb_tx_desc(queue, 0); - macb_set_addr(bp, desc, 0); - desc->ctrl = MACB_BIT(TX_USED); - - /* Make descriptor updates visible to hardware */ - wmb(); - - /* Reinitialize the TX desc queue */ - queue_writel(queue, TBQP, lower_32_bits(queue->tx_ring_dma)); -#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT - if (bp->hw_dma_cap & HW_DMA_CAP_64B) - queue_writel(queue, TBQPH, upper_32_bits(queue->tx_ring_dma)); -#endif - /* Make TX ring reflect state of hardware */ - queue->tx_head = 0; - queue->tx_tail = 0; - - /* Housework before enabling TX IRQ */ - macb_writel(bp, TSR, macb_readl(bp, TSR)); - queue_writel(queue, IER, MACB_TX_INT_FLAGS); - - if (halt_timeout) - macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TE)); - - /* Now we are ready to start transmission again */ - netif_tx_start_all_queues(bp->dev); - macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART)); - - spin_unlock_irqrestore(&bp->lock, flags); - napi_enable(&queue->napi_tx); -} - -static bool ptp_one_step_sync(struct sk_buff *skb) -{ - struct ptp_header *hdr; - unsigned int ptp_class; - u8 msgtype; - - /* No need to parse packet if PTP TS is not involved */ - if (likely(!(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP))) - goto not_oss; - - /* Identify and return whether PTP one step sync is being processed */ - ptp_class = ptp_classify_raw(skb); - if (ptp_class == PTP_CLASS_NONE) - goto not_oss; - - hdr = ptp_parse_header(skb, ptp_class); - if (!hdr) - goto not_oss; - - if (hdr->flag_field[0] & PTP_FLAG_TWOSTEP) - goto not_oss; - - msgtype = ptp_get_msgtype(hdr, ptp_class); - if (msgtype == PTP_MSGTYPE_SYNC) - return true; - -not_oss: - return false; -} - -static int macb_tx_complete(struct macb_queue *queue, int budget) -{ - struct macb *bp = queue->bp; - u16 queue_index = queue - bp->queues; - unsigned long flags; - unsigned int tail; - unsigned int head; - int packets = 0; - - spin_lock_irqsave(&queue->tx_ptr_lock, flags); - head = queue->tx_head; - for (tail = queue->tx_tail; tail != head && packets < budget; tail++) { - struct macb_tx_skb *tx_skb; - struct sk_buff *skb; - struct macb_dma_desc *desc; - u32 ctrl; - - desc = macb_tx_desc(queue, tail); - - /* Make hw descriptor updates visible to CPU */ - rmb(); - - ctrl = desc->ctrl; - - /* TX_USED bit is only set by hardware on the very first buffer - * descriptor of the transmitted frame. - */ - if (!(ctrl & MACB_BIT(TX_USED))) - break; - - /* Process all buffers of the current transmitted frame */ - for (;; tail++) { - tx_skb = macb_tx_skb(queue, tail); - skb = tx_skb->skb; - - /* First, update TX stats if needed */ - if (skb) { - if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && - !ptp_one_step_sync(skb)) - gem_ptp_do_txstamp(bp, skb, desc); - - netdev_vdbg(bp->dev, "skb %u (data %p) TX complete\n", - macb_tx_ring_wrap(bp, tail), - skb->data); - bp->dev->stats.tx_packets++; - queue->stats.tx_packets++; - bp->dev->stats.tx_bytes += skb->len; - queue->stats.tx_bytes += skb->len; - packets++; - } - - /* Now we can safely release resources */ - macb_tx_unmap(bp, tx_skb, budget); - - /* skb is set only for the last buffer of the frame. - * WARNING: at this point skb has been freed by - * macb_tx_unmap(). - */ - if (skb) - break; - } - } - - queue->tx_tail = tail; - if (__netif_subqueue_stopped(bp->dev, queue_index) && - CIRC_CNT(queue->tx_head, queue->tx_tail, - bp->tx_ring_size) <= MACB_TX_WAKEUP_THRESH(bp)) - netif_wake_subqueue(bp->dev, queue_index); - spin_unlock_irqrestore(&queue->tx_ptr_lock, flags); - - return packets; -} - -static void gem_rx_refill(struct macb_queue *queue) -{ - unsigned int entry; - struct sk_buff *skb; - dma_addr_t paddr; - struct macb *bp = queue->bp; - struct macb_dma_desc *desc; - - while (CIRC_SPACE(queue->rx_prepared_head, queue->rx_tail, - bp->rx_ring_size) > 0) { - entry = macb_rx_ring_wrap(bp, queue->rx_prepared_head); - - /* Make hw descriptor updates visible to CPU */ - rmb(); - - desc = macb_rx_desc(queue, entry); - - if (!queue->rx_skbuff[entry]) { - /* allocate sk_buff for this free entry in ring */ - skb = netdev_alloc_skb(bp->dev, bp->rx_buffer_size); - if (unlikely(!skb)) { - netdev_err(bp->dev, - "Unable to allocate sk_buff\n"); - break; - } - - /* now fill corresponding descriptor entry */ - paddr = dma_map_single(&bp->pdev->dev, skb->data, - bp->rx_buffer_size, - DMA_FROM_DEVICE); - if (dma_mapping_error(&bp->pdev->dev, paddr)) { - dev_kfree_skb(skb); - break; - } - - queue->rx_skbuff[entry] = skb; - - if (entry == bp->rx_ring_size - 1) - paddr |= MACB_BIT(RX_WRAP); - desc->ctrl = 0; - /* Setting addr clears RX_USED and allows reception, - * make sure ctrl is cleared first to avoid a race. - */ - dma_wmb(); - macb_set_addr(bp, desc, paddr); - - /* properly align Ethernet header */ - skb_reserve(skb, NET_IP_ALIGN); - } else { - desc->ctrl = 0; - dma_wmb(); - desc->addr &= ~MACB_BIT(RX_USED); - } - queue->rx_prepared_head++; - } - - /* Make descriptor updates visible to hardware */ - wmb(); - - netdev_vdbg(bp->dev, "rx ring: queue: %p, prepared head %d, tail %d\n", - queue, queue->rx_prepared_head, queue->rx_tail); -} - -/* Mark DMA descriptors from begin up to and not including end as unused */ -static void discard_partial_frame(struct macb_queue *queue, unsigned int begin, - unsigned int end) -{ - unsigned int frag; - - for (frag = begin; frag != end; frag++) { - struct macb_dma_desc *desc = macb_rx_desc(queue, frag); - - desc->addr &= ~MACB_BIT(RX_USED); - } - - /* Make descriptor updates visible to hardware */ - wmb(); - - /* When this happens, the hardware stats registers for - * whatever caused this is updated, so we don't have to record - * anything. - */ -} - -static int gem_rx(struct macb_queue *queue, struct napi_struct *napi, - int budget) -{ - struct macb *bp = queue->bp; - unsigned int len; - unsigned int entry; - struct sk_buff *skb; - struct macb_dma_desc *desc; - int count = 0; - - while (count < budget) { - u32 ctrl; - dma_addr_t addr; - bool rxused; - - entry = macb_rx_ring_wrap(bp, queue->rx_tail); - desc = macb_rx_desc(queue, entry); - - /* Make hw descriptor updates visible to CPU */ - rmb(); - - rxused = (desc->addr & MACB_BIT(RX_USED)) ? true : false; - addr = macb_get_addr(bp, desc); - - if (!rxused) - break; - - /* Ensure ctrl is at least as up-to-date as rxused */ - dma_rmb(); - - ctrl = desc->ctrl; - - queue->rx_tail++; - count++; - - if (!(ctrl & MACB_BIT(RX_SOF) && ctrl & MACB_BIT(RX_EOF))) { - netdev_err(bp->dev, - "not whole frame pointed by descriptor\n"); - bp->dev->stats.rx_dropped++; - queue->stats.rx_dropped++; - break; - } - skb = queue->rx_skbuff[entry]; - if (unlikely(!skb)) { - netdev_err(bp->dev, - "inconsistent Rx descriptor chain\n"); - bp->dev->stats.rx_dropped++; - queue->stats.rx_dropped++; - break; - } - /* now everything is ready for receiving packet */ - queue->rx_skbuff[entry] = NULL; - len = ctrl & bp->rx_frm_len_mask; - - netdev_vdbg(bp->dev, "gem_rx %u (len %u)\n", entry, len); - - skb_put(skb, len); - dma_unmap_single(&bp->pdev->dev, addr, - bp->rx_buffer_size, DMA_FROM_DEVICE); - - skb->protocol = eth_type_trans(skb, bp->dev); - skb_checksum_none_assert(skb); - if (bp->dev->features & NETIF_F_RXCSUM && - !(bp->dev->flags & IFF_PROMISC) && - GEM_BFEXT(RX_CSUM, ctrl) & GEM_RX_CSUM_CHECKED_MASK) - skb->ip_summed = CHECKSUM_UNNECESSARY; - - bp->dev->stats.rx_packets++; - queue->stats.rx_packets++; - bp->dev->stats.rx_bytes += skb->len; - queue->stats.rx_bytes += skb->len; - - gem_ptp_do_rxstamp(bp, skb, desc); - -#if defined(DEBUG) && defined(VERBOSE_DEBUG) - netdev_vdbg(bp->dev, "received skb of length %u, csum: %08x\n", - skb->len, skb->csum); - print_hex_dump(KERN_DEBUG, " mac: ", DUMP_PREFIX_ADDRESS, 16, 1, - skb_mac_header(skb), 16, true); - print_hex_dump(KERN_DEBUG, "data: ", DUMP_PREFIX_ADDRESS, 16, 1, - skb->data, 32, true); -#endif - - napi_gro_receive(napi, skb); - } - - gem_rx_refill(queue); - - return count; -} - -static int macb_rx_frame(struct macb_queue *queue, struct napi_struct *napi, - unsigned int first_frag, unsigned int last_frag) -{ - unsigned int len; - unsigned int frag; - unsigned int offset; - struct sk_buff *skb; - struct macb_dma_desc *desc; - struct macb *bp = queue->bp; - - desc = macb_rx_desc(queue, last_frag); - len = desc->ctrl & bp->rx_frm_len_mask; - - netdev_vdbg(bp->dev, "macb_rx_frame frags %u - %u (len %u)\n", - macb_rx_ring_wrap(bp, first_frag), - macb_rx_ring_wrap(bp, last_frag), len); - - /* The ethernet header starts NET_IP_ALIGN bytes into the - * first buffer. Since the header is 14 bytes, this makes the - * payload word-aligned. - * - * Instead of calling skb_reserve(NET_IP_ALIGN), we just copy - * the two padding bytes into the skb so that we avoid hitting - * the slowpath in memcpy(), and pull them off afterwards. - */ - skb = netdev_alloc_skb(bp->dev, len + NET_IP_ALIGN); - if (!skb) { - bp->dev->stats.rx_dropped++; - for (frag = first_frag; ; frag++) { - desc = macb_rx_desc(queue, frag); - desc->addr &= ~MACB_BIT(RX_USED); - if (frag == last_frag) - break; - } - - /* Make descriptor updates visible to hardware */ - wmb(); - - return 1; - } - - offset = 0; - len += NET_IP_ALIGN; - skb_checksum_none_assert(skb); - skb_put(skb, len); - - for (frag = first_frag; ; frag++) { - unsigned int frag_len = bp->rx_buffer_size; - - if (offset + frag_len > len) { - if (unlikely(frag != last_frag)) { - dev_kfree_skb_any(skb); - return -1; - } - frag_len = len - offset; - } - skb_copy_to_linear_data_offset(skb, offset, - macb_rx_buffer(queue, frag), - frag_len); - offset += bp->rx_buffer_size; - desc = macb_rx_desc(queue, frag); - desc->addr &= ~MACB_BIT(RX_USED); - - if (frag == last_frag) - break; - } - - /* Make descriptor updates visible to hardware */ - wmb(); - - __skb_pull(skb, NET_IP_ALIGN); - skb->protocol = eth_type_trans(skb, bp->dev); - - bp->dev->stats.rx_packets++; - bp->dev->stats.rx_bytes += skb->len; - netdev_vdbg(bp->dev, "received skb of length %u, csum: %08x\n", - skb->len, skb->csum); - napi_gro_receive(napi, skb); - - return 0; -} - -static inline void macb_init_rx_ring(struct macb_queue *queue) -{ - struct macb *bp = queue->bp; - dma_addr_t addr; - struct macb_dma_desc *desc = NULL; - int i; - - addr = queue->rx_buffers_dma; - for (i = 0; i < bp->rx_ring_size; i++) { - desc = macb_rx_desc(queue, i); - macb_set_addr(bp, desc, addr); - desc->ctrl = 0; - addr += bp->rx_buffer_size; - } - desc->addr |= MACB_BIT(RX_WRAP); - queue->rx_tail = 0; -} - -static int macb_rx(struct macb_queue *queue, struct napi_struct *napi, - int budget) -{ - struct macb *bp = queue->bp; - bool reset_rx_queue = false; - int received = 0; - unsigned int tail; - int first_frag = -1; - - for (tail = queue->rx_tail; budget > 0; tail++) { - struct macb_dma_desc *desc = macb_rx_desc(queue, tail); - u32 ctrl; - - /* Make hw descriptor updates visible to CPU */ - rmb(); - - if (!(desc->addr & MACB_BIT(RX_USED))) - break; - - /* Ensure ctrl is at least as up-to-date as addr */ - dma_rmb(); - - ctrl = desc->ctrl; - - if (ctrl & MACB_BIT(RX_SOF)) { - if (first_frag != -1) - discard_partial_frame(queue, first_frag, tail); - first_frag = tail; - } - - if (ctrl & MACB_BIT(RX_EOF)) { - int dropped; - - if (unlikely(first_frag == -1)) { - reset_rx_queue = true; - continue; - } - - dropped = macb_rx_frame(queue, napi, first_frag, tail); - first_frag = -1; - if (unlikely(dropped < 0)) { - reset_rx_queue = true; - continue; - } - if (!dropped) { - received++; - budget--; - } - } - } - - if (unlikely(reset_rx_queue)) { - unsigned long flags; - u32 ctrl; - - netdev_err(bp->dev, "RX queue corruption: reset it\n"); - - spin_lock_irqsave(&bp->lock, flags); - - ctrl = macb_readl(bp, NCR); - macb_writel(bp, NCR, ctrl & ~MACB_BIT(RE)); - - macb_init_rx_ring(queue); - queue_writel(queue, RBQP, queue->rx_ring_dma); - - macb_writel(bp, NCR, ctrl | MACB_BIT(RE)); - - spin_unlock_irqrestore(&bp->lock, flags); - return received; - } - - if (first_frag != -1) - queue->rx_tail = first_frag; - else - queue->rx_tail = tail; - - return received; -} - -static bool macb_rx_pending(struct macb_queue *queue) -{ - struct macb *bp = queue->bp; - unsigned int entry; - struct macb_dma_desc *desc; - - entry = macb_rx_ring_wrap(bp, queue->rx_tail); - desc = macb_rx_desc(queue, entry); - - /* Make hw descriptor updates visible to CPU */ - rmb(); - - return (desc->addr & MACB_BIT(RX_USED)) != 0; -} - -static int macb_rx_poll(struct napi_struct *napi, int budget) -{ - struct macb_queue *queue = container_of(napi, struct macb_queue, napi_rx); - struct macb *bp = queue->bp; - int work_done; - - work_done = bp->macbgem_ops.mog_rx(queue, napi, budget); - - netdev_vdbg(bp->dev, "RX poll: queue = %u, work_done = %d, budget = %d\n", - (unsigned int)(queue - bp->queues), work_done, budget); - - if (work_done < budget && napi_complete_done(napi, work_done)) { - queue_writel(queue, IER, bp->rx_intr_mask); - - /* Packet completions only seem to propagate to raise - * interrupts when interrupts are enabled at the time, so if - * packets were received while interrupts were disabled, - * they will not cause another interrupt to be generated when - * interrupts are re-enabled. - * Check for this case here to avoid losing a wakeup. This can - * potentially race with the interrupt handler doing the same - * actions if an interrupt is raised just after enabling them, - * but this should be harmless. - */ - if (macb_rx_pending(queue)) { - queue_writel(queue, IDR, bp->rx_intr_mask); - if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) - queue_writel(queue, ISR, MACB_BIT(RCOMP)); - netdev_vdbg(bp->dev, "poll: packets pending, reschedule\n"); - napi_schedule(napi); - } - } - - /* TODO: Handle errors */ - - return work_done; -} - -static void macb_tx_restart(struct macb_queue *queue) -{ - struct macb *bp = queue->bp; - unsigned int head_idx, tbqp; - unsigned long flags; - - spin_lock_irqsave(&queue->tx_ptr_lock, flags); - - if (queue->tx_head == queue->tx_tail) - goto out_tx_ptr_unlock; - - tbqp = queue_readl(queue, TBQP) - lower_32_bits(queue->tx_ring_dma); - tbqp = tbqp / macb_dma_desc_get_size(bp); - tbqp = macb_adj_dma_desc_idx(bp, macb_tx_ring_wrap(bp, tbqp)); - head_idx = macb_adj_dma_desc_idx(bp, macb_tx_ring_wrap(bp, queue->tx_head)); - - if (tbqp == head_idx) - goto out_tx_ptr_unlock; - - spin_lock(&bp->lock); - macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART)); - spin_unlock(&bp->lock); - -out_tx_ptr_unlock: - spin_unlock_irqrestore(&queue->tx_ptr_lock, flags); -} - -static bool macb_tx_complete_pending(struct macb_queue *queue) -{ - bool retval = false; - unsigned long flags; - - spin_lock_irqsave(&queue->tx_ptr_lock, flags); - if (queue->tx_head != queue->tx_tail) { - /* Make hw descriptor updates visible to CPU */ - rmb(); - - if (macb_tx_desc(queue, queue->tx_tail)->ctrl & MACB_BIT(TX_USED)) - retval = true; - } - spin_unlock_irqrestore(&queue->tx_ptr_lock, flags); - return retval; -} - -static int macb_tx_poll(struct napi_struct *napi, int budget) -{ - struct macb_queue *queue = container_of(napi, struct macb_queue, napi_tx); - struct macb *bp = queue->bp; - int work_done; - - work_done = macb_tx_complete(queue, budget); - - rmb(); // ensure txubr_pending is up to date - if (queue->txubr_pending) { - queue->txubr_pending = false; - netdev_vdbg(bp->dev, "poll: tx restart\n"); - macb_tx_restart(queue); - } - - netdev_vdbg(bp->dev, "TX poll: queue = %u, work_done = %d, budget = %d\n", - (unsigned int)(queue - bp->queues), work_done, budget); - - if (work_done < budget && napi_complete_done(napi, work_done)) { - queue_writel(queue, IER, MACB_BIT(TCOMP)); - - /* Packet completions only seem to propagate to raise - * interrupts when interrupts are enabled at the time, so if - * packets were sent while interrupts were disabled, - * they will not cause another interrupt to be generated when - * interrupts are re-enabled. - * Check for this case here to avoid losing a wakeup. This can - * potentially race with the interrupt handler doing the same - * actions if an interrupt is raised just after enabling them, - * but this should be harmless. - */ - if (macb_tx_complete_pending(queue)) { - queue_writel(queue, IDR, MACB_BIT(TCOMP)); - if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) - queue_writel(queue, ISR, MACB_BIT(TCOMP)); - netdev_vdbg(bp->dev, "TX poll: packets pending, reschedule\n"); - napi_schedule(napi); - } - } - - return work_done; -} - -static void macb_hresp_error_task(struct tasklet_struct *t) -{ - struct macb *bp = from_tasklet(bp, t, hresp_err_tasklet); - struct net_device *dev = bp->dev; - struct macb_queue *queue; - unsigned int q; - u32 ctrl; - - for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { - queue_writel(queue, IDR, bp->rx_intr_mask | - MACB_TX_INT_FLAGS | - MACB_BIT(HRESP)); - } - ctrl = macb_readl(bp, NCR); - ctrl &= ~(MACB_BIT(RE) | MACB_BIT(TE)); - macb_writel(bp, NCR, ctrl); - - netif_tx_stop_all_queues(dev); - netif_carrier_off(dev); - - bp->macbgem_ops.mog_init_rings(bp); - - /* Initialize TX and RX buffers */ - macb_init_buffers(bp); - - /* Enable interrupts */ - for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) - queue_writel(queue, IER, - bp->rx_intr_mask | - MACB_TX_INT_FLAGS | - MACB_BIT(HRESP)); - - ctrl |= MACB_BIT(RE) | MACB_BIT(TE); - macb_writel(bp, NCR, ctrl); - - netif_carrier_on(dev); - netif_tx_start_all_queues(dev); -} - -static irqreturn_t macb_wol_interrupt(int irq, void *dev_id) -{ - struct macb_queue *queue = dev_id; - struct macb *bp = queue->bp; - u32 status; - - status = queue_readl(queue, ISR); - - if (unlikely(!status)) - return IRQ_NONE; - - spin_lock(&bp->lock); - - if (status & MACB_BIT(WOL)) { - queue_writel(queue, IDR, MACB_BIT(WOL)); - macb_writel(bp, WOL, 0); - netdev_vdbg(bp->dev, "MACB WoL: queue = %u, isr = 0x%08lx\n", - (unsigned int)(queue - bp->queues), - (unsigned long)status); - if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) - queue_writel(queue, ISR, MACB_BIT(WOL)); - pm_wakeup_event(&bp->pdev->dev, 0); - } - - spin_unlock(&bp->lock); - - return IRQ_HANDLED; -} - -static irqreturn_t gem_wol_interrupt(int irq, void *dev_id) -{ - struct macb_queue *queue = dev_id; - struct macb *bp = queue->bp; - u32 status; - - status = queue_readl(queue, ISR); - - if (unlikely(!status)) - return IRQ_NONE; - - spin_lock(&bp->lock); - - if (status & GEM_BIT(WOL)) { - queue_writel(queue, IDR, GEM_BIT(WOL)); - gem_writel(bp, WOL, 0); - netdev_vdbg(bp->dev, "GEM WoL: queue = %u, isr = 0x%08lx\n", - (unsigned int)(queue - bp->queues), - (unsigned long)status); - if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) - queue_writel(queue, ISR, GEM_BIT(WOL)); - pm_wakeup_event(&bp->pdev->dev, 0); - } - - spin_unlock(&bp->lock); - - return IRQ_HANDLED; -} - -static irqreturn_t macb_interrupt(int irq, void *dev_id) -{ - struct macb_queue *queue = dev_id; - struct macb *bp = queue->bp; - struct net_device *dev = bp->dev; - u32 status, ctrl; - - status = queue_readl(queue, ISR); - - if (unlikely(!status)) - return IRQ_NONE; - - spin_lock(&bp->lock); - - while (status) { - /* close possible race with dev_close */ - if (unlikely(!netif_running(dev))) { - queue_writel(queue, IDR, -1); - if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) - queue_writel(queue, ISR, -1); - break; - } - - netdev_vdbg(bp->dev, "queue = %u, isr = 0x%08lx\n", - (unsigned int)(queue - bp->queues), - (unsigned long)status); - - if (status & bp->rx_intr_mask) { - /* There's no point taking any more interrupts - * until we have processed the buffers. The - * scheduling call may fail if the poll routine - * is already scheduled, so disable interrupts - * now. - */ - queue_writel(queue, IDR, bp->rx_intr_mask); - if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) - queue_writel(queue, ISR, MACB_BIT(RCOMP)); - - if (napi_schedule_prep(&queue->napi_rx)) { - netdev_vdbg(bp->dev, "scheduling RX softirq\n"); - __napi_schedule(&queue->napi_rx); - } - } - - if (status & (MACB_BIT(TCOMP) | - MACB_BIT(TXUBR))) { - queue_writel(queue, IDR, MACB_BIT(TCOMP)); - if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) - queue_writel(queue, ISR, MACB_BIT(TCOMP) | - MACB_BIT(TXUBR)); - - if (status & MACB_BIT(TXUBR)) { - queue->txubr_pending = true; - wmb(); // ensure softirq can see update - } - - if (napi_schedule_prep(&queue->napi_tx)) { - netdev_vdbg(bp->dev, "scheduling TX softirq\n"); - __napi_schedule(&queue->napi_tx); - } - } - - if (unlikely(status & (MACB_TX_ERR_FLAGS))) { - queue_writel(queue, IDR, MACB_TX_INT_FLAGS); - schedule_work(&queue->tx_error_task); - - if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) - queue_writel(queue, ISR, MACB_TX_ERR_FLAGS); - - break; - } - - /* Link change detection isn't possible with RMII, so we'll - * add that if/when we get our hands on a full-blown MII PHY. - */ - - /* There is a hardware issue under heavy load where DMA can - * stop, this causes endless "used buffer descriptor read" - * interrupts but it can be cleared by re-enabling RX. See - * the at91rm9200 manual, section 41.3.1 or the Zynq manual - * section 16.7.4 for details. RXUBR is only enabled for - * these two versions. - */ - if (status & MACB_BIT(RXUBR)) { - ctrl = macb_readl(bp, NCR); - macb_writel(bp, NCR, ctrl & ~MACB_BIT(RE)); - wmb(); - macb_writel(bp, NCR, ctrl | MACB_BIT(RE)); - - if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) - queue_writel(queue, ISR, MACB_BIT(RXUBR)); - } - - if (status & MACB_BIT(ISR_ROVR)) { - /* We missed at least one packet */ - spin_lock(&bp->stats_lock); - if (macb_is_gem(bp)) - bp->hw_stats.gem.rx_overruns++; - else - bp->hw_stats.macb.rx_overruns++; - spin_unlock(&bp->stats_lock); - - if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) - queue_writel(queue, ISR, MACB_BIT(ISR_ROVR)); - } - - if (status & MACB_BIT(HRESP)) { - tasklet_schedule(&bp->hresp_err_tasklet); - netdev_err(dev, "DMA bus error: HRESP not OK\n"); - - if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) - queue_writel(queue, ISR, MACB_BIT(HRESP)); - } - status = queue_readl(queue, ISR); - } - - spin_unlock(&bp->lock); - - return IRQ_HANDLED; -} - -#ifdef CONFIG_NET_POLL_CONTROLLER -/* Polling receive - used by netconsole and other diagnostic tools - * to allow network i/o with interrupts disabled. - */ -static void macb_poll_controller(struct net_device *dev) -{ - struct macb *bp = netdev_priv(dev); - struct macb_queue *queue; - unsigned long flags; - unsigned int q; - - local_irq_save(flags); - for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) - macb_interrupt(dev->irq, queue); - local_irq_restore(flags); -} -#endif - -static unsigned int macb_tx_map(struct macb *bp, - struct macb_queue *queue, - struct sk_buff *skb, - unsigned int hdrlen) -{ - dma_addr_t mapping; - unsigned int len, entry, i, tx_head = queue->tx_head; - struct macb_tx_skb *tx_skb = NULL; - struct macb_dma_desc *desc; - unsigned int offset, size, count = 0; - unsigned int f, nr_frags = skb_shinfo(skb)->nr_frags; - unsigned int eof = 1, mss_mfs = 0; - u32 ctrl, lso_ctrl = 0, seq_ctrl = 0; - - /* LSO */ - if (skb_shinfo(skb)->gso_size != 0) { - if (ip_hdr(skb)->protocol == IPPROTO_UDP) - /* UDP - UFO */ - lso_ctrl = MACB_LSO_UFO_ENABLE; - else - /* TCP - TSO */ - lso_ctrl = MACB_LSO_TSO_ENABLE; - } - - /* First, map non-paged data */ - len = skb_headlen(skb); - - /* first buffer length */ - size = hdrlen; - - offset = 0; - while (len) { - entry = macb_tx_ring_wrap(bp, tx_head); - tx_skb = &queue->tx_skb[entry]; - - mapping = dma_map_single(&bp->pdev->dev, - skb->data + offset, - size, DMA_TO_DEVICE); - if (dma_mapping_error(&bp->pdev->dev, mapping)) - goto dma_error; - - /* Save info to properly release resources */ - tx_skb->skb = NULL; - tx_skb->mapping = mapping; - tx_skb->size = size; - tx_skb->mapped_as_page = false; - - len -= size; - offset += size; - count++; - tx_head++; - - size = min(len, bp->max_tx_length); - } - - /* Then, map paged data from fragments */ - for (f = 0; f < nr_frags; f++) { - const skb_frag_t *frag = &skb_shinfo(skb)->frags[f]; - - len = skb_frag_size(frag); - offset = 0; - while (len) { - size = min(len, bp->max_tx_length); - entry = macb_tx_ring_wrap(bp, tx_head); - tx_skb = &queue->tx_skb[entry]; - - mapping = skb_frag_dma_map(&bp->pdev->dev, frag, - offset, size, DMA_TO_DEVICE); - if (dma_mapping_error(&bp->pdev->dev, mapping)) - goto dma_error; - - /* Save info to properly release resources */ - tx_skb->skb = NULL; - tx_skb->mapping = mapping; - tx_skb->size = size; - tx_skb->mapped_as_page = true; - - len -= size; - offset += size; - count++; - tx_head++; - } - } - - /* Should never happen */ - if (unlikely(!tx_skb)) { - netdev_err(bp->dev, "BUG! empty skb!\n"); - return 0; - } - - /* This is the last buffer of the frame: save socket buffer */ - tx_skb->skb = skb; - - /* Update TX ring: update buffer descriptors in reverse order - * to avoid race condition - */ - - /* Set 'TX_USED' bit in buffer descriptor at tx_head position - * to set the end of TX queue - */ - i = tx_head; - entry = macb_tx_ring_wrap(bp, i); - ctrl = MACB_BIT(TX_USED); - desc = macb_tx_desc(queue, entry); - desc->ctrl = ctrl; - - if (lso_ctrl) { - if (lso_ctrl == MACB_LSO_UFO_ENABLE) - /* include header and FCS in value given to h/w */ - mss_mfs = skb_shinfo(skb)->gso_size + - skb_transport_offset(skb) + - ETH_FCS_LEN; - else /* TSO */ { - mss_mfs = skb_shinfo(skb)->gso_size; - /* TCP Sequence Number Source Select - * can be set only for TSO - */ - seq_ctrl = 0; - } - } - - do { - i--; - entry = macb_tx_ring_wrap(bp, i); - tx_skb = &queue->tx_skb[entry]; - desc = macb_tx_desc(queue, entry); - - ctrl = (u32)tx_skb->size; - if (eof) { - ctrl |= MACB_BIT(TX_LAST); - eof = 0; - } - if (unlikely(entry == (bp->tx_ring_size - 1))) - ctrl |= MACB_BIT(TX_WRAP); - - /* First descriptor is header descriptor */ - if (i == queue->tx_head) { - ctrl |= MACB_BF(TX_LSO, lso_ctrl); - ctrl |= MACB_BF(TX_TCP_SEQ_SRC, seq_ctrl); - if ((bp->dev->features & NETIF_F_HW_CSUM) && - skb->ip_summed != CHECKSUM_PARTIAL && !lso_ctrl && - !ptp_one_step_sync(skb)) - ctrl |= MACB_BIT(TX_NOCRC); - } else - /* Only set MSS/MFS on payload descriptors - * (second or later descriptor) - */ - ctrl |= MACB_BF(MSS_MFS, mss_mfs); - - /* Set TX buffer descriptor */ - macb_set_addr(bp, desc, tx_skb->mapping); - /* desc->addr must be visible to hardware before clearing - * 'TX_USED' bit in desc->ctrl. - */ - wmb(); - desc->ctrl = ctrl; - } while (i != queue->tx_head); - - queue->tx_head = tx_head; - - return count; - -dma_error: - netdev_err(bp->dev, "TX DMA map failed\n"); - - for (i = queue->tx_head; i != tx_head; i++) { - tx_skb = macb_tx_skb(queue, i); - - macb_tx_unmap(bp, tx_skb, 0); - } - - return 0; -} - -static netdev_features_t macb_features_check(struct sk_buff *skb, - struct net_device *dev, - netdev_features_t features) -{ - unsigned int nr_frags, f; - unsigned int hdrlen; - - /* Validate LSO compatibility */ - - /* there is only one buffer or protocol is not UDP */ - if (!skb_is_nonlinear(skb) || (ip_hdr(skb)->protocol != IPPROTO_UDP)) - return features; - - /* length of header */ - hdrlen = skb_transport_offset(skb); - - /* For UFO only: - * When software supplies two or more payload buffers all payload buffers - * apart from the last must be a multiple of 8 bytes in size. - */ - if (!IS_ALIGNED(skb_headlen(skb) - hdrlen, MACB_TX_LEN_ALIGN)) - return features & ~MACB_NETIF_LSO; - - nr_frags = skb_shinfo(skb)->nr_frags; - /* No need to check last fragment */ - nr_frags--; - for (f = 0; f < nr_frags; f++) { - const skb_frag_t *frag = &skb_shinfo(skb)->frags[f]; - - if (!IS_ALIGNED(skb_frag_size(frag), MACB_TX_LEN_ALIGN)) - return features & ~MACB_NETIF_LSO; - } - return features; -} - -static inline int macb_clear_csum(struct sk_buff *skb) -{ - /* no change for packets without checksum offloading */ - if (skb->ip_summed != CHECKSUM_PARTIAL) - return 0; - - /* make sure we can modify the header */ - if (unlikely(skb_cow_head(skb, 0))) - return -1; - - /* initialize checksum field - * This is required - at least for Zynq, which otherwise calculates - * wrong UDP header checksums for UDP packets with UDP data len <=2 - */ - *(__sum16 *)(skb_checksum_start(skb) + skb->csum_offset) = 0; - return 0; -} - -static int macb_pad_and_fcs(struct sk_buff **skb, struct net_device *ndev) -{ - bool cloned = skb_cloned(*skb) || skb_header_cloned(*skb) || - skb_is_nonlinear(*skb); - int padlen = ETH_ZLEN - (*skb)->len; - int tailroom = skb_tailroom(*skb); - struct sk_buff *nskb; - u32 fcs; - - if (!(ndev->features & NETIF_F_HW_CSUM) || - !((*skb)->ip_summed != CHECKSUM_PARTIAL) || - skb_shinfo(*skb)->gso_size || ptp_one_step_sync(*skb)) - return 0; - - if (padlen <= 0) { - /* FCS could be appeded to tailroom. */ - if (tailroom >= ETH_FCS_LEN) - goto add_fcs; - /* No room for FCS, need to reallocate skb. */ - else - padlen = ETH_FCS_LEN; - } else { - /* Add room for FCS. */ - padlen += ETH_FCS_LEN; - } - - if (cloned || tailroom < padlen) { - nskb = skb_copy_expand(*skb, 0, padlen, GFP_ATOMIC); - if (!nskb) - return -ENOMEM; - - dev_consume_skb_any(*skb); - *skb = nskb; - } - - if (padlen > ETH_FCS_LEN) - skb_put_zero(*skb, padlen - ETH_FCS_LEN); - -add_fcs: - /* set FCS to packet */ - fcs = crc32_le(~0, (*skb)->data, (*skb)->len); - fcs = ~fcs; - - skb_put_u8(*skb, fcs & 0xff); - skb_put_u8(*skb, (fcs >> 8) & 0xff); - skb_put_u8(*skb, (fcs >> 16) & 0xff); - skb_put_u8(*skb, (fcs >> 24) & 0xff); - - return 0; -} - -static netdev_tx_t macb_start_xmit(struct sk_buff *skb, struct net_device *dev) -{ - u16 queue_index = skb_get_queue_mapping(skb); - struct macb *bp = netdev_priv(dev); - struct macb_queue *queue = &bp->queues[queue_index]; - unsigned int desc_cnt, nr_frags, frag_size, f; - unsigned int hdrlen; - unsigned long flags; - bool is_lso; - netdev_tx_t ret = NETDEV_TX_OK; - - if (macb_clear_csum(skb)) { - dev_kfree_skb_any(skb); - return ret; - } - - if (macb_pad_and_fcs(&skb, dev)) { - dev_kfree_skb_any(skb); - return ret; - } - -#ifdef CONFIG_MACB_USE_HWSTAMP - if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && - (bp->hw_dma_cap & HW_DMA_CAP_PTP)) - skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; -#endif - - is_lso = (skb_shinfo(skb)->gso_size != 0); - - if (is_lso) { - /* length of headers */ - if (ip_hdr(skb)->protocol == IPPROTO_UDP) - /* only queue eth + ip headers separately for UDP */ - hdrlen = skb_transport_offset(skb); - else - hdrlen = skb_tcp_all_headers(skb); - if (skb_headlen(skb) < hdrlen) { - netdev_err(bp->dev, "Error - LSO headers fragmented!!!\n"); - /* if this is required, would need to copy to single buffer */ - return NETDEV_TX_BUSY; - } - } else - hdrlen = min(skb_headlen(skb), bp->max_tx_length); - -#if defined(DEBUG) && defined(VERBOSE_DEBUG) - netdev_vdbg(bp->dev, - "start_xmit: queue %hu len %u head %p data %p tail %p end %p\n", - queue_index, skb->len, skb->head, skb->data, - skb_tail_pointer(skb), skb_end_pointer(skb)); - print_hex_dump(KERN_DEBUG, "data: ", DUMP_PREFIX_OFFSET, 16, 1, - skb->data, 16, true); -#endif - - /* Count how many TX buffer descriptors are needed to send this - * socket buffer: skb fragments of jumbo frames may need to be - * split into many buffer descriptors. - */ - if (is_lso && (skb_headlen(skb) > hdrlen)) - /* extra header descriptor if also payload in first buffer */ - desc_cnt = DIV_ROUND_UP((skb_headlen(skb) - hdrlen), bp->max_tx_length) + 1; - else - desc_cnt = DIV_ROUND_UP(skb_headlen(skb), bp->max_tx_length); - nr_frags = skb_shinfo(skb)->nr_frags; - for (f = 0; f < nr_frags; f++) { - frag_size = skb_frag_size(&skb_shinfo(skb)->frags[f]); - desc_cnt += DIV_ROUND_UP(frag_size, bp->max_tx_length); - } - - spin_lock_irqsave(&queue->tx_ptr_lock, flags); - - /* This is a hard error, log it. */ - if (CIRC_SPACE(queue->tx_head, queue->tx_tail, - bp->tx_ring_size) < desc_cnt) { - netif_stop_subqueue(dev, queue_index); - netdev_dbg(bp->dev, "tx_head = %u, tx_tail = %u\n", - queue->tx_head, queue->tx_tail); - ret = NETDEV_TX_BUSY; - goto unlock; - } - - /* Map socket buffer for DMA transfer */ - if (!macb_tx_map(bp, queue, skb, hdrlen)) { - dev_kfree_skb_any(skb); - goto unlock; - } - - /* Make newly initialized descriptor visible to hardware */ - wmb(); - skb_tx_timestamp(skb); - - spin_lock(&bp->lock); - macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART)); - spin_unlock(&bp->lock); - - if (CIRC_SPACE(queue->tx_head, queue->tx_tail, bp->tx_ring_size) < 1) - netif_stop_subqueue(dev, queue_index); - -unlock: - spin_unlock_irqrestore(&queue->tx_ptr_lock, flags); - - return ret; -} - -static void macb_init_rx_buffer_size(struct macb *bp, size_t size) -{ - if (!macb_is_gem(bp)) { - bp->rx_buffer_size = MACB_RX_BUFFER_SIZE; - } else { - bp->rx_buffer_size = size; - - if (bp->rx_buffer_size % RX_BUFFER_MULTIPLE) { - netdev_dbg(bp->dev, - "RX buffer must be multiple of %d bytes, expanding\n", - RX_BUFFER_MULTIPLE); - bp->rx_buffer_size = - roundup(bp->rx_buffer_size, RX_BUFFER_MULTIPLE); - } - } - - netdev_dbg(bp->dev, "mtu [%u] rx_buffer_size [%zu]\n", - bp->dev->mtu, bp->rx_buffer_size); -} - -static void gem_free_rx_buffers(struct macb *bp) -{ - struct sk_buff *skb; - struct macb_dma_desc *desc; - struct macb_queue *queue; - dma_addr_t addr; - unsigned int q; - int i; - - for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { - if (!queue->rx_skbuff) - continue; - - for (i = 0; i < bp->rx_ring_size; i++) { - skb = queue->rx_skbuff[i]; - - if (!skb) - continue; - - desc = macb_rx_desc(queue, i); - addr = macb_get_addr(bp, desc); - - dma_unmap_single(&bp->pdev->dev, addr, bp->rx_buffer_size, - DMA_FROM_DEVICE); - dev_kfree_skb_any(skb); - skb = NULL; - } - - kfree(queue->rx_skbuff); - queue->rx_skbuff = NULL; - } -} - -static void macb_free_rx_buffers(struct macb *bp) -{ - struct macb_queue *queue = &bp->queues[0]; - - if (queue->rx_buffers) { - dma_free_coherent(&bp->pdev->dev, - bp->rx_ring_size * bp->rx_buffer_size, - queue->rx_buffers, queue->rx_buffers_dma); - queue->rx_buffers = NULL; - } -} - -static void macb_free_consistent(struct macb *bp) -{ - struct macb_dma_desc *tx_ring_base = NULL; - struct macb_dma_desc *rx_ring_base = NULL; - dma_addr_t tx_ring_base_addr; - dma_addr_t rx_ring_base_addr; - struct macb_queue *queue; - unsigned int q; - int size; - - if (bp->rx_ring_tieoff) { - dma_free_coherent(&bp->pdev->dev, macb_dma_desc_get_size(bp), - bp->rx_ring_tieoff, bp->rx_ring_tieoff_dma); - bp->rx_ring_tieoff = NULL; - } - - bp->macbgem_ops.mog_free_rx_buffers(bp); - - queue = bp->queues; - if (queue->tx_ring) { - tx_ring_base = queue->tx_ring; - tx_ring_base_addr = queue->tx_ring_dma; - } - if (queue->rx_ring) { - rx_ring_base = queue->rx_ring; - rx_ring_base_addr = queue->rx_ring_dma; - } - - for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { - kfree(queue->tx_skb); - queue->tx_skb = NULL; - if (queue->tx_ring) - queue->tx_ring = NULL; - if (queue->rx_ring) - queue->rx_ring = NULL; - } - - if (tx_ring_base) { - size = bp->num_queues * (TX_RING_BYTES(bp) + - bp->tx_bd_rd_prefetch + - RING_ADDR_INTERVAL); - dma_free_coherent(&bp->pdev->dev, size, tx_ring_base, - tx_ring_base_addr); - } - if (rx_ring_base) { - size = bp->num_queues * (RX_RING_BYTES(bp) + - bp->rx_bd_rd_prefetch + - RING_ADDR_INTERVAL); - dma_free_coherent(&bp->pdev->dev, size, rx_ring_base, - rx_ring_base_addr); - } -} - -static int gem_alloc_rx_buffers(struct macb *bp) -{ - struct macb_queue *queue; - unsigned int q; - int size; - - for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { - size = bp->rx_ring_size * sizeof(struct sk_buff *); - queue->rx_skbuff = kzalloc(size, GFP_KERNEL); - if (!queue->rx_skbuff) - return -ENOMEM; - else - netdev_dbg(bp->dev, - "Allocated %d RX struct sk_buff entries at %p\n", - bp->rx_ring_size, queue->rx_skbuff); - } - return 0; -} - -static int macb_alloc_rx_buffers(struct macb *bp) -{ - struct macb_queue *queue = &bp->queues[0]; - int size; - - size = bp->rx_ring_size * bp->rx_buffer_size; - queue->rx_buffers = dma_alloc_coherent(&bp->pdev->dev, size, - &queue->rx_buffers_dma, GFP_KERNEL); - if (!queue->rx_buffers) - return -ENOMEM; - - netdev_dbg(bp->dev, - "Allocated RX buffers of %d bytes at %08lx (mapped %p)\n", - size, (unsigned long)queue->rx_buffers_dma, queue->rx_buffers); - return 0; -} - -static int macb_queue_phyaddr_check(struct macb *bp, dma_addr_t ring_base_addr, - int offset) -{ - u32 bus_addr_high; - int i; - - bus_addr_high = upper_32_bits(ring_base_addr); - for (i = 1; i < bp->num_queues; i++) { - ring_base_addr += offset; - if (bus_addr_high != upper_32_bits(ring_base_addr)) - return -1; - } - - return 0; -} - -static int macb_alloc_consistent(struct macb *bp) -{ - struct macb_dma_desc *tx_ring_base, *rx_ring_base; - dma_addr_t tx_ring_base_addr, rx_ring_base_addr; - struct macb_queue *queue; - int tx_offset, rx_offset; - int tx_size, rx_size; - unsigned int q; - int ret, i; - int size; - - tx_offset = TX_RING_BYTES(bp) + bp->tx_bd_rd_prefetch + - RING_ADDR_INTERVAL; - tx_size = bp->num_queues * tx_offset; - for (i = 0; i < MAX_RING_ADDR_ALLOC_TIMES + 1; i++) { - if (i == MAX_RING_ADDR_ALLOC_TIMES) - return -ENOMEM; - - tx_ring_base = dma_alloc_coherent(&bp->pdev->dev, tx_size, - &tx_ring_base_addr, - GFP_KERNEL); - if (!tx_ring_base) - continue; - - ret = macb_queue_phyaddr_check(bp, tx_ring_base_addr, - tx_offset); - if (ret) { - dma_free_coherent(&bp->pdev->dev, tx_size, tx_ring_base, - tx_ring_base_addr); - continue; - } else { - break; - } - } - - rx_offset = RX_RING_BYTES(bp) + bp->rx_bd_rd_prefetch + - RING_ADDR_INTERVAL; - rx_size = bp->num_queues * rx_offset; - for (i = 0; i < MAX_RING_ADDR_ALLOC_TIMES + 1; i++) { - if (i == MAX_RING_ADDR_ALLOC_TIMES) { - dma_free_coherent(&bp->pdev->dev, tx_size, tx_ring_base, - tx_ring_base_addr); - return -ENOMEM; - } - - rx_ring_base = dma_alloc_coherent(&bp->pdev->dev, rx_size, - &rx_ring_base_addr, - GFP_KERNEL); - if (!rx_ring_base) - continue; - - ret = macb_queue_phyaddr_check(bp, rx_ring_base_addr, - rx_offset); - if (ret) { - dma_free_coherent(&bp->pdev->dev, rx_size, rx_ring_base, - rx_ring_base_addr); - continue; - } else { - break; - } - } - - for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { - queue->tx_ring = (void *)tx_ring_base + q * tx_offset; - queue->tx_ring_dma = tx_ring_base_addr + q * tx_offset; - if (!queue->tx_ring) - goto out_err; - netdev_dbg(bp->dev, - "Allocated TX ring for queue %u of %d bytes at %08lx (mapped %p)\n", - q, size, (unsigned long)queue->tx_ring_dma, - queue->tx_ring); - - size = bp->tx_ring_size * sizeof(struct macb_tx_skb); - queue->tx_skb = kzalloc(size, GFP_KERNEL); - if (!queue->tx_skb) - goto out_err; - - queue->rx_ring = (void *)rx_ring_base + q * rx_offset; - queue->rx_ring_dma = rx_ring_base_addr + q * rx_offset; - if (!queue->rx_ring) - goto out_err; - netdev_dbg(bp->dev, - "Allocated RX ring of %d bytes at %08lx (mapped %p)\n", - size, (unsigned long)queue->rx_ring_dma, queue->rx_ring); - } - if (bp->macbgem_ops.mog_alloc_rx_buffers(bp)) - goto out_err; - - /* Required for tie off descriptor for PM cases */ - if (!(bp->caps & MACB_CAPS_QUEUE_DISABLE)) { - bp->rx_ring_tieoff = dma_alloc_coherent(&bp->pdev->dev, - macb_dma_desc_get_size(bp), - &bp->rx_ring_tieoff_dma, - GFP_KERNEL); - if (!bp->rx_ring_tieoff) - goto out_err; - } - - return 0; - -out_err: - macb_free_consistent(bp); - return -ENOMEM; -} - -static void macb_init_tieoff(struct macb *bp) -{ - struct macb_dma_desc *desc = bp->rx_ring_tieoff; - - if (bp->caps & MACB_CAPS_QUEUE_DISABLE) - return; - /* Setup a wrapping descriptor with no free slots - * (WRAP and USED) to tie off/disable unused RX queues. - */ - macb_set_addr(bp, desc, MACB_BIT(RX_WRAP) | MACB_BIT(RX_USED)); - desc->ctrl = 0; -} - -static void gem_init_rx_ring(struct macb_queue *queue) -{ - queue->rx_tail = 0; - queue->rx_prepared_head = 0; - - gem_rx_refill(queue); -} - -static void gem_init_rings(struct macb *bp) -{ - struct macb_queue *queue; - struct macb_dma_desc *desc = NULL; - unsigned int q; - int i; - - for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { - for (i = 0; i < bp->tx_ring_size; i++) { - desc = macb_tx_desc(queue, i); - macb_set_addr(bp, desc, 0); - desc->ctrl = MACB_BIT(TX_USED); - } - desc->ctrl |= MACB_BIT(TX_WRAP); - queue->tx_head = 0; - queue->tx_tail = 0; - - for (i = 0; i < bp->rx_ring_size; i++) { - desc = macb_rx_desc(queue, i); - desc->ctrl = 0; - /* make sure ctrl is cleared first, - * and bit RX_USED is set to avoid a race. - */ - dma_wmb(); - desc->addr |= MACB_BIT(RX_USED); - } - - gem_init_rx_ring(queue); - } - - macb_init_tieoff(bp); -} - -static void macb_init_rings(struct macb *bp) -{ - int i; - struct macb_dma_desc *desc = NULL; - - macb_init_rx_ring(&bp->queues[0]); - - for (i = 0; i < bp->tx_ring_size; i++) { - desc = macb_tx_desc(&bp->queues[0], i); - macb_set_addr(bp, desc, 0); - desc->ctrl = MACB_BIT(TX_USED); - } - bp->queues[0].tx_head = 0; - bp->queues[0].tx_tail = 0; - desc->ctrl |= MACB_BIT(TX_WRAP); - - macb_init_tieoff(bp); -} - -static void macb_reset_hw(struct macb *bp) -{ - struct macb_queue *queue; - unsigned int q; - u32 ctrl = macb_readl(bp, NCR); - - /* Disable RX and TX (XXX: Should we halt the transmission - * more gracefully?) - */ - ctrl &= ~(MACB_BIT(RE) | MACB_BIT(TE)); - - /* Clear the stats registers (XXX: Update stats first?) */ - ctrl |= MACB_BIT(CLRSTAT); - - macb_writel(bp, NCR, ctrl); - - /* Clear all status flags */ - macb_writel(bp, TSR, -1); - macb_writel(bp, RSR, -1); - - /* Disable RX partial store and forward and reset watermark value */ - gem_writel(bp, PBUFRXCUT, 0); - - /* Disable all interrupts */ - for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { - queue_writel(queue, IDR, -1); - queue_readl(queue, ISR); - if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) - queue_writel(queue, ISR, -1); - } -} - -static u32 gem_mdc_clk_div(struct macb *bp) -{ - u32 config; - unsigned long pclk_hz = clk_get_rate(bp->pclk); - - if (pclk_hz <= 20000000) - config = GEM_BF(CLK, GEM_CLK_DIV8); - else if (pclk_hz <= 40000000) - config = GEM_BF(CLK, GEM_CLK_DIV16); - else if (pclk_hz <= 80000000) - config = GEM_BF(CLK, GEM_CLK_DIV32); - else if (pclk_hz <= 120000000) - config = GEM_BF(CLK, GEM_CLK_DIV48); - else if (pclk_hz <= 160000000) - config = GEM_BF(CLK, GEM_CLK_DIV64); - else if (pclk_hz <= 240000000) - config = GEM_BF(CLK, GEM_CLK_DIV96); - else if (pclk_hz <= 320000000) - config = GEM_BF(CLK, GEM_CLK_DIV128); - else - config = GEM_BF(CLK, GEM_CLK_DIV224); - - return config; -} - -static u32 macb_mdc_clk_div(struct macb *bp) -{ - u32 config; - unsigned long pclk_hz; - - if (macb_is_gem(bp)) - return gem_mdc_clk_div(bp); - - pclk_hz = clk_get_rate(bp->pclk); - if (pclk_hz <= 20000000) - config = MACB_BF(CLK, MACB_CLK_DIV8); - else if (pclk_hz <= 40000000) - config = MACB_BF(CLK, MACB_CLK_DIV16); - else if (pclk_hz <= 80000000) - config = MACB_BF(CLK, MACB_CLK_DIV32); - else - config = MACB_BF(CLK, MACB_CLK_DIV64); - - return config; -} - -/* Get the DMA bus width field of the network configuration register that we - * should program. We find the width from decoding the design configuration - * register to find the maximum supported data bus width. - */ -static u32 macb_dbw(struct macb *bp) -{ - if (!macb_is_gem(bp)) - return 0; - - switch (GEM_BFEXT(DBWDEF, gem_readl(bp, DCFG1))) { - case 4: - return GEM_BF(DBW, GEM_DBW128); - case 2: - return GEM_BF(DBW, GEM_DBW64); - case 1: - default: - return GEM_BF(DBW, GEM_DBW32); - } -} - -/* Configure the receive DMA engine - * - use the correct receive buffer size - * - set best burst length for DMA operations - * (if not supported by FIFO, it will fallback to default) - * - set both rx/tx packet buffers to full memory size - * These are configurable parameters for GEM. - */ -static void macb_configure_dma(struct macb *bp) -{ - struct macb_queue *queue; - u32 buffer_size; - unsigned int q; - u32 dmacfg; - - buffer_size = bp->rx_buffer_size / RX_BUFFER_MULTIPLE; - if (macb_is_gem(bp)) { - dmacfg = gem_readl(bp, DMACFG) & ~GEM_BF(RXBS, -1L); - for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { - if (q) - queue_writel(queue, RBQS, buffer_size); - else - dmacfg |= GEM_BF(RXBS, buffer_size); - } - if (bp->dma_burst_length) - dmacfg = GEM_BFINS(FBLDO, bp->dma_burst_length, dmacfg); - dmacfg |= GEM_BIT(TXPBMS) | GEM_BF(RXBMS, -1L); - dmacfg &= ~GEM_BIT(ENDIA_PKT); - - if (bp->native_io) - dmacfg &= ~GEM_BIT(ENDIA_DESC); - else - dmacfg |= GEM_BIT(ENDIA_DESC); /* CPU in big endian */ - - if (bp->dev->features & NETIF_F_HW_CSUM) - dmacfg |= GEM_BIT(TXCOEN); - else - dmacfg &= ~GEM_BIT(TXCOEN); - - dmacfg &= ~GEM_BIT(ADDR64); -#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT - if (bp->hw_dma_cap & HW_DMA_CAP_64B) - dmacfg |= GEM_BIT(ADDR64); -#endif -#ifdef CONFIG_MACB_USE_HWSTAMP - if (bp->hw_dma_cap & HW_DMA_CAP_PTP) - dmacfg |= GEM_BIT(RXEXT) | GEM_BIT(TXEXT); -#endif - netdev_dbg(bp->dev, "Cadence configure DMA with 0x%08x\n", - dmacfg); - gem_writel(bp, DMACFG, dmacfg); - } -} - -static int phytium_mac_config(struct macb *bp) -{ - u32 old_ctrl, ctrl; - u32 old_ncr, ncr; - - netdev_dbg(bp->dev, "phytium mac config"); - - ncr = macb_readl(bp, NCR); - old_ncr = ncr; - ctrl = macb_or_gem_readl(bp, NCFGR); - old_ctrl = ctrl; - - ncr &= ~(GEM_BIT(ENABLE_HS_MAC) | MACB_BIT(2PT5G)); - ctrl &= ~(GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL) | - MACB_BIT(SPD) | MACB_BIT(FD)); - if (macb_is_gem(bp)) - ctrl &= ~GEM_BIT(GBE); - - if (bp->phy_interface == PHY_INTERFACE_MODE_2500BASEX) { - ctrl |= GEM_BIT(PCSSEL) | GEM_BIT(SGMIIEN); - ncr |= MACB_BIT(2PT5G); - } else if (bp->phy_interface == PHY_INTERFACE_MODE_USXGMII || - bp->phy_interface == PHY_INTERFACE_MODE_5GBASER) { - ctrl |= GEM_BIT(PCSSEL); - ncr |= GEM_BIT(ENABLE_HS_MAC); - } - - if (bp->duplex) - ctrl |= MACB_BIT(FD); - - /* Apply the new configuration, if any */ - if (old_ctrl ^ ctrl) - macb_or_gem_writel(bp, NCFGR, ctrl); - - if (old_ncr ^ ncr) - macb_or_gem_writel(bp, NCR, ncr); - - return 0; -} - -static void macb_init_hw(struct macb *bp) -{ - u32 config; - - macb_reset_hw(bp); - macb_set_hwaddr(bp); - - config = macb_readl(bp, NCR); - config |= MACB_BIT(MPE); - macb_writel(bp, NCR, config); - - config = macb_mdc_clk_div(bp); - config |= MACB_BF(RBOF, NET_IP_ALIGN); /* Make eth data aligned */ - config |= MACB_BIT(DRFCS); /* Discard Rx FCS */ - if (bp->caps & MACB_CAPS_JUMBO) - config |= MACB_BIT(JFRAME); /* Enable jumbo frames */ - else - config |= MACB_BIT(BIG); /* Receive oversized frames */ - if (bp->dev->flags & IFF_PROMISC) - config |= MACB_BIT(CAF); /* Copy All Frames */ - else if (macb_is_gem(bp) && bp->dev->features & NETIF_F_RXCSUM) - config |= GEM_BIT(RXCOEN); - if (!(bp->dev->flags & IFF_BROADCAST)) - config |= MACB_BIT(NBC); /* No BroadCast */ - config |= macb_dbw(bp); - macb_writel(bp, NCFGR, config); - if ((bp->caps & MACB_CAPS_JUMBO) && bp->jumbo_max_len) - gem_writel(bp, JML, bp->jumbo_max_len); - bp->rx_frm_len_mask = MACB_RX_FRMLEN_MASK; - if (bp->caps & MACB_CAPS_JUMBO) - bp->rx_frm_len_mask = MACB_RX_JFRMLEN_MASK; - - gem_writel(bp, AXI_PIPE, 0x1010); - - if (bp->phy_interface == PHY_INTERFACE_MODE_USXGMII || - bp->phy_interface == PHY_INTERFACE_MODE_5GBASER || - bp->phy_interface == PHY_INTERFACE_MODE_2500BASEX) { - /* phytium need hwclock */ - if (bp->caps & MACB_CAPS_SEL_CLK) - bp->sel_clk_hw(bp, bp->speed); - phytium_mac_config(bp); - if (bp->link) - macb_usx_pcs_link_up(&bp->phylink_usx_pcs, 0, - bp->phy_interface, bp->speed, bp->duplex); - } else { - bp->speed = SPEED_10; - bp->duplex = DUPLEX_HALF; - } - - macb_configure_dma(bp); - - /* Enable RX partial store and forward and set watermark */ - if (bp->rx_watermark) - gem_writel(bp, PBUFRXCUT, (bp->rx_watermark | GEM_BIT(ENCUTTHRU))); -} - -/* The hash address register is 64 bits long and takes up two - * locations in the memory map. The least significant bits are stored - * in EMAC_HSL and the most significant bits in EMAC_HSH. - * - * The unicast hash enable and the multicast hash enable bits in the - * network configuration register enable the reception of hash matched - * frames. The destination address is reduced to a 6 bit index into - * the 64 bit hash register using the following hash function. The - * hash function is an exclusive or of every sixth bit of the - * destination address. - * - * hi[5] = da[5] ^ da[11] ^ da[17] ^ da[23] ^ da[29] ^ da[35] ^ da[41] ^ da[47] - * hi[4] = da[4] ^ da[10] ^ da[16] ^ da[22] ^ da[28] ^ da[34] ^ da[40] ^ da[46] - * hi[3] = da[3] ^ da[09] ^ da[15] ^ da[21] ^ da[27] ^ da[33] ^ da[39] ^ da[45] - * hi[2] = da[2] ^ da[08] ^ da[14] ^ da[20] ^ da[26] ^ da[32] ^ da[38] ^ da[44] - * hi[1] = da[1] ^ da[07] ^ da[13] ^ da[19] ^ da[25] ^ da[31] ^ da[37] ^ da[43] - * hi[0] = da[0] ^ da[06] ^ da[12] ^ da[18] ^ da[24] ^ da[30] ^ da[36] ^ da[42] - * - * da[0] represents the least significant bit of the first byte - * received, that is, the multicast/unicast indicator, and da[47] - * represents the most significant bit of the last byte received. If - * the hash index, hi[n], points to a bit that is set in the hash - * register then the frame will be matched according to whether the - * frame is multicast or unicast. A multicast match will be signalled - * if the multicast hash enable bit is set, da[0] is 1 and the hash - * index points to a bit set in the hash register. A unicast match - * will be signalled if the unicast hash enable bit is set, da[0] is 0 - * and the hash index points to a bit set in the hash register. To - * receive all multicast frames, the hash register should be set with - * all ones and the multicast hash enable bit should be set in the - * network configuration register. - */ - -static inline int hash_bit_value(int bitnr, __u8 *addr) -{ - if (addr[bitnr / 8] & (1 << (bitnr % 8))) - return 1; - return 0; -} - -/* Return the hash index value for the specified address. */ -static int hash_get_index(__u8 *addr) -{ - int i, j, bitval; - int hash_index = 0; - - for (j = 0; j < 6; j++) { - for (i = 0, bitval = 0; i < 8; i++) - bitval ^= hash_bit_value(i * 6 + j, addr); - - hash_index |= (bitval << j); - } - - return hash_index; -} - -/* Add multicast addresses to the internal multicast-hash table. */ -static void macb_sethashtable(struct net_device *dev) -{ - struct netdev_hw_addr *ha; - unsigned long mc_filter[2]; - unsigned int bitnr; - struct macb *bp = netdev_priv(dev); - - mc_filter[0] = 0; - mc_filter[1] = 0; - - netdev_for_each_mc_addr(ha, dev) { - bitnr = hash_get_index(ha->addr); - mc_filter[bitnr >> 5] |= 1 << (bitnr & 31); - } - - macb_or_gem_writel(bp, HRB, mc_filter[0]); - macb_or_gem_writel(bp, HRT, mc_filter[1]); -} - -/* Enable/Disable promiscuous and multicast modes. */ -static void macb_set_rx_mode(struct net_device *dev) -{ - unsigned long cfg; - struct macb *bp = netdev_priv(dev); - - cfg = macb_readl(bp, NCFGR); - - if (dev->flags & IFF_PROMISC) { - /* Enable promiscuous mode */ - cfg |= MACB_BIT(CAF); - - /* Disable RX checksum offload */ - if (macb_is_gem(bp)) - cfg &= ~GEM_BIT(RXCOEN); - } else { - /* Disable promiscuous mode */ - cfg &= ~MACB_BIT(CAF); - - /* Enable RX checksum offload only if requested */ - if (macb_is_gem(bp) && dev->features & NETIF_F_RXCSUM) - cfg |= GEM_BIT(RXCOEN); - } - - if (dev->flags & IFF_ALLMULTI) { - /* Enable all multicast mode */ - macb_or_gem_writel(bp, HRB, -1); - macb_or_gem_writel(bp, HRT, -1); - cfg |= MACB_BIT(NCFGR_MTI); - } else if (!netdev_mc_empty(dev)) { - /* Enable specific multicasts */ - macb_sethashtable(dev); - cfg |= MACB_BIT(NCFGR_MTI); - } else if (dev->flags & (~IFF_ALLMULTI)) { - /* Disable all multicast mode */ - macb_or_gem_writel(bp, HRB, 0); - macb_or_gem_writel(bp, HRT, 0); - cfg &= ~MACB_BIT(NCFGR_MTI); - } - - macb_writel(bp, NCFGR, cfg); -} - -static int macb_open(struct net_device *dev) -{ - size_t bufsz = dev->mtu + ETH_HLEN + ETH_FCS_LEN + NET_IP_ALIGN; - struct macb *bp = netdev_priv(dev); - struct macb_queue *queue; - unsigned int q; - int err; - - netdev_dbg(bp->dev, "open\n"); - - err = pm_runtime_resume_and_get(&bp->pdev->dev); - if (err < 0) - return err; - - /* RX buffers initialization */ - macb_init_rx_buffer_size(bp, bufsz); - - err = macb_alloc_consistent(bp); - if (err) { - netdev_err(dev, "Unable to allocate DMA memory (error %d)\n", - err); - goto pm_exit; - } - - bp->macbgem_ops.mog_init_rings(bp); - macb_init_buffers(bp); - - for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { - napi_enable(&queue->napi_rx); - napi_enable(&queue->napi_tx); - } - - macb_init_hw(bp); - - err = phy_power_on(bp->sgmii_phy); - if (err) - goto reset_hw; - - err = macb_phylink_connect(bp); - if (err) - goto phy_off; - - netif_tx_start_all_queues(dev); - - if (bp->ptp_info) - bp->ptp_info->ptp_init(dev); - - return 0; - -phy_off: - phy_power_off(bp->sgmii_phy); - -reset_hw: - macb_reset_hw(bp); - for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { - napi_disable(&queue->napi_rx); - napi_disable(&queue->napi_tx); - } - macb_free_consistent(bp); -pm_exit: - pm_runtime_put_sync(&bp->pdev->dev); - return err; -} - -static int macb_close(struct net_device *dev) -{ - struct macb *bp = netdev_priv(dev); - struct macb_queue *queue; - unsigned long flags; - unsigned int q; - - netif_tx_stop_all_queues(dev); - - for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { - napi_disable(&queue->napi_rx); - napi_disable(&queue->napi_tx); - } - - phylink_stop(bp->phylink); - phylink_disconnect_phy(bp->phylink); - - phy_power_off(bp->sgmii_phy); - - spin_lock_irqsave(&bp->lock, flags); - macb_reset_hw(bp); - netif_carrier_off(dev); - spin_unlock_irqrestore(&bp->lock, flags); - - macb_free_consistent(bp); - - if (bp->ptp_info) - bp->ptp_info->ptp_remove(dev); - - pm_runtime_put(&bp->pdev->dev); - - return 0; -} - -static int macb_change_mtu(struct net_device *dev, int new_mtu) -{ - if (netif_running(dev)) - return -EBUSY; - - dev->mtu = new_mtu; - - return 0; -} - -static int macb_set_mac_addr(struct net_device *dev, void *addr) -{ - int err; - - err = eth_mac_addr(dev, addr); - if (err < 0) - return err; - - macb_set_hwaddr(netdev_priv(dev)); - return 0; -} - -static void gem_update_stats(struct macb *bp) -{ - struct macb_queue *queue; - unsigned int i, q, idx; - unsigned long *stat; - - u32 *p = &bp->hw_stats.gem.tx_octets_31_0; - - for (i = 0; i < GEM_STATS_LEN; ++i, ++p) { - u32 offset = gem_statistics[i].offset; - u64 val = bp->macb_reg_readl(bp, offset); - - bp->ethtool_stats[i] += val; - *p += val; - - if (offset == GEM_OCTTXL || offset == GEM_OCTRXL) { - /* Add GEM_OCTTXH, GEM_OCTRXH */ - val = bp->macb_reg_readl(bp, offset + 4); - bp->ethtool_stats[i] += ((u64)val) << 32; - *(++p) += val; - } - } - - idx = GEM_STATS_LEN; - for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) - for (i = 0, stat = &queue->stats.first; i < QUEUE_STATS_LEN; ++i, ++stat) - bp->ethtool_stats[idx++] = *stat; -} - -static struct net_device_stats *gem_get_stats(struct macb *bp) -{ - struct gem_stats *hwstat = &bp->hw_stats.gem; - struct net_device_stats *nstat = &bp->dev->stats; - - if (!netif_running(bp->dev)) - return nstat; - - spin_lock_irq(&bp->stats_lock); - gem_update_stats(bp); - - nstat->rx_errors = (hwstat->rx_frame_check_sequence_errors + - hwstat->rx_alignment_errors + - hwstat->rx_resource_errors + - hwstat->rx_overruns + - hwstat->rx_oversize_frames + - hwstat->rx_jabbers + - hwstat->rx_undersized_frames + - hwstat->rx_length_field_frame_errors); - nstat->tx_errors = (hwstat->tx_late_collisions + - hwstat->tx_excessive_collisions + - hwstat->tx_underrun + - hwstat->tx_carrier_sense_errors); - nstat->multicast = hwstat->rx_multicast_frames; - nstat->collisions = (hwstat->tx_single_collision_frames + - hwstat->tx_multiple_collision_frames + - hwstat->tx_excessive_collisions); - nstat->rx_length_errors = (hwstat->rx_oversize_frames + - hwstat->rx_jabbers + - hwstat->rx_undersized_frames + - hwstat->rx_length_field_frame_errors); - nstat->rx_over_errors = hwstat->rx_resource_errors; - nstat->rx_crc_errors = hwstat->rx_frame_check_sequence_errors; - nstat->rx_frame_errors = hwstat->rx_alignment_errors; - nstat->rx_fifo_errors = hwstat->rx_overruns; - nstat->tx_aborted_errors = hwstat->tx_excessive_collisions; - nstat->tx_carrier_errors = hwstat->tx_carrier_sense_errors; - nstat->tx_fifo_errors = hwstat->tx_underrun; - spin_unlock_irq(&bp->stats_lock); - - return nstat; -} - -static void gem_get_ethtool_stats(struct net_device *dev, - struct ethtool_stats *stats, u64 *data) -{ - struct macb *bp = netdev_priv(dev); - - spin_lock_irq(&bp->stats_lock); - gem_update_stats(bp); - memcpy(data, &bp->ethtool_stats, sizeof(u64) - * (GEM_STATS_LEN + QUEUE_STATS_LEN * bp->num_queues)); - spin_unlock_irq(&bp->stats_lock); -} - -static int gem_get_sset_count(struct net_device *dev, int sset) -{ - struct macb *bp = netdev_priv(dev); - - switch (sset) { - case ETH_SS_STATS: - return GEM_STATS_LEN + bp->num_queues * QUEUE_STATS_LEN; - default: - return -EOPNOTSUPP; - } -} - -static void gem_get_ethtool_strings(struct net_device *dev, u32 sset, u8 *p) -{ - char stat_string[ETH_GSTRING_LEN]; - struct macb *bp = netdev_priv(dev); - struct macb_queue *queue; - unsigned int i; - unsigned int q; - - switch (sset) { - case ETH_SS_STATS: - for (i = 0; i < GEM_STATS_LEN; i++, p += ETH_GSTRING_LEN) - memcpy(p, gem_statistics[i].stat_string, - ETH_GSTRING_LEN); - - for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { - for (i = 0; i < QUEUE_STATS_LEN; i++, p += ETH_GSTRING_LEN) { - snprintf(stat_string, ETH_GSTRING_LEN, "q%d_%s", - q, queue_statistics[i].stat_string); - memcpy(p, stat_string, ETH_GSTRING_LEN); - } - } - break; - } -} - -static struct net_device_stats *macb_get_stats(struct net_device *dev) -{ - struct macb *bp = netdev_priv(dev); - struct net_device_stats *nstat = &bp->dev->stats; - struct macb_stats *hwstat = &bp->hw_stats.macb; - - if (macb_is_gem(bp)) - return gem_get_stats(bp); - - /* read stats from hardware */ - spin_lock_irq(&bp->stats_lock); - macb_update_stats(bp); - - /* Convert HW stats into netdevice stats */ - nstat->rx_errors = (hwstat->rx_fcs_errors + - hwstat->rx_align_errors + - hwstat->rx_resource_errors + - hwstat->rx_overruns + - hwstat->rx_oversize_pkts + - hwstat->rx_jabbers + - hwstat->rx_undersize_pkts + - hwstat->rx_length_mismatch); - nstat->tx_errors = (hwstat->tx_late_cols + - hwstat->tx_excessive_cols + - hwstat->tx_underruns + - hwstat->tx_carrier_errors + - hwstat->sqe_test_errors); - nstat->collisions = (hwstat->tx_single_cols + - hwstat->tx_multiple_cols + - hwstat->tx_excessive_cols); - nstat->rx_length_errors = (hwstat->rx_oversize_pkts + - hwstat->rx_jabbers + - hwstat->rx_undersize_pkts + - hwstat->rx_length_mismatch); - nstat->rx_over_errors = hwstat->rx_resource_errors + - hwstat->rx_overruns; - nstat->rx_crc_errors = hwstat->rx_fcs_errors; - nstat->rx_frame_errors = hwstat->rx_align_errors; - nstat->rx_fifo_errors = hwstat->rx_overruns; - /* XXX: What does "missed" mean? */ - nstat->tx_aborted_errors = hwstat->tx_excessive_cols; - nstat->tx_carrier_errors = hwstat->tx_carrier_errors; - nstat->tx_fifo_errors = hwstat->tx_underruns; - /* Don't know about heartbeat or window errors... */ - spin_unlock_irq(&bp->stats_lock); - - return nstat; -} - -static int macb_get_regs_len(struct net_device *netdev) -{ - return MACB_GREGS_NBR * sizeof(u32); -} - -static void macb_get_regs(struct net_device *dev, struct ethtool_regs *regs, - void *p) -{ - struct macb *bp = netdev_priv(dev); - unsigned int tail, head; - u32 *regs_buff = p; - - regs->version = (macb_readl(bp, MID) & ((1 << MACB_REV_SIZE) - 1)) - | MACB_GREGS_VERSION; - - tail = macb_tx_ring_wrap(bp, bp->queues[0].tx_tail); - head = macb_tx_ring_wrap(bp, bp->queues[0].tx_head); - - regs_buff[0] = macb_readl(bp, NCR); - regs_buff[1] = macb_or_gem_readl(bp, NCFGR); - regs_buff[2] = macb_readl(bp, NSR); - regs_buff[3] = macb_readl(bp, TSR); - regs_buff[4] = macb_readl(bp, RBQP); - regs_buff[5] = macb_readl(bp, TBQP); - regs_buff[6] = macb_readl(bp, RSR); - regs_buff[7] = macb_readl(bp, IMR); - - regs_buff[8] = tail; - regs_buff[9] = head; - regs_buff[10] = macb_tx_dma(&bp->queues[0], tail); - regs_buff[11] = macb_tx_dma(&bp->queues[0], head); - - if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) - regs_buff[12] = macb_or_gem_readl(bp, USRIO); - if (macb_is_gem(bp)) - regs_buff[13] = gem_readl(bp, DMACFG); -} - -static void macb_get_wol(struct net_device *netdev, struct ethtool_wolinfo *wol) -{ - struct macb *bp = netdev_priv(netdev); - - if (bp->wol & MACB_WOL_HAS_MAGIC_PACKET) { - phylink_ethtool_get_wol(bp->phylink, wol); - wol->supported |= WAKE_MAGIC; - - if (bp->wol & MACB_WOL_ENABLED) - wol->wolopts |= WAKE_MAGIC; - } -} - -static int macb_set_wol(struct net_device *netdev, struct ethtool_wolinfo *wol) -{ - struct macb *bp = netdev_priv(netdev); - int ret; - - /* Pass the order to phylink layer */ - ret = phylink_ethtool_set_wol(bp->phylink, wol); - /* Don't manage WoL on MAC if handled by the PHY - * or if there's a failure in talking to the PHY - */ - if (!ret || ret != -EOPNOTSUPP) - return ret; - - if (!(bp->wol & MACB_WOL_HAS_MAGIC_PACKET) || - (wol->wolopts & ~WAKE_MAGIC)) - return -EOPNOTSUPP; - - if (wol->wolopts & WAKE_MAGIC) - bp->wol |= MACB_WOL_ENABLED; - else - bp->wol &= ~MACB_WOL_ENABLED; - - device_set_wakeup_enable(&bp->pdev->dev, bp->wol & MACB_WOL_ENABLED); - - return 0; -} - -static int macb_get_link_ksettings(struct net_device *netdev, - struct ethtool_link_ksettings *kset) -{ - struct macb *bp = netdev_priv(netdev); - - return phylink_ethtool_ksettings_get(bp->phylink, kset); -} - -static int macb_set_link_ksettings(struct net_device *netdev, - const struct ethtool_link_ksettings *kset) -{ - struct macb *bp = netdev_priv(netdev); - - return phylink_ethtool_ksettings_set(bp->phylink, kset); -} - -static void macb_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring, - struct kernel_ethtool_ringparam *kernel_ring, - struct netlink_ext_ack *extack) -{ - struct macb *bp = netdev_priv(netdev); - - ring->rx_max_pending = MAX_RX_RING_SIZE; - ring->tx_max_pending = MAX_TX_RING_SIZE; - - ring->rx_pending = bp->rx_ring_size; - ring->tx_pending = bp->tx_ring_size; -} - -static int macb_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring, - struct kernel_ethtool_ringparam *kernel_ring, - struct netlink_ext_ack *extack) -{ - struct macb *bp = netdev_priv(netdev); - u32 new_rx_size, new_tx_size; - unsigned int reset = 0; - - if ((ring->rx_mini_pending) || (ring->rx_jumbo_pending)) - return -EINVAL; - - new_rx_size = clamp_t(u32, ring->rx_pending, - MIN_RX_RING_SIZE, MAX_RX_RING_SIZE); - new_rx_size = roundup_pow_of_two(new_rx_size); - - new_tx_size = clamp_t(u32, ring->tx_pending, - MIN_TX_RING_SIZE, MAX_TX_RING_SIZE); - new_tx_size = roundup_pow_of_two(new_tx_size); - - if ((new_tx_size == bp->tx_ring_size) && - (new_rx_size == bp->rx_ring_size)) { - /* nothing to do */ - return 0; - } - - if (netif_running(bp->dev)) { - reset = 1; - macb_close(bp->dev); - } - - bp->rx_ring_size = new_rx_size; - bp->tx_ring_size = new_tx_size; - - if (reset) - macb_open(bp->dev); - - return 0; -} - -#ifdef CONFIG_MACB_USE_HWSTAMP -static unsigned int gem_get_tsu_rate(struct macb *bp) -{ - struct clk *tsu_clk; - unsigned int tsu_rate; - - tsu_clk = devm_clk_get(&bp->pdev->dev, "tsu_clk"); - if (!IS_ERR(tsu_clk)) - tsu_rate = clk_get_rate(tsu_clk); - /* try pclk instead */ - else if (!IS_ERR(bp->pclk)) { - tsu_clk = bp->pclk; - tsu_rate = clk_get_rate(tsu_clk); - } else - return -ENOTSUPP; - return tsu_rate; -} - -static s32 gem_get_ptp_max_adj(void) -{ - return 64000000; -} - -static int gem_get_ts_info(struct net_device *dev, - struct ethtool_ts_info *info) -{ - struct macb *bp = netdev_priv(dev); - - if ((bp->hw_dma_cap & HW_DMA_CAP_PTP) == 0) { - ethtool_op_get_ts_info(dev, info); - return 0; - } - - info->so_timestamping = - SOF_TIMESTAMPING_TX_SOFTWARE | - SOF_TIMESTAMPING_RX_SOFTWARE | - SOF_TIMESTAMPING_SOFTWARE | - SOF_TIMESTAMPING_TX_HARDWARE | - SOF_TIMESTAMPING_RX_HARDWARE | - SOF_TIMESTAMPING_RAW_HARDWARE; - info->tx_types = - (1 << HWTSTAMP_TX_ONESTEP_SYNC) | - (1 << HWTSTAMP_TX_OFF) | - (1 << HWTSTAMP_TX_ON); - info->rx_filters = - (1 << HWTSTAMP_FILTER_NONE) | - (1 << HWTSTAMP_FILTER_ALL); - - info->phc_index = bp->ptp_clock ? ptp_clock_index(bp->ptp_clock) : -1; - - return 0; -} - -static struct macb_ptp_info gem_ptp_info = { - .ptp_init = gem_ptp_init, - .ptp_remove = gem_ptp_remove, - .get_ptp_max_adj = gem_get_ptp_max_adj, - .get_tsu_rate = gem_get_tsu_rate, - .get_ts_info = gem_get_ts_info, - .get_hwtst = gem_get_hwtst, - .set_hwtst = gem_set_hwtst, -}; -#endif - -static int macb_get_ts_info(struct net_device *netdev, - struct ethtool_ts_info *info) -{ - struct macb *bp = netdev_priv(netdev); - - if (bp->ptp_info) - return bp->ptp_info->get_ts_info(netdev, info); - - return ethtool_op_get_ts_info(netdev, info); -} - -static void gem_enable_flow_filters(struct macb *bp, bool enable) -{ - struct net_device *netdev = bp->dev; - struct ethtool_rx_fs_item *item; - u32 t2_scr; - int num_t2_scr; - - if (!(netdev->features & NETIF_F_NTUPLE)) - return; - - num_t2_scr = GEM_BFEXT(T2SCR, gem_readl(bp, DCFG8)); - - list_for_each_entry(item, &bp->rx_fs_list.list, list) { - struct ethtool_rx_flow_spec *fs = &item->fs; - struct ethtool_tcpip4_spec *tp4sp_m; - - if (fs->location >= num_t2_scr) - continue; - - t2_scr = gem_readl_n(bp, SCRT2, fs->location); - - /* enable/disable screener regs for the flow entry */ - t2_scr = GEM_BFINS(ETHTEN, enable, t2_scr); - - /* only enable fields with no masking */ - tp4sp_m = &(fs->m_u.tcp_ip4_spec); - - if (enable && (tp4sp_m->ip4src == 0xFFFFFFFF)) - t2_scr = GEM_BFINS(CMPAEN, 1, t2_scr); - else - t2_scr = GEM_BFINS(CMPAEN, 0, t2_scr); - - if (enable && (tp4sp_m->ip4dst == 0xFFFFFFFF)) - t2_scr = GEM_BFINS(CMPBEN, 1, t2_scr); - else - t2_scr = GEM_BFINS(CMPBEN, 0, t2_scr); - - if (enable && ((tp4sp_m->psrc == 0xFFFF) || (tp4sp_m->pdst == 0xFFFF))) - t2_scr = GEM_BFINS(CMPCEN, 1, t2_scr); - else - t2_scr = GEM_BFINS(CMPCEN, 0, t2_scr); - - gem_writel_n(bp, SCRT2, fs->location, t2_scr); - } -} - -static void gem_prog_cmp_regs(struct macb *bp, struct ethtool_rx_flow_spec *fs) -{ - struct ethtool_tcpip4_spec *tp4sp_v, *tp4sp_m; - uint16_t index = fs->location; - u32 w0, w1, t2_scr; - bool cmp_a = false; - bool cmp_b = false; - bool cmp_c = false; - - if (!macb_is_gem(bp)) - return; - - tp4sp_v = &(fs->h_u.tcp_ip4_spec); - tp4sp_m = &(fs->m_u.tcp_ip4_spec); - - /* ignore field if any masking set */ - if (tp4sp_m->ip4src == 0xFFFFFFFF) { - /* 1st compare reg - IP source address */ - w0 = 0; - w1 = 0; - w0 = tp4sp_v->ip4src; - w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */ - w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_ETYPE, w1); - w1 = GEM_BFINS(T2OFST, ETYPE_SRCIP_OFFSET, w1); - gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_IP4SRC_CMP(index)), w0); - gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_IP4SRC_CMP(index)), w1); - cmp_a = true; - } - - /* ignore field if any masking set */ - if (tp4sp_m->ip4dst == 0xFFFFFFFF) { - /* 2nd compare reg - IP destination address */ - w0 = 0; - w1 = 0; - w0 = tp4sp_v->ip4dst; - w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */ - w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_ETYPE, w1); - w1 = GEM_BFINS(T2OFST, ETYPE_DSTIP_OFFSET, w1); - gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_IP4DST_CMP(index)), w0); - gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_IP4DST_CMP(index)), w1); - cmp_b = true; - } - - /* ignore both port fields if masking set in both */ - if ((tp4sp_m->psrc == 0xFFFF) || (tp4sp_m->pdst == 0xFFFF)) { - /* 3rd compare reg - source port, destination port */ - w0 = 0; - w1 = 0; - w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_IPHDR, w1); - if (tp4sp_m->psrc == tp4sp_m->pdst) { - w0 = GEM_BFINS(T2MASK, tp4sp_v->psrc, w0); - w0 = GEM_BFINS(T2CMP, tp4sp_v->pdst, w0); - w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */ - w1 = GEM_BFINS(T2OFST, IPHDR_SRCPORT_OFFSET, w1); - } else { - /* only one port definition */ - w1 = GEM_BFINS(T2DISMSK, 0, w1); /* 16-bit compare */ - w0 = GEM_BFINS(T2MASK, 0xFFFF, w0); - if (tp4sp_m->psrc == 0xFFFF) { /* src port */ - w0 = GEM_BFINS(T2CMP, tp4sp_v->psrc, w0); - w1 = GEM_BFINS(T2OFST, IPHDR_SRCPORT_OFFSET, w1); - } else { /* dst port */ - w0 = GEM_BFINS(T2CMP, tp4sp_v->pdst, w0); - w1 = GEM_BFINS(T2OFST, IPHDR_DSTPORT_OFFSET, w1); - } - } - gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_PORT_CMP(index)), w0); - gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_PORT_CMP(index)), w1); - cmp_c = true; - } - - t2_scr = 0; - t2_scr = GEM_BFINS(QUEUE, (fs->ring_cookie) & 0xFF, t2_scr); - t2_scr = GEM_BFINS(ETHT2IDX, SCRT2_ETHT, t2_scr); - if (cmp_a) - t2_scr = GEM_BFINS(CMPA, GEM_IP4SRC_CMP(index), t2_scr); - if (cmp_b) - t2_scr = GEM_BFINS(CMPB, GEM_IP4DST_CMP(index), t2_scr); - if (cmp_c) - t2_scr = GEM_BFINS(CMPC, GEM_PORT_CMP(index), t2_scr); - gem_writel_n(bp, SCRT2, index, t2_scr); -} - -static int gem_add_flow_filter(struct net_device *netdev, - struct ethtool_rxnfc *cmd) -{ - struct macb *bp = netdev_priv(netdev); - struct ethtool_rx_flow_spec *fs = &cmd->fs; - struct ethtool_rx_fs_item *item, *newfs; - unsigned long flags; - int ret = -EINVAL; - bool added = false; - - newfs = kmalloc(sizeof(*newfs), GFP_KERNEL); - if (newfs == NULL) - return -ENOMEM; - memcpy(&newfs->fs, fs, sizeof(newfs->fs)); - - netdev_dbg(netdev, - "Adding flow filter entry,type=%u,queue=%u,loc=%u,src=%08X,dst=%08X,ps=%u,pd=%u\n", - fs->flow_type, (int)fs->ring_cookie, fs->location, - htonl(fs->h_u.tcp_ip4_spec.ip4src), - htonl(fs->h_u.tcp_ip4_spec.ip4dst), - be16_to_cpu(fs->h_u.tcp_ip4_spec.psrc), - be16_to_cpu(fs->h_u.tcp_ip4_spec.pdst)); - - spin_lock_irqsave(&bp->rx_fs_lock, flags); - - /* find correct place to add in list */ - list_for_each_entry(item, &bp->rx_fs_list.list, list) { - if (item->fs.location > newfs->fs.location) { - list_add_tail(&newfs->list, &item->list); - added = true; - break; - } else if (item->fs.location == fs->location) { - netdev_err(netdev, "Rule not added: location %d not free!\n", - fs->location); - ret = -EBUSY; - goto err; - } - } - if (!added) - list_add_tail(&newfs->list, &bp->rx_fs_list.list); - - gem_prog_cmp_regs(bp, fs); - bp->rx_fs_list.count++; - /* enable filtering if NTUPLE on */ - gem_enable_flow_filters(bp, 1); - - spin_unlock_irqrestore(&bp->rx_fs_lock, flags); - return 0; - -err: - spin_unlock_irqrestore(&bp->rx_fs_lock, flags); - kfree(newfs); - return ret; -} - -static int gem_del_flow_filter(struct net_device *netdev, - struct ethtool_rxnfc *cmd) -{ - struct macb *bp = netdev_priv(netdev); - struct ethtool_rx_fs_item *item; - struct ethtool_rx_flow_spec *fs; - unsigned long flags; - - spin_lock_irqsave(&bp->rx_fs_lock, flags); - - list_for_each_entry(item, &bp->rx_fs_list.list, list) { - if (item->fs.location == cmd->fs.location) { - /* disable screener regs for the flow entry */ - fs = &(item->fs); - netdev_dbg(netdev, - "Deleting flow filter entry,type=%u,queue=%u,loc=%u,src=%08X,dst=%08X,ps=%u,pd=%u\n", - fs->flow_type, (int)fs->ring_cookie, fs->location, - htonl(fs->h_u.tcp_ip4_spec.ip4src), - htonl(fs->h_u.tcp_ip4_spec.ip4dst), - be16_to_cpu(fs->h_u.tcp_ip4_spec.psrc), - be16_to_cpu(fs->h_u.tcp_ip4_spec.pdst)); - - gem_writel_n(bp, SCRT2, fs->location, 0); - - list_del(&item->list); - bp->rx_fs_list.count--; - spin_unlock_irqrestore(&bp->rx_fs_lock, flags); - kfree(item); - return 0; - } - } - - spin_unlock_irqrestore(&bp->rx_fs_lock, flags); - return -EINVAL; -} - -static int gem_get_flow_entry(struct net_device *netdev, - struct ethtool_rxnfc *cmd) -{ - struct macb *bp = netdev_priv(netdev); - struct ethtool_rx_fs_item *item; - - list_for_each_entry(item, &bp->rx_fs_list.list, list) { - if (item->fs.location == cmd->fs.location) { - memcpy(&cmd->fs, &item->fs, sizeof(cmd->fs)); - return 0; - } - } - return -EINVAL; -} - -static int gem_get_all_flow_entries(struct net_device *netdev, - struct ethtool_rxnfc *cmd, u32 *rule_locs) -{ - struct macb *bp = netdev_priv(netdev); - struct ethtool_rx_fs_item *item; - uint32_t cnt = 0; - - list_for_each_entry(item, &bp->rx_fs_list.list, list) { - if (cnt == cmd->rule_cnt) - return -EMSGSIZE; - rule_locs[cnt] = item->fs.location; - cnt++; - } - cmd->data = bp->max_tuples; - cmd->rule_cnt = cnt; - - return 0; -} - -static int gem_get_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd, - u32 *rule_locs) -{ - struct macb *bp = netdev_priv(netdev); - int ret = 0; - - switch (cmd->cmd) { - case ETHTOOL_GRXRINGS: - cmd->data = bp->num_queues; - break; - case ETHTOOL_GRXCLSRLCNT: - cmd->rule_cnt = bp->rx_fs_list.count; - break; - case ETHTOOL_GRXCLSRULE: - ret = gem_get_flow_entry(netdev, cmd); - break; - case ETHTOOL_GRXCLSRLALL: - ret = gem_get_all_flow_entries(netdev, cmd, rule_locs); - break; - default: - netdev_err(netdev, - "Command parameter %d is not supported\n", cmd->cmd); - ret = -EOPNOTSUPP; - } - - return ret; -} - -static int gem_set_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd) -{ - struct macb *bp = netdev_priv(netdev); - int ret; - - if (!(netdev->hw_features & NETIF_F_NTUPLE)) - return -EOPNOTSUPP; - - switch (cmd->cmd) { - case ETHTOOL_SRXCLSRLINS: - if ((cmd->fs.location >= bp->max_tuples) - || (cmd->fs.ring_cookie >= bp->num_queues)) { - ret = -EINVAL; - break; - } - ret = gem_add_flow_filter(netdev, cmd); - break; - case ETHTOOL_SRXCLSRLDEL: - ret = gem_del_flow_filter(netdev, cmd); - break; - default: - netdev_err(netdev, - "Command parameter %d is not supported\n", cmd->cmd); - ret = -EOPNOTSUPP; - } - - return ret; -} - -static const struct ethtool_ops macb_ethtool_ops = { - .get_regs_len = macb_get_regs_len, - .get_regs = macb_get_regs, - .get_link = ethtool_op_get_link, - .get_ts_info = ethtool_op_get_ts_info, - .get_wol = macb_get_wol, - .set_wol = macb_set_wol, - .get_link_ksettings = macb_get_link_ksettings, - .set_link_ksettings = macb_set_link_ksettings, - .get_ringparam = macb_get_ringparam, - .set_ringparam = macb_set_ringparam, -}; - -static const struct ethtool_ops gem_ethtool_ops = { - .get_regs_len = macb_get_regs_len, - .get_regs = macb_get_regs, - .get_wol = macb_get_wol, - .set_wol = macb_set_wol, - .get_link = ethtool_op_get_link, - .get_ts_info = macb_get_ts_info, - .get_ethtool_stats = gem_get_ethtool_stats, - .get_strings = gem_get_ethtool_strings, - .get_sset_count = gem_get_sset_count, - .get_link_ksettings = macb_get_link_ksettings, - .set_link_ksettings = macb_set_link_ksettings, - .get_ringparam = macb_get_ringparam, - .set_ringparam = macb_set_ringparam, - .get_rxnfc = gem_get_rxnfc, - .set_rxnfc = gem_set_rxnfc, -}; - -static int macb_ioctl(struct net_device *dev, struct ifreq *rq, int cmd) -{ - struct macb *bp = netdev_priv(dev); - - if (!netif_running(dev)) - return -EINVAL; - - if (bp->ptp_info) { - switch (cmd) { - case SIOCSHWTSTAMP: - return bp->ptp_info->set_hwtst(dev, rq, cmd); - case SIOCGHWTSTAMP: - return bp->ptp_info->get_hwtst(dev, rq); - } - } - - return phylink_mii_ioctl(bp->phylink, rq, cmd); -} - -static inline void macb_set_txcsum_feature(struct macb *bp, - netdev_features_t features) -{ - u32 val; - - if (!macb_is_gem(bp)) - return; - - val = gem_readl(bp, DMACFG); - if (features & NETIF_F_HW_CSUM) - val |= GEM_BIT(TXCOEN); - else - val &= ~GEM_BIT(TXCOEN); - - gem_writel(bp, DMACFG, val); -} - -static inline void macb_set_rxcsum_feature(struct macb *bp, - netdev_features_t features) -{ - struct net_device *netdev = bp->dev; - u32 val; - - if (!macb_is_gem(bp)) - return; - - val = gem_readl(bp, NCFGR); - if ((features & NETIF_F_RXCSUM) && !(netdev->flags & IFF_PROMISC)) - val |= GEM_BIT(RXCOEN); - else - val &= ~GEM_BIT(RXCOEN); - - gem_writel(bp, NCFGR, val); -} - -static inline void macb_set_rxflow_feature(struct macb *bp, - netdev_features_t features) -{ - if (!macb_is_gem(bp)) - return; - - gem_enable_flow_filters(bp, !!(features & NETIF_F_NTUPLE)); -} - -static int macb_set_features(struct net_device *netdev, - netdev_features_t features) -{ - struct macb *bp = netdev_priv(netdev); - netdev_features_t changed = features ^ netdev->features; - - /* TX checksum offload */ - if (changed & NETIF_F_HW_CSUM) - macb_set_txcsum_feature(bp, features); - - /* RX checksum offload */ - if (changed & NETIF_F_RXCSUM) - macb_set_rxcsum_feature(bp, features); - - /* RX Flow Filters */ - if (changed & NETIF_F_NTUPLE) - macb_set_rxflow_feature(bp, features); - - return 0; -} - -static void macb_restore_features(struct macb *bp) -{ - struct net_device *netdev = bp->dev; - netdev_features_t features = netdev->features; - struct ethtool_rx_fs_item *item; - - /* TX checksum offload */ - macb_set_txcsum_feature(bp, features); - - /* RX checksum offload */ - macb_set_rxcsum_feature(bp, features); - - /* RX Flow Filters */ - list_for_each_entry(item, &bp->rx_fs_list.list, list) - gem_prog_cmp_regs(bp, &item->fs); - - macb_set_rxflow_feature(bp, features); -} - -static const struct net_device_ops macb_netdev_ops = { - .ndo_open = macb_open, - .ndo_stop = macb_close, - .ndo_start_xmit = macb_start_xmit, - .ndo_set_rx_mode = macb_set_rx_mode, - .ndo_get_stats = macb_get_stats, - .ndo_eth_ioctl = macb_ioctl, - .ndo_validate_addr = eth_validate_addr, - .ndo_change_mtu = macb_change_mtu, - .ndo_set_mac_address = macb_set_mac_addr, -#ifdef CONFIG_NET_POLL_CONTROLLER - .ndo_poll_controller = macb_poll_controller, -#endif - .ndo_set_features = macb_set_features, - .ndo_features_check = macb_features_check, - .ndo_vlan_rx_add_vid = ncsi_vlan_rx_add_vid, - .ndo_vlan_rx_kill_vid = ncsi_vlan_rx_kill_vid, -}; - -/* Configure peripheral capabilities according to device tree - * and integration options used - */ -static void macb_configure_caps(struct macb *bp, - const struct macb_config *dt_conf) -{ - u32 dcfg; - - if (dt_conf) - bp->caps = dt_conf->caps; - - if (hw_is_gem(bp->regs, bp->native_io)) { - bp->caps |= MACB_CAPS_MACB_IS_GEM; - - dcfg = gem_readl(bp, DCFG1); - if (GEM_BFEXT(IRQCOR, dcfg) == 0) - bp->caps |= MACB_CAPS_ISR_CLEAR_ON_WRITE; - if (GEM_BFEXT(NO_PCS, dcfg) == 0) - bp->caps |= MACB_CAPS_PCS; - dcfg = gem_readl(bp, DCFG12); - if (GEM_BFEXT(HIGH_SPEED, dcfg) == 1) - bp->caps |= MACB_CAPS_HIGH_SPEED; - dcfg = gem_readl(bp, DCFG2); - if ((dcfg & (GEM_BIT(RX_PKT_BUFF) | GEM_BIT(TX_PKT_BUFF))) == 0) - bp->caps |= MACB_CAPS_FIFO_MODE; - if (gem_has_ptp(bp)) { - if (!GEM_BFEXT(TSU, gem_readl(bp, DCFG5))) - dev_err(&bp->pdev->dev, - "GEM doesn't support hardware ptp.\n"); - else { -#ifdef CONFIG_MACB_USE_HWSTAMP - bp->hw_dma_cap |= HW_DMA_CAP_PTP; - bp->ptp_info = &gem_ptp_info; -#endif - } - } - } - - dev_dbg(&bp->pdev->dev, "Cadence caps 0x%08x\n", bp->caps); -} - -static void macb_probe_queues(void __iomem *mem, - bool native_io, - unsigned int *queue_mask, - unsigned int *num_queues) -{ - *queue_mask = 0x1; - *num_queues = 1; - - /* is it macb or gem ? - * - * We need to read directly from the hardware here because - * we are early in the probe process and don't have the - * MACB_CAPS_MACB_IS_GEM flag positioned - */ - if (!hw_is_gem(mem, native_io)) - return; - - /* bit 0 is never set but queue 0 always exists */ - *queue_mask |= readl_relaxed(mem + GEM_DCFG6) & 0xff; - *num_queues = hweight32(*queue_mask); -} - -static void macb_clks_disable(struct clk *pclk, struct clk *hclk, struct clk *tx_clk, - struct clk *rx_clk, struct clk *tsu_clk) -{ - struct clk_bulk_data clks[] = { - { .clk = tsu_clk, }, - { .clk = rx_clk, }, - { .clk = pclk, }, - { .clk = hclk, }, - { .clk = tx_clk }, - }; - - clk_bulk_disable_unprepare(ARRAY_SIZE(clks), clks); -} - -static int macb_clk_init(struct platform_device *pdev, struct clk **pclk, - struct clk **hclk, struct clk **tx_clk, - struct clk **rx_clk, struct clk **tsu_clk) -{ - struct macb_platform_data *pdata; - int err; - - pdata = dev_get_platdata(&pdev->dev); - if (pdata) { - *pclk = pdata->pclk; - *hclk = pdata->hclk; - } else { - *pclk = devm_clk_get(&pdev->dev, "pclk"); - *hclk = devm_clk_get(&pdev->dev, "hclk"); - } - - if (IS_ERR_OR_NULL(*pclk)) - return dev_err_probe(&pdev->dev, - IS_ERR(*pclk) ? PTR_ERR(*pclk) : -ENODEV, - "failed to get pclk\n"); - - if (IS_ERR_OR_NULL(*hclk)) - return dev_err_probe(&pdev->dev, - IS_ERR(*hclk) ? PTR_ERR(*hclk) : -ENODEV, - "failed to get hclk\n"); - - *tx_clk = devm_clk_get_optional(&pdev->dev, "tx_clk"); - if (IS_ERR(*tx_clk)) - return PTR_ERR(*tx_clk); - - *rx_clk = devm_clk_get_optional(&pdev->dev, "rx_clk"); - if (IS_ERR(*rx_clk)) - return PTR_ERR(*rx_clk); - - *tsu_clk = devm_clk_get_optional(&pdev->dev, "tsu_clk"); - if (IS_ERR(*tsu_clk)) - return PTR_ERR(*tsu_clk); - - err = clk_prepare_enable(*pclk); - if (err) { - dev_err(&pdev->dev, "failed to enable pclk (%d)\n", err); - return err; - } - - err = clk_prepare_enable(*hclk); - if (err) { - dev_err(&pdev->dev, "failed to enable hclk (%d)\n", err); - goto err_disable_pclk; - } - - err = clk_prepare_enable(*tx_clk); - if (err) { - dev_err(&pdev->dev, "failed to enable tx_clk (%d)\n", err); - goto err_disable_hclk; - } - - err = clk_prepare_enable(*rx_clk); - if (err) { - dev_err(&pdev->dev, "failed to enable rx_clk (%d)\n", err); - goto err_disable_txclk; - } - - err = clk_prepare_enable(*tsu_clk); - if (err) { - dev_err(&pdev->dev, "failed to enable tsu_clk (%d)\n", err); - goto err_disable_rxclk; - } - - return 0; - -err_disable_rxclk: - clk_disable_unprepare(*rx_clk); - -err_disable_txclk: - clk_disable_unprepare(*tx_clk); - -err_disable_hclk: - clk_disable_unprepare(*hclk); - -err_disable_pclk: - clk_disable_unprepare(*pclk); - - return err; -} - -static int phytium_clk_init(struct platform_device *pdev, struct clk **pclk, - struct clk **hclk, struct clk **tx_clk, - struct clk **rx_clk, struct clk **tsu_clk) -{ - struct macb_platform_data *pdata; - struct device_node *np = pdev->dev.of_node; - int err; - - pdata = dev_get_platdata(&pdev->dev); - if (pdata) { - *pclk = pdata->pclk; - *hclk = pdata->hclk; - } else { - if (has_acpi_companion(&pdev->dev)) { - *pclk = NULL; - *hclk = NULL; - } else if (np) { - *pclk = devm_clk_get(&pdev->dev, "pclk"); - *hclk = devm_clk_get(&pdev->dev, "hclk"); - } - } - - if (IS_ERR(*pclk)) - return dev_err_probe(&pdev->dev, - IS_ERR(*pclk) ? PTR_ERR(*pclk) : -ENODEV, - "failed to get pclk\n"); - - if (IS_ERR(*hclk)) - return dev_err_probe(&pdev->dev, - IS_ERR(*hclk) ? PTR_ERR(*hclk) : -ENODEV, - "failed to get hclk\n"); - - *tx_clk = devm_clk_get_optional(&pdev->dev, "tx_clk"); - if (IS_ERR(*tx_clk)) - return PTR_ERR(*tx_clk); - - *rx_clk = devm_clk_get_optional(&pdev->dev, "rx_clk"); - if (IS_ERR(*rx_clk)) - return PTR_ERR(*rx_clk); - - *tsu_clk = devm_clk_get_optional(&pdev->dev, "tsu_clk"); - if (IS_ERR(*tsu_clk)) - return PTR_ERR(*tsu_clk); - - err = clk_prepare_enable(*pclk); - if (err) { - dev_err(&pdev->dev, "failed to enable pclk (%d)\n", err); - return err; - } - - err = clk_prepare_enable(*hclk); - if (err) { - dev_err(&pdev->dev, "failed to enable hclk (%d)\n", err); - goto err_disable_pclk; - } - - err = clk_prepare_enable(*tx_clk); - if (err) { - dev_err(&pdev->dev, "failed to enable tx_clk (%d)\n", err); - goto err_disable_hclk; - } - - err = clk_prepare_enable(*rx_clk); - if (err) { - dev_err(&pdev->dev, "failed to enable rx_clk (%d)\n", err); - goto err_disable_txclk; - } - - err = clk_prepare_enable(*tsu_clk); - if (err) { - dev_err(&pdev->dev, "failed to enable tsu_clk (%d)\n", err); - goto err_disable_rxclk; - } - - return 0; - -err_disable_rxclk: - clk_disable_unprepare(*rx_clk); - -err_disable_txclk: - clk_disable_unprepare(*tx_clk); - -err_disable_hclk: - clk_disable_unprepare(*hclk); - -err_disable_pclk: - clk_disable_unprepare(*pclk); - - return err; -} - -static int macb_init(struct platform_device *pdev) -{ - struct net_device *dev = platform_get_drvdata(pdev); - unsigned int hw_q, q; - struct macb *bp = netdev_priv(dev); - struct macb_queue *queue; - int err; - u32 val, reg; - - bp->tx_ring_size = DEFAULT_TX_RING_SIZE; - bp->rx_ring_size = DEFAULT_RX_RING_SIZE; - - /* set the queue register mapping once for all: queue0 has a special - * register mapping but we don't want to test the queue index then - * compute the corresponding register offset at run time. - */ - for (hw_q = 0, q = 0; hw_q < MACB_MAX_QUEUES; ++hw_q) { - if (!(bp->queue_mask & (1 << hw_q))) - continue; - - queue = &bp->queues[q]; - queue->bp = bp; - spin_lock_init(&queue->tx_ptr_lock); - netif_napi_add(dev, &queue->napi_rx, macb_rx_poll); - netif_napi_add(dev, &queue->napi_tx, macb_tx_poll); - if (hw_q) { - queue->ISR = GEM_ISR(hw_q - 1); - queue->IER = GEM_IER(hw_q - 1); - queue->IDR = GEM_IDR(hw_q - 1); - queue->IMR = GEM_IMR(hw_q - 1); - queue->TBQP = GEM_TBQP(hw_q - 1); - queue->RBQP = GEM_RBQP(hw_q - 1); - queue->RBQS = GEM_RBQS(hw_q - 1); -#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT - if (bp->hw_dma_cap & HW_DMA_CAP_64B) { - queue->TBQPH = GEM_TBQPH(hw_q - 1); - queue->RBQPH = GEM_RBQPH(hw_q - 1); - } -#endif - } else { - /* queue0 uses legacy registers */ - queue->ISR = MACB_ISR; - queue->IER = MACB_IER; - queue->IDR = MACB_IDR; - queue->IMR = MACB_IMR; - queue->TBQP = MACB_TBQP; - queue->RBQP = MACB_RBQP; -#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT - if (bp->hw_dma_cap & HW_DMA_CAP_64B) { - queue->TBQPH = MACB_TBQPH; - queue->RBQPH = MACB_RBQPH; - } -#endif - } - - /* get irq: here we use the linux queue index, not the hardware - * queue index. the queue irq definitions in the device tree - * must remove the optional gaps that could exist in the - * hardware queue mask. - */ - queue->irq = platform_get_irq(pdev, q); - err = devm_request_irq(&pdev->dev, queue->irq, macb_interrupt, - IRQF_SHARED, dev->name, queue); - if (err) { - dev_err(&pdev->dev, - "Unable to request IRQ %d (error %d)\n", - queue->irq, err); - return err; - } - - INIT_WORK(&queue->tx_error_task, macb_tx_error_task); - q++; - } - - dev->netdev_ops = &macb_netdev_ops; - - /* setup appropriated routines according to adapter type */ - if (macb_is_gem(bp)) { - bp->max_tx_length = GEM_MAX_TX_LEN; - bp->macbgem_ops.mog_alloc_rx_buffers = gem_alloc_rx_buffers; - bp->macbgem_ops.mog_free_rx_buffers = gem_free_rx_buffers; - bp->macbgem_ops.mog_init_rings = gem_init_rings; - bp->macbgem_ops.mog_rx = gem_rx; - dev->ethtool_ops = &gem_ethtool_ops; - } else { - bp->max_tx_length = MACB_MAX_TX_LEN; - bp->macbgem_ops.mog_alloc_rx_buffers = macb_alloc_rx_buffers; - bp->macbgem_ops.mog_free_rx_buffers = macb_free_rx_buffers; - bp->macbgem_ops.mog_init_rings = macb_init_rings; - bp->macbgem_ops.mog_rx = macb_rx; - dev->ethtool_ops = &macb_ethtool_ops; - } - - dev->priv_flags |= IFF_LIVE_ADDR_CHANGE; - - /* Set features */ - dev->hw_features = NETIF_F_SG; - - /* Check LSO capability */ - if (GEM_BFEXT(PBUF_LSO, gem_readl(bp, DCFG6))) - dev->hw_features |= MACB_NETIF_LSO; - - /* Checksum offload is only available on gem with packet buffer */ - if (macb_is_gem(bp) && !(bp->caps & MACB_CAPS_FIFO_MODE)) - dev->hw_features |= NETIF_F_HW_CSUM | NETIF_F_RXCSUM; - if (bp->caps & MACB_CAPS_SG_DISABLED) - dev->hw_features &= ~NETIF_F_SG; - dev->features = dev->hw_features; - - /* Check RX Flow Filters support. - * Max Rx flows set by availability of screeners & compare regs: - * each 4-tuple define requires 1 T2 screener reg + 3 compare regs - */ - reg = gem_readl(bp, DCFG8); - bp->max_tuples = min((GEM_BFEXT(SCR2CMP, reg) / 3), - GEM_BFEXT(T2SCR, reg)); - INIT_LIST_HEAD(&bp->rx_fs_list.list); - if (bp->max_tuples > 0) { - /* also needs one ethtype match to check IPv4 */ - if (GEM_BFEXT(SCR2ETH, reg) > 0) { - /* program this reg now */ - reg = 0; - reg = GEM_BFINS(ETHTCMP, (uint16_t)ETH_P_IP, reg); - gem_writel_n(bp, ETHT, SCRT2_ETHT, reg); - /* Filtering is supported in hw but don't enable it in kernel now */ - dev->hw_features |= NETIF_F_NTUPLE; - /* init Rx flow definitions */ - bp->rx_fs_list.count = 0; - spin_lock_init(&bp->rx_fs_lock); - } else - bp->max_tuples = 0; - } - - if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) { - val = 0; - if (phy_interface_mode_is_rgmii(bp->phy_interface)) - val = bp->usrio->rgmii; - else if (bp->phy_interface == PHY_INTERFACE_MODE_RMII && - (bp->caps & MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII)) - val = bp->usrio->rmii; - else if (!(bp->caps & MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII)) - val = bp->usrio->mii; - - if (bp->caps & MACB_CAPS_USRIO_HAS_CLKEN) - val |= bp->usrio->refclk; - - macb_or_gem_writel(bp, USRIO, val); - } - - /* Set MII management clock divider */ - val = macb_mdc_clk_div(bp); - val |= macb_dbw(bp); - if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) - val |= GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL); - macb_writel(bp, NCFGR, val); - - return 0; -} - -static const struct macb_usrio_config macb_default_usrio = { - .mii = MACB_BIT(MII), - .rmii = MACB_BIT(RMII), - .rgmii = GEM_BIT(RGMII), - .refclk = MACB_BIT(CLKEN), -}; - -static const struct macb_config phytium_gem1p0_config = { - .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | - MACB_CAPS_JUMBO | - MACB_CAPS_GEM_HAS_PTP | - MACB_CAPS_BD_RD_PREFETCH | - MACB_CAPS_SEL_CLK, - .dma_burst_length = 16, - .clk_init = phytium_clk_init, - .init = macb_init, - .jumbo_max_len = 10240, - .sel_clk_hw = phytium_gem1p0_sel_clk, - .usrio = &macb_default_usrio, -}; - -/* 1518 rounded up */ -#define AT91ETHER_MAX_RBUFF_SZ 0x600 -/* max number of receive buffers */ -#define AT91ETHER_MAX_RX_DESCR 9 - -static struct sifive_fu540_macb_mgmt *mgmt; - -static int at91ether_alloc_coherent(struct macb *lp) -{ - struct macb_queue *q = &lp->queues[0]; - - q->rx_ring = dma_alloc_coherent(&lp->pdev->dev, - (AT91ETHER_MAX_RX_DESCR * - macb_dma_desc_get_size(lp)), - &q->rx_ring_dma, GFP_KERNEL); - if (!q->rx_ring) - return -ENOMEM; - - q->rx_buffers = dma_alloc_coherent(&lp->pdev->dev, - AT91ETHER_MAX_RX_DESCR * - AT91ETHER_MAX_RBUFF_SZ, - &q->rx_buffers_dma, GFP_KERNEL); - if (!q->rx_buffers) { - dma_free_coherent(&lp->pdev->dev, - AT91ETHER_MAX_RX_DESCR * - macb_dma_desc_get_size(lp), - q->rx_ring, q->rx_ring_dma); - q->rx_ring = NULL; - return -ENOMEM; - } - - return 0; -} - -static void at91ether_free_coherent(struct macb *lp) -{ - struct macb_queue *q = &lp->queues[0]; - - if (q->rx_ring) { - dma_free_coherent(&lp->pdev->dev, - AT91ETHER_MAX_RX_DESCR * - macb_dma_desc_get_size(lp), - q->rx_ring, q->rx_ring_dma); - q->rx_ring = NULL; - } - - if (q->rx_buffers) { - dma_free_coherent(&lp->pdev->dev, - AT91ETHER_MAX_RX_DESCR * - AT91ETHER_MAX_RBUFF_SZ, - q->rx_buffers, q->rx_buffers_dma); - q->rx_buffers = NULL; - } -} - -/* Initialize and start the Receiver and Transmit subsystems */ -static int at91ether_start(struct macb *lp) -{ - struct macb_queue *q = &lp->queues[0]; - struct macb_dma_desc *desc; - dma_addr_t addr; - u32 ctl; - int i, ret; - - ret = at91ether_alloc_coherent(lp); - if (ret) - return ret; - - addr = q->rx_buffers_dma; - for (i = 0; i < AT91ETHER_MAX_RX_DESCR; i++) { - desc = macb_rx_desc(q, i); - macb_set_addr(lp, desc, addr); - desc->ctrl = 0; - addr += AT91ETHER_MAX_RBUFF_SZ; - } - - /* Set the Wrap bit on the last descriptor */ - desc->addr |= MACB_BIT(RX_WRAP); - - /* Reset buffer index */ - q->rx_tail = 0; - - /* Program address of descriptor list in Rx Buffer Queue register */ - macb_writel(lp, RBQP, q->rx_ring_dma); - - /* Enable Receive and Transmit */ - ctl = macb_readl(lp, NCR); - macb_writel(lp, NCR, ctl | MACB_BIT(RE) | MACB_BIT(TE)); - - /* Enable MAC interrupts */ - macb_writel(lp, IER, MACB_BIT(RCOMP) | - MACB_BIT(RXUBR) | - MACB_BIT(ISR_TUND) | - MACB_BIT(ISR_RLE) | - MACB_BIT(TCOMP) | - MACB_BIT(ISR_ROVR) | - MACB_BIT(HRESP)); - - return 0; -} - -static void at91ether_stop(struct macb *lp) -{ - u32 ctl; - - /* Disable MAC interrupts */ - macb_writel(lp, IDR, MACB_BIT(RCOMP) | - MACB_BIT(RXUBR) | - MACB_BIT(ISR_TUND) | - MACB_BIT(ISR_RLE) | - MACB_BIT(TCOMP) | - MACB_BIT(ISR_ROVR) | - MACB_BIT(HRESP)); - - /* Disable Receiver and Transmitter */ - ctl = macb_readl(lp, NCR); - macb_writel(lp, NCR, ctl & ~(MACB_BIT(TE) | MACB_BIT(RE))); - - /* Free resources. */ - at91ether_free_coherent(lp); -} - -/* Open the ethernet interface */ -static int at91ether_open(struct net_device *dev) -{ - struct macb *lp = netdev_priv(dev); - u32 ctl; - int ret; - - ret = pm_runtime_resume_and_get(&lp->pdev->dev); - if (ret < 0) - return ret; - - /* Clear internal statistics */ - ctl = macb_readl(lp, NCR); - macb_writel(lp, NCR, ctl | MACB_BIT(CLRSTAT)); - - macb_set_hwaddr(lp); - - ret = at91ether_start(lp); - if (ret) - goto pm_exit; - - ret = macb_phylink_connect(lp); - if (ret) - goto stop; - - netif_start_queue(dev); - - return 0; - -stop: - at91ether_stop(lp); -pm_exit: - pm_runtime_put_sync(&lp->pdev->dev); - return ret; -} - -/* Close the interface */ -static int at91ether_close(struct net_device *dev) -{ - struct macb *lp = netdev_priv(dev); - - netif_stop_queue(dev); - - phylink_stop(lp->phylink); - phylink_disconnect_phy(lp->phylink); - - at91ether_stop(lp); - - return pm_runtime_put(&lp->pdev->dev); -} - -/* Transmit packet */ -static netdev_tx_t at91ether_start_xmit(struct sk_buff *skb, - struct net_device *dev) -{ - struct macb *lp = netdev_priv(dev); - - if (macb_readl(lp, TSR) & MACB_BIT(RM9200_BNQ)) { - int desc = 0; - - netif_stop_queue(dev); - - /* Store packet information (to free when Tx completed) */ - lp->rm9200_txq[desc].skb = skb; - lp->rm9200_txq[desc].size = skb->len; - lp->rm9200_txq[desc].mapping = dma_map_single(&lp->pdev->dev, skb->data, - skb->len, DMA_TO_DEVICE); - if (dma_mapping_error(&lp->pdev->dev, lp->rm9200_txq[desc].mapping)) { - dev_kfree_skb_any(skb); - dev->stats.tx_dropped++; - netdev_err(dev, "%s: DMA mapping error\n", __func__); - return NETDEV_TX_OK; - } - - /* Set address of the data in the Transmit Address register */ - macb_writel(lp, TAR, lp->rm9200_txq[desc].mapping); - /* Set length of the packet in the Transmit Control register */ - macb_writel(lp, TCR, skb->len); - - } else { - netdev_err(dev, "%s called, but device is busy!\n", __func__); - return NETDEV_TX_BUSY; - } - - return NETDEV_TX_OK; -} - -/* Extract received frame from buffer descriptors and sent to upper layers. - * (Called from interrupt context) - */ -static void at91ether_rx(struct net_device *dev) -{ - struct macb *lp = netdev_priv(dev); - struct macb_queue *q = &lp->queues[0]; - struct macb_dma_desc *desc; - unsigned char *p_recv; - struct sk_buff *skb; - unsigned int pktlen; - - desc = macb_rx_desc(q, q->rx_tail); - while (desc->addr & MACB_BIT(RX_USED)) { - p_recv = q->rx_buffers + q->rx_tail * AT91ETHER_MAX_RBUFF_SZ; - pktlen = MACB_BF(RX_FRMLEN, desc->ctrl); - skb = netdev_alloc_skb(dev, pktlen + 2); - if (skb) { - skb_reserve(skb, 2); - skb_put_data(skb, p_recv, pktlen); - - skb->protocol = eth_type_trans(skb, dev); - dev->stats.rx_packets++; - dev->stats.rx_bytes += pktlen; - netif_rx(skb); - } else { - dev->stats.rx_dropped++; - } - - if (desc->ctrl & MACB_BIT(RX_MHASH_MATCH)) - dev->stats.multicast++; - - /* reset ownership bit */ - desc->addr &= ~MACB_BIT(RX_USED); - - /* wrap after last buffer */ - if (q->rx_tail == AT91ETHER_MAX_RX_DESCR - 1) - q->rx_tail = 0; - else - q->rx_tail++; - - desc = macb_rx_desc(q, q->rx_tail); - } -} - -/* MAC interrupt handler */ -static irqreturn_t at91ether_interrupt(int irq, void *dev_id) -{ - struct net_device *dev = dev_id; - struct macb *lp = netdev_priv(dev); - u32 intstatus, ctl; - unsigned int desc; - - /* MAC Interrupt Status register indicates what interrupts are pending. - * It is automatically cleared once read. - */ - intstatus = macb_readl(lp, ISR); - - /* Receive complete */ - if (intstatus & MACB_BIT(RCOMP)) - at91ether_rx(dev); - - /* Transmit complete */ - if (intstatus & MACB_BIT(TCOMP)) { - /* The TCOM bit is set even if the transmission failed */ - if (intstatus & (MACB_BIT(ISR_TUND) | MACB_BIT(ISR_RLE))) - dev->stats.tx_errors++; - - desc = 0; - if (lp->rm9200_txq[desc].skb) { - dev_consume_skb_irq(lp->rm9200_txq[desc].skb); - lp->rm9200_txq[desc].skb = NULL; - dma_unmap_single(&lp->pdev->dev, lp->rm9200_txq[desc].mapping, - lp->rm9200_txq[desc].size, DMA_TO_DEVICE); - dev->stats.tx_packets++; - dev->stats.tx_bytes += lp->rm9200_txq[desc].size; - } - netif_wake_queue(dev); - } - - /* Work-around for EMAC Errata section 41.3.1 */ - if (intstatus & MACB_BIT(RXUBR)) { - ctl = macb_readl(lp, NCR); - macb_writel(lp, NCR, ctl & ~MACB_BIT(RE)); - wmb(); - macb_writel(lp, NCR, ctl | MACB_BIT(RE)); - } - - if (intstatus & MACB_BIT(ISR_ROVR)) - netdev_err(dev, "ROVR error\n"); - - return IRQ_HANDLED; -} - -#ifdef CONFIG_NET_POLL_CONTROLLER -static void at91ether_poll_controller(struct net_device *dev) -{ - unsigned long flags; - - local_irq_save(flags); - at91ether_interrupt(dev->irq, dev); - local_irq_restore(flags); -} -#endif - -static const struct net_device_ops at91ether_netdev_ops = { - .ndo_open = at91ether_open, - .ndo_stop = at91ether_close, - .ndo_start_xmit = at91ether_start_xmit, - .ndo_get_stats = macb_get_stats, - .ndo_set_rx_mode = macb_set_rx_mode, - .ndo_set_mac_address = eth_mac_addr, - .ndo_eth_ioctl = macb_ioctl, - .ndo_validate_addr = eth_validate_addr, -#ifdef CONFIG_NET_POLL_CONTROLLER - .ndo_poll_controller = at91ether_poll_controller, -#endif -}; - -static int at91ether_clk_init(struct platform_device *pdev, struct clk **pclk, - struct clk **hclk, struct clk **tx_clk, - struct clk **rx_clk, struct clk **tsu_clk) -{ - int err; - - *hclk = NULL; - *tx_clk = NULL; - *rx_clk = NULL; - *tsu_clk = NULL; - - *pclk = devm_clk_get(&pdev->dev, "ether_clk"); - if (IS_ERR(*pclk)) - return PTR_ERR(*pclk); - - err = clk_prepare_enable(*pclk); - if (err) { - dev_err(&pdev->dev, "failed to enable pclk (%d)\n", err); - return err; - } - - return 0; -} - -static int at91ether_init(struct platform_device *pdev) -{ - struct net_device *dev = platform_get_drvdata(pdev); - struct macb *bp = netdev_priv(dev); - int err; - - bp->queues[0].bp = bp; - - dev->netdev_ops = &at91ether_netdev_ops; - dev->ethtool_ops = &macb_ethtool_ops; - - err = devm_request_irq(&pdev->dev, dev->irq, at91ether_interrupt, - 0, dev->name, dev); - if (err) - return err; - - macb_writel(bp, NCR, 0); - - macb_writel(bp, NCFGR, MACB_BF(CLK, MACB_CLK_DIV32) | MACB_BIT(BIG)); - - return 0; -} - -static unsigned long fu540_macb_tx_recalc_rate(struct clk_hw *hw, - unsigned long parent_rate) -{ - return mgmt->rate; -} - -static long fu540_macb_tx_round_rate(struct clk_hw *hw, unsigned long rate, - unsigned long *parent_rate) -{ - if (WARN_ON(rate < 2500000)) - return 2500000; - else if (rate == 2500000) - return 2500000; - else if (WARN_ON(rate < 13750000)) - return 2500000; - else if (WARN_ON(rate < 25000000)) - return 25000000; - else if (rate == 25000000) - return 25000000; - else if (WARN_ON(rate < 75000000)) - return 25000000; - else if (WARN_ON(rate < 125000000)) - return 125000000; - else if (rate == 125000000) - return 125000000; - - WARN_ON(rate > 125000000); - - return 125000000; -} - -static int fu540_macb_tx_set_rate(struct clk_hw *hw, unsigned long rate, - unsigned long parent_rate) -{ - rate = fu540_macb_tx_round_rate(hw, rate, &parent_rate); - if (rate != 125000000) - iowrite32(1, mgmt->reg); - else - iowrite32(0, mgmt->reg); - mgmt->rate = rate; - - return 0; -} - -static const struct clk_ops fu540_c000_ops = { - .recalc_rate = fu540_macb_tx_recalc_rate, - .round_rate = fu540_macb_tx_round_rate, - .set_rate = fu540_macb_tx_set_rate, -}; - -static int fu540_c000_clk_init(struct platform_device *pdev, struct clk **pclk, - struct clk **hclk, struct clk **tx_clk, - struct clk **rx_clk, struct clk **tsu_clk) -{ - struct clk_init_data init; - int err = 0; - - err = macb_clk_init(pdev, pclk, hclk, tx_clk, rx_clk, tsu_clk); - if (err) - return err; - - mgmt = devm_kzalloc(&pdev->dev, sizeof(*mgmt), GFP_KERNEL); - if (!mgmt) { - err = -ENOMEM; - goto err_disable_clks; - } - - init.name = "sifive-gemgxl-mgmt"; - init.ops = &fu540_c000_ops; - init.flags = 0; - init.num_parents = 0; - - mgmt->rate = 0; - mgmt->hw.init = &init; - - *tx_clk = devm_clk_register(&pdev->dev, &mgmt->hw); - if (IS_ERR(*tx_clk)) { - err = PTR_ERR(*tx_clk); - goto err_disable_clks; - } - - err = clk_prepare_enable(*tx_clk); - if (err) { - dev_err(&pdev->dev, "failed to enable tx_clk (%u)\n", err); - *tx_clk = NULL; - goto err_disable_clks; - } else { - dev_info(&pdev->dev, "Registered clk switch '%s'\n", init.name); - } - - return 0; - -err_disable_clks: - macb_clks_disable(*pclk, *hclk, *tx_clk, *rx_clk, *tsu_clk); - - return err; -} - -static int fu540_c000_init(struct platform_device *pdev) -{ - mgmt->reg = devm_platform_ioremap_resource(pdev, 1); - if (IS_ERR(mgmt->reg)) - return PTR_ERR(mgmt->reg); - - return macb_init(pdev); -} - -static int init_reset_optional(struct platform_device *pdev) -{ - struct net_device *dev = platform_get_drvdata(pdev); - struct macb *bp = netdev_priv(dev); - int ret; - - if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) { - /* Ensure PHY device used in SGMII mode is ready */ - bp->sgmii_phy = devm_phy_optional_get(&pdev->dev, NULL); - - if (IS_ERR(bp->sgmii_phy)) - return dev_err_probe(&pdev->dev, PTR_ERR(bp->sgmii_phy), - "failed to get SGMII PHY\n"); - - ret = phy_init(bp->sgmii_phy); - if (ret) - return dev_err_probe(&pdev->dev, ret, - "failed to init SGMII PHY\n"); - - ret = zynqmp_pm_is_function_supported(PM_IOCTL, IOCTL_SET_GEM_CONFIG); - if (!ret) { - u32 pm_info[2]; - - ret = of_property_read_u32_array(pdev->dev.of_node, "power-domains", - pm_info, ARRAY_SIZE(pm_info)); - if (ret) { - dev_err(&pdev->dev, "Failed to read power management information\n"); - goto err_out_phy_exit; - } - ret = zynqmp_pm_set_gem_config(pm_info[1], GEM_CONFIG_FIXED, 0); - if (ret) - goto err_out_phy_exit; - - ret = zynqmp_pm_set_gem_config(pm_info[1], GEM_CONFIG_SGMII_MODE, 1); - if (ret) - goto err_out_phy_exit; - } - - } - - /* Fully reset controller at hardware level if mapped in device tree */ - ret = device_reset_optional(&pdev->dev); - if (ret) { - phy_exit(bp->sgmii_phy); - return dev_err_probe(&pdev->dev, ret, "failed to reset controller"); - } - - ret = macb_init(pdev); - -err_out_phy_exit: - if (ret) - phy_exit(bp->sgmii_phy); - - return ret; -} - -static const struct macb_usrio_config sama7g5_usrio = { - .mii = 0, - .rmii = 1, - .rgmii = 2, - .refclk = BIT(2), - .hdfctlen = BIT(6), -}; - -static const struct macb_config fu540_c000_config = { - .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_JUMBO | - MACB_CAPS_GEM_HAS_PTP, - .dma_burst_length = 16, - .clk_init = fu540_c000_clk_init, - .init = fu540_c000_init, - .jumbo_max_len = 10240, - .usrio = &macb_default_usrio, -}; - -static const struct macb_config at91sam9260_config = { - .caps = MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, - .clk_init = macb_clk_init, - .init = macb_init, - .usrio = &macb_default_usrio, -}; - -static const struct macb_config sama5d3macb_config = { - .caps = MACB_CAPS_SG_DISABLED | - MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, - .clk_init = macb_clk_init, - .init = macb_init, - .usrio = &macb_default_usrio, -}; - -static const struct macb_config pc302gem_config = { - .caps = MACB_CAPS_SG_DISABLED | MACB_CAPS_GIGABIT_MODE_AVAILABLE, - .dma_burst_length = 16, - .clk_init = macb_clk_init, - .init = macb_init, - .usrio = &macb_default_usrio, -}; - -static const struct macb_config sama5d2_config = { - .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, - .dma_burst_length = 16, - .clk_init = macb_clk_init, - .init = macb_init, - .usrio = &macb_default_usrio, -}; - -static const struct macb_config sama5d29_config = { - .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | MACB_CAPS_GEM_HAS_PTP, - .dma_burst_length = 16, - .clk_init = macb_clk_init, - .init = macb_init, - .usrio = &macb_default_usrio, -}; - -static const struct macb_config sama5d3_config = { - .caps = MACB_CAPS_SG_DISABLED | MACB_CAPS_GIGABIT_MODE_AVAILABLE | - MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | MACB_CAPS_JUMBO, - .dma_burst_length = 16, - .clk_init = macb_clk_init, - .init = macb_init, - .jumbo_max_len = 10240, - .usrio = &macb_default_usrio, -}; - -static const struct macb_config sama5d4_config = { - .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, - .dma_burst_length = 4, - .clk_init = macb_clk_init, - .init = macb_init, - .usrio = &macb_default_usrio, -}; - -static const struct macb_config emac_config = { - .caps = MACB_CAPS_NEEDS_RSTONUBR | MACB_CAPS_MACB_IS_EMAC, - .clk_init = at91ether_clk_init, - .init = at91ether_init, - .usrio = &macb_default_usrio, -}; - -static const struct macb_config np4_config = { - .caps = MACB_CAPS_USRIO_DISABLED, - .clk_init = macb_clk_init, - .init = macb_init, - .usrio = &macb_default_usrio, -}; - -static const struct macb_config zynqmp_config = { - .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | - MACB_CAPS_JUMBO | - MACB_CAPS_GEM_HAS_PTP | MACB_CAPS_BD_RD_PREFETCH, - .dma_burst_length = 16, - .clk_init = macb_clk_init, - .init = init_reset_optional, - .jumbo_max_len = 10240, - .usrio = &macb_default_usrio, -}; - -static const struct macb_config zynq_config = { - .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_NO_GIGABIT_HALF | - MACB_CAPS_NEEDS_RSTONUBR, - .dma_burst_length = 16, - .clk_init = macb_clk_init, - .init = macb_init, - .usrio = &macb_default_usrio, -}; - -static const struct macb_config mpfs_config = { - .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | - MACB_CAPS_JUMBO | - MACB_CAPS_GEM_HAS_PTP, - .dma_burst_length = 16, - .clk_init = macb_clk_init, - .init = init_reset_optional, - .usrio = &macb_default_usrio, - .max_tx_length = 4040, /* Cadence Erratum 1686 */ - .jumbo_max_len = 4040, -}; - -static const struct macb_config sama7g5_gem_config = { - .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_CLK_HW_CHG | - MACB_CAPS_MIIONRGMII | MACB_CAPS_GEM_HAS_PTP, - .dma_burst_length = 16, - .clk_init = macb_clk_init, - .init = macb_init, - .usrio = &sama7g5_usrio, -}; - -static const struct macb_config sama7g5_emac_config = { - .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | - MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_MIIONRGMII | - MACB_CAPS_GEM_HAS_PTP, - .dma_burst_length = 16, - .clk_init = macb_clk_init, - .init = macb_init, - .usrio = &sama7g5_usrio, -}; - -static const struct macb_config versal_config = { - .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_JUMBO | - MACB_CAPS_GEM_HAS_PTP | MACB_CAPS_BD_RD_PREFETCH | MACB_CAPS_NEED_TSUCLK, - .dma_burst_length = 16, - .clk_init = macb_clk_init, - .init = init_reset_optional, - .jumbo_max_len = 10240, - .usrio = &macb_default_usrio, -}; - -static const struct macb_config phytium_gem2p0_config = { - .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | - MACB_CAPS_JUMBO | - MACB_CAPS_GEM_HAS_PTP | - MACB_CAPS_BD_RD_PREFETCH | - MACB_CAPS_SEL_CLK, - .dma_burst_length = 16, - .clk_init = phytium_clk_init, - .init = macb_init, - .jumbo_max_len = 10240, - .sel_clk_hw = phytium_gem2p0_sel_clk, - .usrio = &macb_default_usrio, -}; - -#if defined(CONFIG_OF) -static const struct of_device_id macb_dt_ids[] = { - { .compatible = "cdns,at91sam9260-macb", .data = &at91sam9260_config }, - { .compatible = "cdns,macb" }, - { .compatible = "cdns,np4-macb", .data = &np4_config }, - { .compatible = "cdns,pc302-gem", .data = &pc302gem_config }, - { .compatible = "cdns,gem", .data = &pc302gem_config }, - { .compatible = "cdns,sam9x60-macb", .data = &at91sam9260_config }, - { .compatible = "atmel,sama5d2-gem", .data = &sama5d2_config }, - { .compatible = "atmel,sama5d29-gem", .data = &sama5d29_config }, - { .compatible = "atmel,sama5d3-gem", .data = &sama5d3_config }, - { .compatible = "atmel,sama5d3-macb", .data = &sama5d3macb_config }, - { .compatible = "atmel,sama5d4-gem", .data = &sama5d4_config }, - { .compatible = "cdns,at91rm9200-emac", .data = &emac_config }, - { .compatible = "cdns,emac", .data = &emac_config }, - { .compatible = "cdns,zynqmp-gem", .data = &zynqmp_config}, /* deprecated */ - { .compatible = "cdns,zynq-gem", .data = &zynq_config }, /* deprecated */ - { .compatible = "sifive,fu540-c000-gem", .data = &fu540_c000_config }, - { .compatible = "microchip,mpfs-macb", .data = &mpfs_config }, - { .compatible = "microchip,sama7g5-gem", .data = &sama7g5_gem_config }, - { .compatible = "microchip,sama7g5-emac", .data = &sama7g5_emac_config }, - { .compatible = "xlnx,zynqmp-gem", .data = &zynqmp_config}, - { .compatible = "xlnx,zynq-gem", .data = &zynq_config }, - { .compatible = "xlnx,versal-gem", .data = &versal_config}, - { .compatible = "cdns,phytium-gem-1.0", .data = &phytium_gem1p0_config }, - { .compatible = "cdns,phytium-gem-2.0", .data = &phytium_gem2p0_config }, - { /* sentinel */ } -}; -MODULE_DEVICE_TABLE(of, macb_dt_ids); -#endif /* CONFIG_OF */ - -#ifdef CONFIG_ACPI -static const struct acpi_device_id macb_acpi_ids[] = { - { .id = "PHYT0036", .driver_data = (kernel_ulong_t)&phytium_gem1p0_config }, - { } -}; - -MODULE_DEVICE_TABLE(acpi, macb_acpi_ids); -#else -#define macb_acpi_ids NULL -#endif - -static const struct macb_config default_gem_config = { - .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | - MACB_CAPS_JUMBO | - MACB_CAPS_GEM_HAS_PTP, - .dma_burst_length = 16, - .clk_init = macb_clk_init, - .init = macb_init, - .usrio = &macb_default_usrio, - .jumbo_max_len = 10240, -}; - -static void gem_ncsi_handler(struct ncsi_dev *nd) -{ - if (unlikely(nd->state != ncsi_dev_state_functional)) - return; - - netdev_dbg(nd->dev, "NCSI interface %s\n", - nd->link_up ? "up" : "down"); -} - -static int macb_get_phy_mode(struct platform_device *pdev) -{ - const char *pm; - int err, i; - - err = device_property_read_string(&pdev->dev, "phy-mode", &pm); - if (err < 0) - return err; - - for (i = 0; i < PHY_INTERFACE_MODE_MAX; i++) { - if (!strcasecmp(pm, phy_modes(i))) - return i; - } - - return -ENODEV; -} - -static int macb_probe(struct platform_device *pdev) -{ - const struct macb_config *macb_config = &default_gem_config; - int (*clk_init)(struct platform_device *, struct clk **, - struct clk **, struct clk **, struct clk **, - struct clk **) = macb_config->clk_init; - int (*init)(struct platform_device *) = macb_config->init; - struct device_node *np = pdev->dev.of_node; - struct clk *pclk, *hclk = NULL, *tx_clk = NULL, *rx_clk = NULL; - struct clk *tsu_clk = NULL; - unsigned int queue_mask, num_queues; - bool native_io; - struct net_device *dev; - struct resource *regs; - u32 wtrmrk_rst_val; - void __iomem *mem; - struct macb *bp; - int err, val; - - mem = devm_platform_get_and_ioremap_resource(pdev, 0, ®s); - if (IS_ERR(mem)) - return PTR_ERR(mem); - - if (np) { - const struct of_device_id *match; - - match = of_match_node(macb_dt_ids, np); - if (match && match->data) { - macb_config = match->data; - clk_init = macb_config->clk_init; - init = macb_config->init; - } - } else if (has_acpi_companion(&pdev->dev)) { - const struct acpi_device_id *match; - - match = acpi_match_device(macb_acpi_ids, &pdev->dev); - if (match && match->driver_data) { - macb_config = (void *)match->driver_data; - clk_init = macb_config->clk_init; - init = macb_config->init; - } - } - - err = clk_init(pdev, &pclk, &hclk, &tx_clk, &rx_clk, &tsu_clk); - if (err) - return err; - - pm_runtime_set_autosuspend_delay(&pdev->dev, MACB_PM_TIMEOUT); - pm_runtime_use_autosuspend(&pdev->dev); - pm_runtime_get_noresume(&pdev->dev); - pm_runtime_set_active(&pdev->dev); - pm_runtime_enable(&pdev->dev); - native_io = hw_is_native_io(mem); - - macb_probe_queues(mem, native_io, &queue_mask, &num_queues); - dev = alloc_etherdev_mq(sizeof(*bp), num_queues); - if (!dev) { - err = -ENOMEM; - goto err_disable_clocks; - } - - dev->base_addr = regs->start; - - SET_NETDEV_DEV(dev, &pdev->dev); - - bp = netdev_priv(dev); - bp->pdev = pdev; - bp->dev = dev; - bp->regs = mem; - bp->native_io = native_io; - if (native_io) { - bp->macb_reg_readl = hw_readl_native; - bp->macb_reg_writel = hw_writel_native; - } else { - bp->macb_reg_readl = hw_readl; - bp->macb_reg_writel = hw_writel; - } - bp->num_queues = num_queues; - bp->queue_mask = queue_mask; - if (macb_config) - bp->dma_burst_length = macb_config->dma_burst_length; - bp->pclk = pclk; - bp->hclk = hclk; - bp->tx_clk = tx_clk; - bp->rx_clk = rx_clk; - bp->tsu_clk = tsu_clk; - if (macb_config) - bp->jumbo_max_len = macb_config->jumbo_max_len; - - if (macb_config) - bp->sel_clk_hw = macb_config->sel_clk_hw; - - if (!hw_is_gem(bp->regs, bp->native_io)) - bp->max_tx_length = MACB_MAX_TX_LEN; - else if (macb_config->max_tx_length) - bp->max_tx_length = macb_config->max_tx_length; - else - bp->max_tx_length = GEM_MAX_TX_LEN; - - bp->wol = 0; - if (device_property_read_bool(&pdev->dev, "magic-packet")) - bp->wol |= MACB_WOL_HAS_MAGIC_PACKET; - device_set_wakeup_capable(&pdev->dev, bp->wol & MACB_WOL_HAS_MAGIC_PACKET); - - bp->usrio = macb_config->usrio; - - /* By default we set to partial store and forward mode for zynqmp. - * Disable if not set in devicetree. - */ - if (GEM_BFEXT(PBUF_CUTTHRU, gem_readl(bp, DCFG6))) { - err = of_property_read_u32(bp->pdev->dev.of_node, - "cdns,rx-watermark", - &bp->rx_watermark); - - if (!err) { - /* Disable partial store and forward in case of error or - * invalid watermark value - */ - wtrmrk_rst_val = (1 << (GEM_BFEXT(RX_PBUF_ADDR, gem_readl(bp, DCFG2)))) - 1; - if (bp->rx_watermark > wtrmrk_rst_val || !bp->rx_watermark) { - dev_info(&bp->pdev->dev, "Invalid watermark value\n"); - bp->rx_watermark = 0; - } - } - } - spin_lock_init(&bp->lock); - spin_lock_init(&bp->stats_lock); - - /* setup capabilities */ - macb_configure_caps(bp, macb_config); - -#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT - if (GEM_BFEXT(DAW64, gem_readl(bp, DCFG6))) { - err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(44)); - if (err) { - dev_err(&pdev->dev, "failed to set DMA mask\n"); - goto err_out_free_netdev; - } - bp->hw_dma_cap |= HW_DMA_CAP_64B; - } -#endif - platform_set_drvdata(pdev, dev); - - dev->irq = platform_get_irq(pdev, 0); - if (dev->irq < 0) { - err = dev->irq; - goto err_out_free_netdev; - } - - /* MTU range: 68 - 1500 or 10240 */ - dev->min_mtu = GEM_MTU_MIN_SIZE; - if ((bp->caps & MACB_CAPS_JUMBO) && bp->jumbo_max_len) - dev->max_mtu = bp->jumbo_max_len - ETH_HLEN - ETH_FCS_LEN; - else - dev->max_mtu = ETH_DATA_LEN; - - if (bp->caps & MACB_CAPS_BD_RD_PREFETCH) { - val = GEM_BFEXT(RXBD_RDBUFF, gem_readl(bp, DCFG10)); - if (val) - bp->rx_bd_rd_prefetch = (2 << (val - 1)) * - macb_dma_desc_get_size(bp); - - val = GEM_BFEXT(TXBD_RDBUFF, gem_readl(bp, DCFG10)); - if (val) - bp->tx_bd_rd_prefetch = (2 << (val - 1)) * - macb_dma_desc_get_size(bp); - } - - bp->rx_intr_mask = MACB_RX_INT_FLAGS; - if (bp->caps & MACB_CAPS_NEEDS_RSTONUBR) - bp->rx_intr_mask |= MACB_BIT(RXUBR); - - err = of_get_ethdev_address(np, bp->dev); - if (err == -EPROBE_DEFER) - goto err_out_free_netdev; - else if (err) - macb_get_hwaddr(bp); - - err = macb_get_phy_mode(pdev); - if (err < 0) - bp->phy_interface = PHY_INTERFACE_MODE_MII; - else - bp->phy_interface = err; - - - /* IP specific init */ - err = init(pdev); - if (err) - goto err_out_free_netdev; - - err = macb_mii_init(bp); - if (err) - goto err_out_phy_exit; - - if (device_property_read_bool(&pdev->dev, "use-ncsi")) { - if (!IS_ENABLED(CONFIG_NET_NCSI)) { - dev_err(&pdev->dev, "NCSI stack not enabled\n"); - goto err_out_free_netdev; - } - dev_notice(&pdev->dev, "Using NCSI interface\n"); - bp->use_ncsi = 1; - bp->ndev = ncsi_register_dev(dev, gem_ncsi_handler); - if (!bp->ndev) - goto err_out_free_netdev; - } else { - bp->use_ncsi = 0; - } - - netif_carrier_off(dev); - - err = register_netdev(dev); - if (err) { - dev_err(&pdev->dev, "Cannot register net device, aborting.\n"); - goto err_out_unregister_mdio; - } - - tasklet_setup(&bp->hresp_err_tasklet, macb_hresp_error_task); - - netdev_info(dev, "Cadence %s rev 0x%08x at 0x%08lx irq %d (%pM)\n", - macb_is_gem(bp) ? "GEM" : "MACB", macb_readl(bp, MID), - dev->base_addr, dev->irq, dev->dev_addr); - - pm_runtime_mark_last_busy(&bp->pdev->dev); - pm_runtime_put_autosuspend(&bp->pdev->dev); - - return 0; - -err_out_unregister_mdio: - mdiobus_unregister(bp->mii_bus); - mdiobus_free(bp->mii_bus); - -err_out_phy_exit: - phy_exit(bp->sgmii_phy); - -err_out_free_netdev: - free_netdev(dev); - -err_disable_clocks: - macb_clks_disable(pclk, hclk, tx_clk, rx_clk, tsu_clk); - pm_runtime_disable(&pdev->dev); - pm_runtime_set_suspended(&pdev->dev); - pm_runtime_dont_use_autosuspend(&pdev->dev); - - return err; -} - -static int macb_remove(struct platform_device *pdev) -{ - struct net_device *dev; - struct macb *bp; - - dev = platform_get_drvdata(pdev); - - if (dev) { - bp = netdev_priv(dev); - unregister_netdev(dev); - phy_exit(bp->sgmii_phy); - unregister_netdev(dev); - mdiobus_unregister(bp->mii_bus); - mdiobus_free(bp->mii_bus); - - tasklet_kill(&bp->hresp_err_tasklet); - pm_runtime_disable(&pdev->dev); - pm_runtime_dont_use_autosuspend(&pdev->dev); - if (!pm_runtime_suspended(&pdev->dev)) { - if (__clk_is_enabled(bp->pclk)) - clk_disable_unprepare(bp->pclk); - if (__clk_is_enabled(bp->hclk)) - clk_disable_unprepare(bp->hclk); - if (__clk_is_enabled(bp->tx_clk)) - clk_disable_unprepare(bp->tx_clk); - if (__clk_is_enabled(bp->rx_clk)) - clk_disable_unprepare(bp->rx_clk); - if (__clk_is_enabled(bp->tsu_clk)) - clk_disable_unprepare(bp->tsu_clk); - pm_runtime_set_suspended(&pdev->dev); - } - phylink_destroy(bp->phylink); - free_netdev(dev); - } - - return 0; -} - -static int __maybe_unused macb_suspend(struct device *dev) -{ - struct net_device *netdev = dev_get_drvdata(dev); - struct macb *bp = netdev_priv(netdev); - struct macb_queue *queue; - unsigned long flags; - unsigned int q; - int err; - u32 tmp; - - if (!device_may_wakeup(&bp->dev->dev)) - phy_exit(bp->sgmii_phy); - - if (!netif_running(netdev)) - return 0; - - if (bp->wol & MACB_WOL_ENABLED) { - spin_lock_irqsave(&bp->lock, flags); - - /* Disable Tx and Rx engines before disabling the queues, - * this is mandatory as per the IP spec sheet - */ - tmp = macb_readl(bp, NCR); - macb_writel(bp, NCR, tmp & ~(MACB_BIT(TE) | MACB_BIT(RE))); - for (q = 0, queue = bp->queues; q < bp->num_queues; - ++q, ++queue) { - /* Disable RX queues */ - if (bp->caps & MACB_CAPS_QUEUE_DISABLE) { - queue_writel(queue, RBQP, MACB_BIT(QUEUE_DISABLE)); - } else { - /* Tie off RX queues */ - queue_writel(queue, RBQP, - lower_32_bits(bp->rx_ring_tieoff_dma)); -#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT - queue_writel(queue, RBQPH, - upper_32_bits(bp->rx_ring_tieoff_dma)); -#endif - } - /* Disable all interrupts */ - queue_writel(queue, IDR, -1); - queue_readl(queue, ISR); - if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) - queue_writel(queue, ISR, -1); - } - /* Enable Receive engine */ - macb_writel(bp, NCR, tmp | MACB_BIT(RE)); - /* Flush all status bits */ - macb_writel(bp, TSR, -1); - macb_writel(bp, RSR, -1); - - spin_unlock_irqrestore(&bp->lock, flags); - - /* Change interrupt handler and - * Enable WoL IRQ on queue 0 - */ - devm_free_irq(dev, bp->queues[0].irq, bp->queues); - if (macb_is_gem(bp)) { - err = devm_request_irq(dev, bp->queues[0].irq, gem_wol_interrupt, - IRQF_SHARED, netdev->name, bp->queues); - if (err) { - dev_err(dev, - "Unable to request IRQ %d (error %d)\n", - bp->queues[0].irq, err); - return err; - } - spin_lock_irqsave(&bp->lock, flags); - queue_writel(bp->queues, IER, GEM_BIT(WOL)); - gem_writel(bp, WOL, MACB_BIT(MAG)); - spin_unlock_irqrestore(&bp->lock, flags); - } else { - err = devm_request_irq(dev, bp->queues[0].irq, macb_wol_interrupt, - IRQF_SHARED, netdev->name, bp->queues); - if (err) { - dev_err(dev, - "Unable to request IRQ %d (error %d)\n", - bp->queues[0].irq, err); - return err; - } - spin_lock_irqsave(&bp->lock, flags); - queue_writel(bp->queues, IER, MACB_BIT(WOL)); - macb_writel(bp, WOL, MACB_BIT(MAG)); - spin_unlock_irqrestore(&bp->lock, flags); - } - - enable_irq_wake(bp->queues[0].irq); - } - - netif_device_detach(netdev); - for (q = 0, queue = bp->queues; q < bp->num_queues; - ++q, ++queue) { - napi_disable(&queue->napi_rx); - napi_disable(&queue->napi_tx); - } - - if (!(bp->wol & MACB_WOL_ENABLED)) { - rtnl_lock(); - phylink_stop(bp->phylink); - rtnl_unlock(); - spin_lock_irqsave(&bp->lock, flags); - macb_reset_hw(bp); - spin_unlock_irqrestore(&bp->lock, flags); - } - - if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) - bp->pm_data.usrio = macb_or_gem_readl(bp, USRIO); - - if (netdev->hw_features & NETIF_F_NTUPLE) - bp->pm_data.scrt2 = gem_readl_n(bp, ETHT, SCRT2_ETHT); - - if (bp->ptp_info) - bp->ptp_info->ptp_remove(netdev); - if (!device_may_wakeup(dev)) - pm_runtime_force_suspend(dev); - - return 0; -} - -static int __maybe_unused macb_resume(struct device *dev) -{ - struct net_device *netdev = dev_get_drvdata(dev); - struct macb *bp = netdev_priv(netdev); - struct macb_queue *queue; - unsigned long flags; - unsigned int q; - int err; - - if (!device_may_wakeup(&bp->dev->dev)) - phy_init(bp->sgmii_phy); - - if (!netif_running(netdev)) - return 0; - - if (!device_may_wakeup(dev)) - pm_runtime_force_resume(dev); - - if (bp->wol & MACB_WOL_ENABLED) { - spin_lock_irqsave(&bp->lock, flags); - /* Disable WoL */ - if (macb_is_gem(bp)) { - queue_writel(bp->queues, IDR, GEM_BIT(WOL)); - gem_writel(bp, WOL, 0); - } else { - queue_writel(bp->queues, IDR, MACB_BIT(WOL)); - macb_writel(bp, WOL, 0); - } - /* Clear ISR on queue 0 */ - queue_readl(bp->queues, ISR); - if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) - queue_writel(bp->queues, ISR, -1); - spin_unlock_irqrestore(&bp->lock, flags); - - /* Replace interrupt handler on queue 0 */ - devm_free_irq(dev, bp->queues[0].irq, bp->queues); - err = devm_request_irq(dev, bp->queues[0].irq, macb_interrupt, - IRQF_SHARED, netdev->name, bp->queues); - if (err) { - dev_err(dev, - "Unable to request IRQ %d (error %d)\n", - bp->queues[0].irq, err); - return err; - } - - disable_irq_wake(bp->queues[0].irq); - - /* Now make sure we disable phy before moving - * to common restore path - */ - rtnl_lock(); - phylink_stop(bp->phylink); - rtnl_unlock(); - } - - if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC)) - macb_init_buffers(bp); - - for (q = 0, queue = bp->queues; q < bp->num_queues; - ++q, ++queue) { - if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC)) { - if (macb_is_gem(bp)) - gem_init_rx_ring(queue); - else - macb_init_rx_ring(queue); - } - - napi_enable(&queue->napi_rx); - napi_enable(&queue->napi_tx); - } - - if (netdev->hw_features & NETIF_F_NTUPLE) - gem_writel_n(bp, ETHT, SCRT2_ETHT, bp->pm_data.scrt2); - - if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) - macb_or_gem_writel(bp, USRIO, bp->pm_data.usrio); - - macb_writel(bp, NCR, MACB_BIT(MPE)); - macb_init_hw(bp); - macb_set_rx_mode(netdev); - macb_restore_features(bp); - rtnl_lock(); - phylink_start(bp->phylink); - rtnl_unlock(); - - netif_device_attach(netdev); - if (bp->ptp_info) - bp->ptp_info->ptp_init(netdev); - - return 0; -} - -static int __maybe_unused macb_runtime_suspend(struct device *dev) -{ - struct net_device *netdev = dev_get_drvdata(dev); - struct macb *bp = netdev_priv(netdev); - - if (!(device_may_wakeup(dev))) { - if (__clk_is_enabled(bp->pclk)) - clk_disable_unprepare(bp->pclk); - if (__clk_is_enabled(bp->hclk)) - clk_disable_unprepare(bp->hclk); - if (__clk_is_enabled(bp->tx_clk)) - clk_disable_unprepare(bp->tx_clk); - if (__clk_is_enabled(bp->rx_clk)) - clk_disable_unprepare(bp->rx_clk); - if (__clk_is_enabled(bp->tsu_clk)) - clk_disable_unprepare(bp->tsu_clk); - } else if (!(bp->caps & MACB_CAPS_NEED_TSUCLK)) { - if (__clk_is_enabled(bp->tsu_clk)) - clk_disable_unprepare(bp->tsu_clk); - } - - return 0; -} - -static int __maybe_unused macb_runtime_resume(struct device *dev) -{ - struct net_device *netdev = dev_get_drvdata(dev); - struct macb *bp = netdev_priv(netdev); - - if (!(device_may_wakeup(dev))) { - clk_prepare_enable(bp->pclk); - clk_prepare_enable(bp->hclk); - clk_prepare_enable(bp->tx_clk); - clk_prepare_enable(bp->rx_clk); - clk_prepare_enable(bp->tsu_clk); - } else if (!(bp->caps & MACB_CAPS_NEED_TSUCLK)) { - clk_prepare_enable(bp->tsu_clk); - } - - return 0; -} - -static const struct dev_pm_ops macb_pm_ops = { - SET_SYSTEM_SLEEP_PM_OPS(macb_suspend, macb_resume) - SET_RUNTIME_PM_OPS(macb_runtime_suspend, macb_runtime_resume, NULL) -}; - -static struct platform_driver macb_driver = { - .probe = macb_probe, - .remove = macb_remove, - .driver = { - .name = "macb", - .of_match_table = of_match_ptr(macb_dt_ids), - .acpi_match_table = macb_acpi_ids, - .pm = &macb_pm_ops, - }, -}; - -module_platform_driver(macb_driver); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("Cadence MACB/GEM Ethernet driver"); -MODULE_AUTHOR("Haavard Skinnemoen (Atmel)"); -MODULE_ALIAS("platform:macb"); diff --git a/drivers/net/ethernet/phytium/phytmac.h b/drivers/net/ethernet/phytium/phytmac.h index 2fd43be183c95..82f24410eb9e9 100644 --- a/drivers/net/ethernet/phytium/phytmac.h +++ b/drivers/net/ethernet/phytium/phytmac.h @@ -17,7 +17,7 @@ #define PHYTMAC_PCI_DRV_NAME "phytmac_pci" #define PHYTMAC_PLAT_DRV_NAME "phytmac_platform" #define PHYTMAC_DRV_DESC "PHYTIUM Ethernet Driver" -#define PHYTMAC_DRIVER_VERSION "1.0.54" +#define PHYTMAC_DRIVER_VERSION "1.0.55" #define PHYTMAC_DEFAULT_MSG_ENABLE \ (NETIF_MSG_DRV | \ NETIF_MSG_PROBE | \ @@ -30,6 +30,14 @@ #define IRQ_TYPE_MSI 1 #define IRQ_TYPE_INTX 2 +#define PHY_INIT_SMC_ID 0xc2000015 +#define PHY_SLOT(i) ((i) * 4) +#define PHY_INIT(i) (0x1U << (i)) +#define PHY_MULTIPLEX_GMAC(i) (0x2U << PHY_SLOT(i)) +#define PHY_MODE_SGMII(i) (0x3U << PHY_SLOT(i)) +#define PHY_SPEED_1000(i) (0x3U << PHY_SLOT(i)) +#define PHY_SPEED_2500(i) (0x4U << PHY_SLOT(i)) + #define PHYTMAC_MAX_QUEUES 8 #define DEFAULT_DMA_BURST_LENGTH 16 #define DEFAULT_JUMBO_MAX_LENGTH 10240 @@ -182,6 +190,13 @@ struct packet_info { u32 seq; }; +struct phyinit_cfg { + u64 phy_domain; + u64 phy_multiplex; + u64 phy_mode; + u64 phy_speed; +}; + #define DEV_TYPE_PLATFORM 0 #define DEV_TYPE_PCI 1 @@ -494,6 +509,7 @@ struct phytmac { int pause; phy_interface_t phy_interface; enum phytmac_interface phytmac_v2_interface; + int ori_speed; int speed; int duplex; int autoneg; @@ -572,6 +588,7 @@ struct phytmac_hw_if { int (*del_fdir_entry)(struct phytmac *pdata, struct ethtool_rx_flow_spec *rx_flow); /* mido ops */ + void (*phy_speed_switch)(struct phytmac *pdata, int speed); int (*enable_mdio_control)(struct phytmac *pdata, int enable); int (*mdio_read)(struct phytmac *pdata, int mii_id, int regnum); int (*mdio_write)(struct phytmac *pdata, int mii_id, diff --git a/drivers/net/ethernet/phytium/phytmac_main.c b/drivers/net/ethernet/phytium/phytmac_main.c index 1f4e9a2b97ee4..70611ef867f47 100644 --- a/drivers/net/ethernet/phytium/phytmac_main.c +++ b/drivers/net/ethernet/phytium/phytmac_main.c @@ -1852,6 +1852,36 @@ static int phytmac_phylink_connect(struct phytmac *pdata) return 0; } +static void phytmac_sgmii_speed_switch(struct phytmac *pdata, int speed) +{ + struct phytmac_hw_if *hw_if = pdata->hw_if; + int ori_speed = pdata->ori_speed; + + pdata->ori_speed = speed; + + if (pdata->phy_interface != PHY_INTERFACE_MODE_SGMII) + goto out; + + if (pdata->version != VERSION_V0) + goto out; + + if (ori_speed == speed) + goto out; + + /* Reject switching between 1G/100M/10M, since these share the same + * internal PHY speed mode. + */ + if ((ori_speed == SPEED_1000 || ori_speed == SPEED_100 || ori_speed == SPEED_10) && + (speed == SPEED_1000 || speed == SPEED_100 || speed == SPEED_10)) + goto out; + + if (hw_if->phy_speed_switch) + hw_if->phy_speed_switch(pdata, speed); + +out: + return; +} + int phytmac_pcs_config(struct phylink_pcs *pcs, unsigned int mode, phy_interface_t interface, const unsigned long *advertising, @@ -1979,7 +2009,10 @@ static void phytmac_mac_link_up(struct phylink_config *config, spin_lock_irqsave(&pdata->lock, flags); - hw_if->mac_linkup(pdata, interface, speed, duplex); + if (phy) + phytmac_sgmii_speed_switch(pdata, speed); + + hw_if->mac_linkup(pdata, pdata->phy_interface, speed, duplex); if (rx_pause != pdata->pause) { hw_if->enable_pause(pdata, rx_pause); diff --git a/drivers/net/ethernet/phytium/phytmac_v1.c b/drivers/net/ethernet/phytium/phytmac_v1.c index befb3e83e56e8..29eb6274c4b09 100644 --- a/drivers/net/ethernet/phytium/phytmac_v1.c +++ b/drivers/net/ethernet/phytium/phytmac_v1.c @@ -191,6 +191,52 @@ static int phytmac_pcs_software_reset(struct phytmac *pdata, int reset) return 0; } +static void phytmac_phy_speed_switch(struct phytmac *pdata, int speed) +{ + struct arm_smccc_res res; + struct phyinit_cfg cfg; + int id; + + switch (pdata->ndev->base_addr) { + case MAC0_ADDR_BASE: + id = 0; + break; + case MAC1_ADDR_BASE: + id = 1; + break; + default: + return; + } + + switch (speed) { + case SPEED_1000: + case SPEED_100: + case SPEED_10: + cfg.phy_speed = PHY_SPEED_1000(id); + break; + case SPEED_2500: + cfg.phy_speed = PHY_SPEED_2500(id); + break; + default: + return; + } + + cfg.phy_domain = PHY_INIT(id); + cfg.phy_multiplex = PHY_MULTIPLEX_GMAC(id); + cfg.phy_mode = PHY_MODE_SGMII(id); + + arm_smccc_smc(PHY_INIT_SMC_ID, cfg.phy_domain, cfg.phy_multiplex, + cfg.phy_mode, cfg.phy_speed, 0, 0, 0, &res); + if (res.a0 == SMCCC_RET_NOT_SUPPORTED) { + netdev_warn(pdata->ndev, "PHY_INIT_SMC_ID not implemented, skipping ....\n"); + goto out; + } + netdev_info(pdata->ndev, "switch speed to %dMb/s successfully\n", speed); + +out: + return; +} + static int phytmac_mac_linkup(struct phytmac *pdata, phy_interface_t interface, int speed, int duplex) { @@ -230,6 +276,8 @@ static int phytmac_mac_linkup(struct phytmac *pdata, phy_interface_t interface, if (interface == PHY_INTERFACE_MODE_SGMII) { if (speed == SPEED_2500) phytmac_enable_autoneg(pdata, 0); + else + phytmac_enable_autoneg(pdata, 1); } return 0; @@ -1432,6 +1480,7 @@ struct phytmac_hw_if phytmac_1p0_hw = { .get_stats = phytmac_get_hw_stats, .set_mac_address = phytmac_set_mac_addr, .get_mac_address = phytmac_get_mac_addr, + .phy_speed_switch = phytmac_phy_speed_switch, .mdio_idle = phytmac_mdio_idle, .mdio_read = phytmac_mdio_data_read_c22, .mdio_write = phytmac_mdio_data_write_c22, diff --git a/drivers/net/ethernet/phytium/phytmac_v1.h b/drivers/net/ethernet/phytium/phytmac_v1.h index e0ca0ab835474..603c96b55cc42 100644 --- a/drivers/net/ethernet/phytium/phytmac_v1.h +++ b/drivers/net/ethernet/phytium/phytmac_v1.h @@ -9,6 +9,9 @@ extern struct phytmac_hw_if phytmac_1p0_hw; #define PHYTMAC_FRAME_MASK 0x1fff #define PHYTMAC_JUMBO_FRAME_MASK 0x3fff +#define MAC0_ADDR_BASE 0x36ce0000 +#define MAC1_ADDR_BASE 0x36ce2000 + #define PHYTMAC_SPEED_100M 0 #define PHYTMAC_SPEED_1000M 1 #define PHYTMAC_SPEED_2500M 2 diff --git a/drivers/net/ethernet/phytium/phytmac_v2.c b/drivers/net/ethernet/phytium/phytmac_v2.c index 8dd3ecbec098d..98ed55421d623 100644 --- a/drivers/net/ethernet/phytium/phytmac_v2.c +++ b/drivers/net/ethernet/phytium/phytmac_v2.c @@ -929,6 +929,8 @@ static int phytmac_v2_interface_linkup(struct phytmac *pdata, phy_interface_t in if (interface == PHY_INTERFACE_MODE_SGMII) { if (speed == SPEED_2500) pdata->autoneg = 0; + else + pdata->autoneg = 1; } memset(¶, 0, sizeof(para)); From 99078ca00ee1dc469a247ccb6805e1f0c1bf2a1a Mon Sep 17 00:00:00 2001 From: Ma Mingrui Date: Sat, 9 May 2026 14:41:36 +0800 Subject: [PATCH 10/13] net/phytmac: Fix compilation errors caused by disabling CONFIG_ACPI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit isabling CONFIG_ACPI will cause errors like 'error: implicit declaration of function ‘acpi_evaluate_integer’, so it is necessary to add #ifdef CONFIG_ACPI for protection. Signed-off-by: Ma Mingrui Signed-off-by: Li Tongfeng Signed-off-by: Wang Yinfeng --- drivers/net/ethernet/phytium/phytmac.h | 2 +- drivers/net/ethernet/phytium/phytmac_v1.c | 20 ++++++++++++-------- drivers/net/ethernet/phytium/phytmac_v2.c | 19 +++++++++++-------- 3 files changed, 24 insertions(+), 17 deletions(-) diff --git a/drivers/net/ethernet/phytium/phytmac.h b/drivers/net/ethernet/phytium/phytmac.h index 82f24410eb9e9..894e4053cd107 100644 --- a/drivers/net/ethernet/phytium/phytmac.h +++ b/drivers/net/ethernet/phytium/phytmac.h @@ -17,7 +17,7 @@ #define PHYTMAC_PCI_DRV_NAME "phytmac_pci" #define PHYTMAC_PLAT_DRV_NAME "phytmac_platform" #define PHYTMAC_DRV_DESC "PHYTIUM Ethernet Driver" -#define PHYTMAC_DRIVER_VERSION "1.0.55" +#define PHYTMAC_DRIVER_VERSION "1.0.56" #define PHYTMAC_DEFAULT_MSG_ENABLE \ (NETIF_MSG_DRV | \ NETIF_MSG_PROBE | \ diff --git a/drivers/net/ethernet/phytium/phytmac_v1.c b/drivers/net/ethernet/phytium/phytmac_v1.c index 29eb6274c4b09..372544d6304bc 100644 --- a/drivers/net/ethernet/phytium/phytmac_v1.c +++ b/drivers/net/ethernet/phytium/phytmac_v1.c @@ -16,6 +16,7 @@ #include #include #include +#include #include "phytmac.h" #include "phytmac_v1.h" @@ -505,14 +506,6 @@ static int phytmac_powerup_hw(struct phytmac *pdata, int on) { u32 status, data0, data1, rdata1; int ret; - acpi_handle handle; - union acpi_object args[3]; - struct acpi_object_list arg_list = { - .pointer = args, - .count = ARRAY_SIZE(args), - }; - acpi_status acpi_sts; - unsigned long long rv; if (!(pdata->capacities & PHYTMAC_CAPS_PWCTRL)) { pdata->power_state = on; @@ -520,6 +513,16 @@ static int phytmac_powerup_hw(struct phytmac *pdata, int on) } if (has_acpi_companion(pdata->dev)) { +#ifdef CONFIG_ACPI + acpi_handle handle; + union acpi_object args[3]; + struct acpi_object_list arg_list = { + .pointer = args, + .count = ARRAY_SIZE(args), + }; + acpi_status acpi_sts; + unsigned long long rv; + handle = ACPI_HANDLE(pdata->dev); netdev_info(pdata->ndev, "set gmac power %s\n", @@ -544,6 +547,7 @@ static int phytmac_powerup_hw(struct phytmac *pdata, int on) if (rv) netdev_err(pdata->ndev, "Failed to power off\n"); } +#endif } else { ret = readx_poll_timeout(PHYTMAC_READ_STAT, pdata, status, !status, 1, PHYTMAC_TIMEOUT); diff --git a/drivers/net/ethernet/phytium/phytmac_v2.c b/drivers/net/ethernet/phytium/phytmac_v2.c index 98ed55421d623..46b1e5ce20a04 100644 --- a/drivers/net/ethernet/phytium/phytmac_v2.c +++ b/drivers/net/ethernet/phytium/phytmac_v2.c @@ -536,14 +536,6 @@ static int phytmac_v2_powerup_hw(struct phytmac *pdata, int on) { u32 status, data0, data1, rdata1; int ret; - acpi_handle handle; - union acpi_object args[3]; - struct acpi_object_list arg_list = { - .pointer = args, - .count = ARRAY_SIZE(args), - }; - acpi_status acpi_sts; - unsigned long long rv; if (!(pdata->capacities & PHYTMAC_CAPS_PWCTRL)) { pdata->power_state = on; @@ -551,6 +543,16 @@ static int phytmac_v2_powerup_hw(struct phytmac *pdata, int on) } if (has_acpi_companion(pdata->dev)) { +#ifdef CONFIG_ACPI + acpi_handle handle; + union acpi_object args[3]; + struct acpi_object_list arg_list = { + .pointer = args, + .count = ARRAY_SIZE(args), + }; + acpi_status acpi_sts; + unsigned long long rv; + handle = ACPI_HANDLE(pdata->dev); netdev_info(pdata->ndev, "set gmac power %s\n", @@ -575,6 +577,7 @@ static int phytmac_v2_powerup_hw(struct phytmac *pdata, int on) if (rv) netdev_err(pdata->ndev, "Failed to power off\n"); } +#endif } else { ret = readx_poll_timeout(PHYTMAC_READ_STAT, pdata, status, !status, 1, PHYTMAC_TIMEOUT); From 21962c75b0569c8ea0a70bc2802999510c4e1a6e Mon Sep 17 00:00:00 2001 From: Ma Mingrui Date: Sat, 9 May 2026 14:45:35 +0800 Subject: [PATCH 11/13] char: phytnetled: Fix compilation errors caused by disabling CONFIG_ACPI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Disabling CONFIG_ACPI will cause errors like 'error: implicit declaration of function ‘acpi_get_acpi_dev’', so it is necessary to add #ifdef CONFIG_ACPI for protection. Signed-off-by: Ma Mingrui Signed-off-by: Li Tongfeng Signed-off-by: Wang Yinfeng --- drivers/char/phytnetled/phytnet_led.c | 10 +++++++--- drivers/net/ethernet/phytium/phytmac_v1.c | 14 -------------- 2 files changed, 7 insertions(+), 17 deletions(-) diff --git a/drivers/char/phytnetled/phytnet_led.c b/drivers/char/phytnetled/phytnet_led.c index de02f05eebba5..ce97285af86af 100644 --- a/drivers/char/phytnetled/phytnet_led.c +++ b/drivers/char/phytnetled/phytnet_led.c @@ -137,7 +137,7 @@ of_ndev_init(struct led_data *phytnet_led) return 0; } - +#ifdef CONFIG_ACPI static int acpi_ndev_init(struct led_data *phytnet_led) { @@ -188,6 +188,7 @@ acpi_ndev_init(struct led_data *phytnet_led) return 0; } +#endif static int gpio_init(struct led_data *phytnet_led) @@ -235,10 +236,13 @@ led_init_and_control(struct work_struct *work) int err = -1; struct led_data *phytnet_led = container_of(work, struct led_data, led_control_work.work); - if (phytnet_led->pdev->dev.of_node) + if (phytnet_led->pdev->dev.of_node) { err = of_ndev_init(phytnet_led); - else if (has_acpi_companion(&phytnet_led->pdev->dev)) + } else if (has_acpi_companion(&phytnet_led->pdev->dev)) { +#ifdef CONFIG_ACPI err = acpi_ndev_init(phytnet_led); +#endif + } if (err) { dev_err(&phytnet_led->pdev->dev, "ndev init wrong\n"); diff --git a/drivers/net/ethernet/phytium/phytmac_v1.c b/drivers/net/ethernet/phytium/phytmac_v1.c index 372544d6304bc..d467644c4a293 100644 --- a/drivers/net/ethernet/phytium/phytmac_v1.c +++ b/drivers/net/ethernet/phytium/phytmac_v1.c @@ -178,20 +178,6 @@ static int phytmac_pcs_software_reset(struct phytmac *pdata, int reset) return 0; } -static int phytmac_pcs_software_reset(struct phytmac *pdata, int reset) -{ - u32 value = PHYTMAC_READ(pdata, PHYTMAC_PCSCTRL); - - if (reset) - value |= PHYTMAC_BIT(PCS_RESET); - else - value &= ~PHYTMAC_BIT(PCS_RESET); - - PHYTMAC_WRITE(pdata, PHYTMAC_PCSCTRL, value); - - return 0; -} - static void phytmac_phy_speed_switch(struct phytmac *pdata, int speed) { struct arm_smccc_res res; From db8b82f08c8dd4ca5485ce00db853b4d802b86f6 Mon Sep 17 00:00:00 2001 From: Ma Mingrui Date: Mon, 11 May 2026 15:44:32 +0800 Subject: [PATCH 12/13] net: pci-rc-vnet: Fix CRC errors caused by TX reclaim and index handling bug Fix CRC errors observed on the EP side during high-rate UDP transmission. Two issues in the RC TX path could lead to EP DMA reading stale or incorrect data: - The TX reclaim threshold condition was never met, causing descriptors to accumulate and TX queue state to become inconsistent. - Incorrect TX index management allowed descriptors to be reused before their payloads were safely reclaimed. Fix both the reclaim condition and index handling to ensure TX descriptors are reclaimed in time and payload/DMA address mapping remains correct to preventing EP-side CRC mismatches. Signed-off-by: Ma Mingrui Signed-off-by: Li Tongfeng Signed-off-by: Wang Yinfeng --- .../phytium-pci-vnet/pci-rc-net/pci_rc_vnet.c | 19 ++++++++++--------- .../phytium-pci-vnet/pci-rc-net/pci_rc_vnet.h | 4 ++-- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/drivers/net/ethernet/phytium-pci-vnet/pci-rc-net/pci_rc_vnet.c b/drivers/net/ethernet/phytium-pci-vnet/pci-rc-net/pci_rc_vnet.c index 52239e22fc5af..3a1f51677b3a9 100644 --- a/drivers/net/ethernet/phytium-pci-vnet/pci-rc-net/pci_rc_vnet.c +++ b/drivers/net/ethernet/phytium-pci-vnet/pci-rc-net/pci_rc_vnet.c @@ -161,23 +161,25 @@ static netdev_tx_t pci_rc_vnet_start_xmit(struct sk_buff *skb, struct net_device data = skb->data; len = skb->len; - if (unlikely(tp->tx_reclaim_num >= PCI_RC_VNET_TX_QUEUE_STOP_THRESHOLD)) { + if (unlikely(tx_queue->nb_desc - tp->tx_reclaim_num <= + PCI_RC_VNET_TX_QUEUE_STOP_THRESHOLD)) { netif_stop_queue(tp->netdev); netdev_info(tp->netdev, "desc queue reclaim is not finished\n"); return NETDEV_TX_BUSY; } + spin_lock_irqsave(&tp->tx_lock, tx_lock_flags); + tail = READ_ONCE(tx_queue->tail); head = READ_ONCE(tx_queue->head); crc32 = crc32_le(~0, data, len); phys_addr = dma_map_single(dev, data, mul32(len), DMA_TO_DEVICE); - if (dma_mapping_error(dev, phys_addr)) { + spin_unlock_irqrestore(&tp->tx_lock, tx_lock_flags); netdev_err(tp->netdev, "tx map failed\n"); - tp->netdev->stats.tx_errors++; return NETDEV_TX_BUSY; } @@ -187,7 +189,6 @@ static netdev_tx_t pci_rc_vnet_start_xmit(struct sk_buff *skb, struct net_device PCI_RC_VNET_DMA_DESC_SET(PKT_LEN, len) | PCI_RC_VNET_DMA_DESC_SET(CHECKSUM, crc32); - spin_lock_irqsave(&tp->tx_lock, tx_lock_flags); if (unlikely(head == (tail + 1) % tx_queue->nb_desc)) { spin_unlock_irqrestore(&tp->tx_lock, tx_lock_flags); @@ -203,11 +204,6 @@ static netdev_tx_t pci_rc_vnet_start_xmit(struct sk_buff *skb, struct net_device /* Make descriptor updates visible to device */ wmb(); - tail = (tail + 1) % ring_size; - WRITE_ONCE(tx_queue->tail, tail); - - spin_unlock_irqrestore(&tp->tx_lock, tx_lock_flags); - spin_lock_irqsave(&tp->tx_reclaim_lock, tx_reclaim_lock_flags); tp->tx_reclaim_num++; spin_unlock_irqrestore(&tp->tx_reclaim_lock, tx_reclaim_lock_flags); @@ -215,8 +211,13 @@ static netdev_tx_t pci_rc_vnet_start_xmit(struct sk_buff *skb, struct net_device tp->tx_skbuff[tail] = skb; tp->tx_phys_addr_list[tail] = phys_addr; + tail = (tail + 1) % ring_size; + WRITE_ONCE(tx_queue->tail, tail); + writel(SEND_MSI_IRQ, tp->msi_irq_addr); + spin_unlock_irqrestore(&tp->tx_lock, tx_lock_flags); + return NETDEV_TX_OK; } diff --git a/drivers/net/ethernet/phytium-pci-vnet/pci-rc-net/pci_rc_vnet.h b/drivers/net/ethernet/phytium-pci-vnet/pci-rc-net/pci_rc_vnet.h index 0b640b59bc7f8..55ce55d29ee66 100644 --- a/drivers/net/ethernet/phytium-pci-vnet/pci-rc-net/pci_rc_vnet.h +++ b/drivers/net/ethernet/phytium-pci-vnet/pci-rc-net/pci_rc_vnet.h @@ -39,7 +39,7 @@ #include #define DRV_MODULE_NAME "pci_rc_vnet" -#define DRV_VERSION "1.0.0" +#define DRV_VERSION "1.0.1" #define PCI_ENDPOINT_VNET_IRQ_NUMBER 0x00 #define PCI_ENDPOINT_VNET_QUEUE_TAIL 0x04 @@ -75,7 +75,7 @@ #define PCI_RC_VNET_DMA_BUFF_NUM 4096 #define PCI_RC_VNET_WAKE_NET_QUEUE_THRESHOLD 10 -#define PCI_RC_VNET_TX_QUEUE_STOP_THRESHOLD (PCI_RC_VNET_DMA_BUFF_NUM - 16) +#define PCI_RC_VNET_TX_QUEUE_STOP_THRESHOLD 64 #define PCI_RC_VNET_TX_QUEUE_RESTART_THRESHOLD 64 #define PCI_RC_VNET_WATCHDOG_PERIOD (2 * HZ) From f7e9be9a0b043f3c75a22bff19139512d7c0c20c Mon Sep 17 00:00:00 2001 From: Ma Mingrui Date: Mon, 11 May 2026 15:47:20 +0800 Subject: [PATCH 13/13] net: pci-ep-vnet: Fix NULL pointer on probe/bind error Fix error handling paths in pci_epf_vnet_probe() to avoid unregistering an unregistered char device when alloc_chrdev_region() fails. Also add a NULL check for vnet in pci_epf_vnet_bind() to prevent NULL pointer dereference when bind is called after a failed probe. Signed-off-by: Ma Mingrui Signed-off-by: Li Tongfeng Signed-off-by: Wang Yinfeng --- .../ethernet/phytium-pci-vnet/pci-ep-net/pci_epf_vnet.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/net/ethernet/phytium-pci-vnet/pci-ep-net/pci_epf_vnet.c b/drivers/net/ethernet/phytium-pci-vnet/pci-ep-net/pci_epf_vnet.c index 8ca505d4c76ba..9fe5ee34f786d 100644 --- a/drivers/net/ethernet/phytium-pci-vnet/pci-ep-net/pci_epf_vnet.c +++ b/drivers/net/ethernet/phytium-pci-vnet/pci-ep-net/pci_epf_vnet.c @@ -1306,7 +1306,7 @@ static int pci_epf_vnet_probe(struct pci_epf *epf, /* cdev register */ err = alloc_chrdev_region(&epf_cdev_mdata.dev_num, 0, 1, DEVICE_NAME); if (err < 0) - goto err_unregister_chrdev; + goto err_unregister_netdev; epf_cdev_mdata.epf_cdev = cdev_alloc(); if (!epf_cdev_mdata.epf_cdev) { @@ -1352,6 +1352,8 @@ static int pci_epf_vnet_probe(struct pci_epf *epf, cdev_del(epf_cdev_mdata.epf_cdev); err_unregister_chrdev: unregister_chrdev_region(epf_cdev_mdata.dev_num, 1); +err_unregister_netdev: + unregister_netdev(netdev); netdev_reg_failed: free_netdev(netdev); return err; @@ -1466,6 +1468,9 @@ static int pci_epf_vnet_bind(struct pci_epf *epf) if (WARN_ON_ONCE(!epc)) return -EINVAL; + if (!vnet) + return -EINVAL; + vnet->vnet_reg_barno = BAR_2; ret = pci_epc_write_header(epc, epf->func_no, epf->vfunc_no, header);