diff --git a/drivers/char/phytnetled/phytnet_led.c b/drivers/char/phytnetled/phytnet_led.c index de02f05eebba5..ce97285af86af 100644 --- a/drivers/char/phytnetled/phytnet_led.c +++ b/drivers/char/phytnetled/phytnet_led.c @@ -137,7 +137,7 @@ of_ndev_init(struct led_data *phytnet_led) return 0; } - +#ifdef CONFIG_ACPI static int acpi_ndev_init(struct led_data *phytnet_led) { @@ -188,6 +188,7 @@ acpi_ndev_init(struct led_data *phytnet_led) return 0; } +#endif static int gpio_init(struct led_data *phytnet_led) @@ -235,10 +236,13 @@ led_init_and_control(struct work_struct *work) int err = -1; struct led_data *phytnet_led = container_of(work, struct led_data, led_control_work.work); - if (phytnet_led->pdev->dev.of_node) + if (phytnet_led->pdev->dev.of_node) { err = of_ndev_init(phytnet_led); - else if (has_acpi_companion(&phytnet_led->pdev->dev)) + } else if (has_acpi_companion(&phytnet_led->pdev->dev)) { +#ifdef CONFIG_ACPI err = acpi_ndev_init(phytnet_led); +#endif + } if (err) { dev_err(&phytnet_led->pdev->dev, "ndev init wrong\n"); diff --git a/drivers/net/ethernet/Kconfig b/drivers/net/ethernet/Kconfig index 0bf3c92c88bba..157d1f59aee40 100644 --- a/drivers/net/ethernet/Kconfig +++ b/drivers/net/ethernet/Kconfig @@ -166,6 +166,8 @@ config ETHOC source "drivers/net/ethernet/packetengines/Kconfig" source "drivers/net/ethernet/pasemi/Kconfig" source "drivers/net/ethernet/pensando/Kconfig" +source "drivers/net/ethernet/phytium/Kconfig" +source "drivers/net/ethernet/phytium-pci-vnet/Kconfig" source "drivers/net/ethernet/qlogic/Kconfig" source "drivers/net/ethernet/brocade/Kconfig" source "drivers/net/ethernet/qualcomm/Kconfig" @@ -195,7 +197,6 @@ source "drivers/net/ethernet/wangxun/Kconfig" source "drivers/net/ethernet/wiznet/Kconfig" source "drivers/net/ethernet/xilinx/Kconfig" source "drivers/net/ethernet/xircom/Kconfig" -source "drivers/net/ethernet/phytium/Kconfig" source "drivers/net/ethernet/guangruntong/Kconfig" source "drivers/net/ethernet/bzwx/Kconfig" source "drivers/net/ethernet/linkdata/Kconfig" diff --git a/drivers/net/ethernet/Makefile b/drivers/net/ethernet/Makefile index d2e95e407e0fe..e8a6674b2cb4b 100644 --- a/drivers/net/ethernet/Makefile +++ b/drivers/net/ethernet/Makefile @@ -78,6 +78,8 @@ obj-$(CONFIG_NET_VENDOR_OKI) += oki-semi/ obj-$(CONFIG_ETHOC) += ethoc.o obj-$(CONFIG_NET_VENDOR_PACKET_ENGINES) += packetengines/ obj-$(CONFIG_NET_VENDOR_PASEMI) += pasemi/ +obj-$(CONFIG_NET_VENDOR_PHYTIUM) += phytium/ +obj-$(CONFIG_VNET_VENDOR_PHYTIUM) += phytium-pci-vnet/ obj-$(CONFIG_NET_VENDOR_QLOGIC) += qlogic/ obj-$(CONFIG_NET_VENDOR_QUALCOMM) += qualcomm/ obj-$(CONFIG_NET_VENDOR_REALTEK) += realtek/ @@ -107,7 +109,6 @@ obj-$(CONFIG_NET_VENDOR_XILINX) += xilinx/ obj-$(CONFIG_NET_VENDOR_XIRCOM) += xircom/ obj-$(CONFIG_NET_VENDOR_SYNOPSYS) += synopsys/ obj-$(CONFIG_NET_VENDOR_PENSANDO) += pensando/ -obj-$(CONFIG_NET_VENDOR_PHYTIUM) += phytium/ obj-$(CONFIG_NET_VENDOR_GRT) += guangruntong/ obj-$(CONFIG_NET_VENDOR_BZWX) += bzwx/ obj-$(CONFIG_NET_VENDOR_LINKDATA) += linkdata/ diff --git a/drivers/net/ethernet/cadence/macb_main.c b/drivers/net/ethernet/cadence/macb_main.c index fcfb5cad6c435..0b1c9610d4db5 100644 --- a/drivers/net/ethernet/cadence/macb_main.c +++ b/drivers/net/ethernet/cadence/macb_main.c @@ -89,7 +89,7 @@ struct sifive_fu540_macb_mgmt { #define GEM_MAX_TX_LEN (unsigned int)(0x3FC0) #define GEM_MTU_MIN_SIZE ETH_MIN_MTU -#define MACB_NETIF_LSO NETIF_F_TSO +#define MACB_NETIF_LSO (NETIF_F_TSO | NETIF_F_TSO6) #define MACB_WOL_HAS_MAGIC_PACKET (0x1 << 0) #define MACB_WOL_ENABLED (0x1 << 1) @@ -767,7 +767,7 @@ static void macb_mac_link_down(struct phylink_config *config, unsigned int mode, /* Tx clean */ for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { - spin_lock(&queue->tx_ptr_lock); + spin_lock_bh(&queue->tx_ptr_lock); for (i = 0; i < bp->tx_ring_size; i++) { tx_skb = macb_tx_skb(queue, i); /* free unsent skb buffers */ @@ -778,7 +778,7 @@ static void macb_mac_link_down(struct phylink_config *config, unsigned int mode, macb_set_addr(bp, tx_desc, 0); tx_desc->ctrl &= ~MACB_BIT(TX_USED); } - spin_unlock(&queue->tx_ptr_lock); + spin_unlock_bh(&queue->tx_ptr_lock); } netif_tx_stop_all_queues(ndev); diff --git a/drivers/net/ethernet/phytium-pci-vnet/Kconfig b/drivers/net/ethernet/phytium-pci-vnet/Kconfig new file mode 100644 index 0000000000000..94934b2250df4 --- /dev/null +++ b/drivers/net/ethernet/phytium-pci-vnet/Kconfig @@ -0,0 +1,26 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Phytium PCI virtual network device configuration +# + +menu "Phytium PCI Virtual Ethernet Drivers" +config VNET_VENDOR_PHYTIUM + bool "Phytium Virtual Ethernet devices" + depends on ARCH_PHYTIUM + depends on HAS_IOMEM + default y + help + If you have a network (Ethernet) card belonging to this class, say Y. + + Note that the answer to this question doesn't directly affect the + kernel: saying N will just cause the configurator to skip all the + remaining Cadence network card questions. If you say Y, you will be + asked for your specific card in the following questions. + +if VNET_VENDOR_PHYTIUM + +source "drivers/net/ethernet/phytium-pci-vnet/pci-ep-net/Kconfig" +source "drivers/net/ethernet/phytium-pci-vnet/pci-rc-net/Kconfig" + +endif #VNET_VENDOR_PHYTIUM +endmenu diff --git a/drivers/net/ethernet/phytium-pci-vnet/Makefile b/drivers/net/ethernet/phytium-pci-vnet/Makefile new file mode 100644 index 0000000000000..33d25e17d2d86 --- /dev/null +++ b/drivers/net/ethernet/phytium-pci-vnet/Makefile @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0 +# Copyright(c) 2022 - 2025 Phytium Technology Co., Ltd. +# +# Makefile for the Phytium PCI virtual network device drivers. + +obj-$(CONFIG_PHYTIUM_PCI_EPF_VNET) += pci-ep-net/ +obj-$(CONFIG_PHYTIUM_PCI_RC_VNET) += pci-rc-net/ diff --git a/drivers/net/ethernet/phytium-pci-vnet/pci-ep-net/Kconfig b/drivers/net/ethernet/phytium-pci-vnet/pci-ep-net/Kconfig new file mode 100644 index 0000000000000..16a458c91c6d6 --- /dev/null +++ b/drivers/net/ethernet/phytium-pci-vnet/pci-ep-net/Kconfig @@ -0,0 +1,19 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Phytium PCI EP virtual network device configuration +# + +config PHYTIUM_PCI_EPF_VNET + tristate "Phytium PCI Endpoint Virtual Ethernet Device" + depends on PCI_ENDPOINT + depends on PCIE_PHYTIUM_EP + help + This option enables support for the Phytium PCI Endpoint Virtual Ethernet function + driver. It allows the device to appear as a network interface to a PCIe root complex. + + Typically used in conjunction with a Phytium PCIe Root Complex host that supports + the corresponding RC virtual network function. + + If you want this endpoint to provide virtual networking over PCIe, say Y or M. + + If unsure, say N. diff --git a/drivers/net/ethernet/phytium-pci-vnet/pci-ep-net/Makefile b/drivers/net/ethernet/phytium-pci-vnet/pci-ep-net/Makefile new file mode 100644 index 0000000000000..755dd26dceb9d --- /dev/null +++ b/drivers/net/ethernet/phytium-pci-vnet/pci-ep-net/Makefile @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0 +# Copyright(c) 2022 - 2025 Phytium Technology Co., Ltd. +# +# Makefile for the Phytium network device drivers. + +obj-$(CONFIG_PHYTIUM_PCI_EPF_VNET) += pci_epf_vnet.o diff --git a/drivers/net/ethernet/phytium-pci-vnet/pci-ep-net/pci_epf_vnet.c b/drivers/net/ethernet/phytium-pci-vnet/pci-ep-net/pci_epf_vnet.c new file mode 100644 index 0000000000000..9fe5ee34f786d --- /dev/null +++ b/drivers/net/ethernet/phytium-pci-vnet/pci-ep-net/pci_epf_vnet.c @@ -0,0 +1,1577 @@ +// SPDX-License-Identifier: GPL-2.0 +/** + * Virtual network driver based on the pcie interface in EP + * + * Copyright (C) 2023 Phytium Corporation + * Author: netgroup@phytium.com.cn + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "pci_epf_vnet.h" + +char dev_addr[6] = {0x04, 0x05, 0x06, 0x00, 0x00, 0x16}; + +static int debug = 3; +module_param(debug, int, 0); + +static struct cdev_mdata epf_cdev_mdata; +static struct epf_ioctl_data ioctl_data; + +static struct sk_buff *ep_build_skb(struct ep_rx_buffer *rx_buffer, + unsigned int size) +{ + struct sk_buff *skb; + unsigned int truesize; + void *va; + +#if (PAGE_SIZE < 8192) + truesize = EP_RX_PAGE_SIZE / 2; +#else + truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) + + SKB_DATA_ALIGN(EP_SKB_PAD + size); +#endif + + va = page_address(rx_buffer->page) + rx_buffer->page_offset; + /* prefetch first cache line of first page */ + prefetch(va); + + /* build an skb around the page buffer */ + skb = build_skb(va - EP_SKB_PAD, truesize); + if (unlikely(!skb)) + return NULL; + + /* update pointers within the skb to store the data */ + skb_reserve(skb, EP_SKB_PAD); + __skb_put(skb, size); + + /* update buffer offset */ +#if (PAGE_SIZE < 8192) + rx_buffer->page_offset ^= truesize; +#else + rx_buffer->page_offset += truesize; +#endif + + return skb; +} + +static inline int mul32(int size) +{ + int ret = 0; + + if (size % 32) + ret = size + (32 - size % 32); + else + ret = size; + + return ret; +} + +static inline struct device *vnet_to_dev(struct pci_epf_vnet *vnet) +{ + return &vnet->epf->dev; +} + +static inline unsigned int vnet_tx_ring_wrap(struct pci_epf_vnet *vnet, + unsigned int index) +{ + return index & (vnet->tx_ring_size - 1); +} + +static inline struct pci_ep_dma_desc *vnet_tx_desc(struct pci_epf_vnet *vnet, + unsigned int index) +{ + struct pci_ep_queue *queue = vnet->queue; + struct ep_queue *tx_queue = &queue->ep_tx_queue; + /* Obtain the index of descriptors */ + index = vnet_tx_ring_wrap(vnet, index); + return &tx_queue->dma_desc_base[index]; +} + +static inline unsigned int vnet_rx_ring_wrap(struct pci_epf_vnet *vnet, + unsigned int index) +{ + return index & (vnet->rx_ring_size - 1); +} + +static inline struct pci_ep_dma_desc *vnet_rx_desc(struct pci_epf_vnet *vnet, + unsigned int index) +{ + struct pci_ep_queue *queue = vnet->queue; + struct ep_queue *rx_queue = &queue->ep_rx_queue; + /* Obtain the index of descriptors */ + index = vnet_rx_ring_wrap(vnet, index); + return &rx_queue->dma_desc_base[index]; +} + +static inline int pci_epf_vnet_ep2rc_dma(struct pci_epf_vnet *vnet, + dma_addr_t buffer_dma_addr, + u64 addr, u32 data_size, + enum dma_data_direction dir) +{ + int ret; + struct pci_epc *epc = vnet->epf->epc; + + dma_sync_single_for_device(epc->dev.parent, buffer_dma_addr, data_size, + dir); + + ret = pci_epc_start_dma(epc, vnet->epf->func_no, buffer_dma_addr, addr, + data_size, DMA_WRITE); + if (ret) { + dev_err(&epc->dev, "ep2rc pci_epc_start_dma fail! ret %d\n", + ret); + return -EIO; + } + + return 0; +} + +static inline int pci_epf_vnet_rc2ep_dma(struct pci_epf_vnet *vnet, + dma_addr_t buffer_dma_addr, + u64 addr, u32 data_size) +{ + int ret; + struct pci_epc *epc = vnet->epf->epc; + + ret = pci_epc_start_dma(epc, vnet->epf->func_no, buffer_dma_addr, addr, + data_size, DMA_READ); + if (ret) { + dev_err(&epc->dev, "rc2ep pci_epc_start_dma fail! ret %d\n", + ret); + return -EIO; + } + + return 0; +} + +static inline bool vnet_tx_desc_queue_full(struct pci_epf_vnet *vnet) +{ + u32 tail, head; + struct pci_ep_queue *queue = vnet->queue; + struct ep_queue *tx_queue = &queue->ep_tx_queue; + + head = READ_ONCE(tx_queue->head); + tail = READ_ONCE(tx_queue->tail); + + if (head == (tail + 1) % vnet->tx_ring_size) + return true; + else + return false; +} + +static inline int vnet_get_tx_desc_tail_idx(struct pci_epf_vnet *vnet) +{ + u32 tail; + struct pci_ep_queue *queue = vnet->queue; + struct ep_queue *tx_queue = &queue->ep_tx_queue; + + tail = READ_ONCE(tx_queue->tail); + return tail; +} + +static inline int vnet_adj_tx_desc_tail_idx(struct pci_epf_vnet *vnet) +{ + struct pci_ep_queue *queue = vnet->queue; + struct ep_queue *tx_queue = &queue->ep_tx_queue; + u32 tail = READ_ONCE(tx_queue->tail); + + /* Move tail pointer */ + tail = (tail + 1) % vnet->tx_ring_size; + WRITE_ONCE(tx_queue->tail, tail); + + return 0; +} + +static bool ep_alloc_mapped_page(struct pci_epf_vnet *vnet, + struct ep_rx_buffer *rx_buffer_info) +{ + dma_addr_t paddr; + struct pci_epf *epf = vnet->epf; + struct pci_epc *epc = epf->epc; + struct page *page = rx_buffer_info->page; + + /* since we are recycling buffers we should seldom need to alloc */ + if (likely(page)) + return true; + + page = __dev_alloc_pages(EP_GFP_FLAGS, EP_RX_PAGE_ORDER); + if (unlikely(!page)) { + netdev_err(vnet->netdev, "rx alloc page failed\n"); + rx_buffer_info->page = NULL; + return false; + } + + paddr = dma_map_page_attrs(epc->dev.parent, page, 0, + EP_RX_PAGE_SIZE, + DMA_FROM_DEVICE, EP_DMA_ATTR); + if (dma_mapping_error(epc->dev.parent, paddr)) { + __free_pages(page, EP_RX_PAGE_ORDER); + rx_buffer_info->page = NULL; + return false; + } + + rx_buffer_info->addr = paddr; + rx_buffer_info->page = page; + rx_buffer_info->pagecnt_bias = 1; + rx_buffer_info->page_offset = EP_SKB_PAD; + + return true; +} + +static void vnet_rx_refill(struct pci_epf_vnet *vnet, bool init) +{ + struct pci_epf *epf = vnet->epf; + struct pci_epc *epc = epf->epc; + unsigned int entry, space; + struct ep_rx_buffer *rx_buffer_info; + + if (init) + space = vnet->rx_ring_size - 1; + else + space = CIRC_SPACE(vnet->rx_refill_start, + READ_ONCE(vnet->queue->ep_rx_queue.head), + vnet->rx_ring_size); + while (space > 0) { + entry = vnet_rx_ring_wrap(vnet, vnet->rx_refill_start); + + rx_buffer_info = &vnet->rx_buffer_info[entry]; + if (!ep_alloc_mapped_page(vnet, rx_buffer_info)) + break; + /* sync the buffer for use by the device */ + dma_sync_single_range_for_device(epc->dev.parent, + rx_buffer_info->addr, + rx_buffer_info->page_offset, + vnet->rx_buffer_len, + DMA_FROM_DEVICE); + vnet->rx_refill_start = (vnet->rx_refill_start + 1) % + vnet->rx_ring_size; + space--; + } + + vnet->rx_next_to_alloc = vnet->rx_refill_start; +} + +static int vnet_tx_fill(struct pci_epf_vnet *vnet, u32 pktnum) +{ + struct page *page; + dma_addr_t paddr; + struct pci_epf *epf = vnet->epf; + struct pci_epc *epc = epf->epc; + int count; + void *buffer_virt; + + for (count = 0; count < pktnum; count++) { + page = __dev_alloc_pages(EP_GFP_FLAGS, EP_TX_PAGE_ORDER); + if (unlikely(!page)) { + netdev_err(vnet->netdev, "tx alloc page failed\n"); + break; + } + + buffer_virt = page_to_virt(page); + + paddr = dma_map_page_attrs(epc->dev.parent, page, 0, + EP_TX_PAGE_SIZE, + DMA_FROM_DEVICE, EP_DMA_ATTR); + if (dma_mapping_error(epc->dev.parent, paddr)) { + netdev_err(vnet->netdev, "tx map page failed\n"); + break; + } + + vnet->tx_buffer_info[count].page = page; + vnet->tx_buffer_info[count].addr = paddr; + vnet->tx_buffer_info[count].vaddr = buffer_virt; + } + + return count; +} + +static void vnet_free_rx_skb_ring(struct pci_epf_vnet *vnet, int count) +{ + int i; + struct pci_epf *epf = vnet->epf; + struct pci_epc *epc = epf->epc; + struct ep_rx_buffer *rx_buffer_info; + + for (i = 0; i < count; i++) { + rx_buffer_info = &vnet->rx_buffer_info[i]; + if (!rx_buffer_info->page) + continue; + + dma_unmap_page_attrs(epc->dev.parent, rx_buffer_info->addr, + EP_RX_PAGE_SIZE, DMA_FROM_DEVICE, EP_DMA_ATTR); + + __page_frag_cache_drain(rx_buffer_info->page, + rx_buffer_info->pagecnt_bias); + } + + vfree(vnet->rx_buffer_info); + vnet->rx_buffer_info = NULL; +} + +static inline int ep_calc_rx_buf_len(void) +{ +#if (PAGE_SIZE < 8192) + return rounddown(EP_MAX_FRAME_BUILD_SKB, RX_BUFFER_MULTIPLE); +#endif + return rounddown(EP_RXBUFFER_2048, RX_BUFFER_MULTIPLE); +} + +static int vnet_alloc_rx_skb_ring(struct pci_epf_vnet *vnet) +{ + int size; + + vnet->rx_buffer_len = ep_calc_rx_buf_len(); + + /* Allocate pointer array for rx buffers */ + size = vnet->rx_ring_size * sizeof(struct ep_rx_buffer); + vnet->rx_buffer_info = vzalloc(size); + if (!vnet->rx_buffer_info) { + netdev_err(vnet->netdev, + "%s:Unable to allocate buffer memory\n", __func__); + return -ENOMEM; + } + + netdev_dbg(vnet->netdev, + "Allocated %d RX struct buffer entries at %p\n", + vnet->rx_ring_size, vnet->rx_skb); + + /* Pre allocated rx skb */ + vnet_rx_refill(vnet, true); + + return 0; +} + +static void vnet_free_tx_skb_ring(struct pci_epf_vnet *vnet, int count) +{ + int i; + struct pci_epf *epf = vnet->epf; + struct pci_epc *epc = epf->epc; + struct ep_tx_buffer *tx_buffer_info; + + for (i = 0; i < count; i++) { + tx_buffer_info = &vnet->tx_buffer_info[i]; + if (!tx_buffer_info->page) + continue; + + dma_unmap_page_attrs(epc->dev.parent, tx_buffer_info->addr, + EP_RX_PAGE_SIZE, DMA_FROM_DEVICE, EP_DMA_ATTR); + + __page_frag_cache_drain(tx_buffer_info->page, + tx_buffer_info->pagecnt_bias); + } + + vfree(vnet->tx_buffer_info); + vnet->tx_buffer_info = NULL; +} + +static int vnet_alloc_tx_skb_ring(struct pci_epf_vnet *vnet) +{ + int size; + int count; + + size = vnet->tx_ring_size * sizeof(struct ep_tx_buffer); + vnet->tx_buffer_info = vzalloc(size); + if (!vnet->tx_buffer_info) { + netdev_err(vnet->netdev, + "%s:Unable to allocate buffer memory\n", __func__); + return -ENOMEM; + } + + netdev_dbg(vnet->netdev, + "Allocated %d TX struct buffer entries at %p\n", + vnet->tx_ring_size, vnet->tx_buffer_info); + + /* Pre allocated tx buffer */ + count = vnet_tx_fill(vnet, vnet->tx_ring_size); + if (count != vnet->tx_ring_size) { + netdev_err(vnet->netdev, + "%s:There is that page allocation failed!\n", + __func__); + vnet_free_tx_skb_ring(vnet, count); + return -ENOMEM; + } + + return 0; +} + +static int vnet_down(struct net_device *netdev) +{ + struct pci_epf_vnet *vnet = netdev_priv(netdev); + + napi_disable(&vnet->rx_napi); + + netif_stop_queue(netdev); + + netif_carrier_off(netdev); + + vnet_free_rx_skb_ring(vnet, vnet->rx_ring_size); + vnet_free_tx_skb_ring(vnet, vnet->tx_ring_size); + + return 0; +} + +static int vnet_up(struct net_device *netdev) +{ + struct pci_epf_vnet *vnet = netdev_priv(netdev); + struct ep_queue *rx_queue; + struct ep_queue *tx_queue; + int err; + + rx_queue = &vnet->queue->ep_rx_queue; + tx_queue = &vnet->queue->ep_tx_queue; + + err = vnet_alloc_rx_skb_ring(vnet); + if (err) { + netdev_err(vnet->netdev, "%s: failed to alloc rx buffer!!\n", + __func__); + return err; + } + + err = vnet_alloc_tx_skb_ring(vnet); + if (err) { + netdev_err(vnet->netdev, "%s: failed to alloc tx buffer!!\n", + __func__); + return err; + } + + netdev->flags |= IFF_RUNNING | IFF_UP; + + netif_wake_queue(netdev); + + napi_enable(&vnet->rx_napi); + + return 0; +} + +static void vnet_get_drvinfo(struct net_device *netdev, + struct ethtool_drvinfo *drvinfo) +{ + strscpy(drvinfo->driver, DRV_MODULE_NAME, sizeof(drvinfo->driver)); + strscpy(drvinfo->version, DRV_VERSION, sizeof(drvinfo->version)); +} + +static u32 vnet_get_msglevel(struct net_device *netdev) +{ + struct pci_epf_vnet *vnet = netdev_priv(netdev); + + return vnet->msg_enable; +} + +static void vnet_set_msglevel(struct net_device *netdev, u32 value) +{ + struct pci_epf_vnet *vnet = netdev_priv(netdev); + + vnet->msg_enable = value; +} + +static u32 vnet_get_link(struct net_device *netdev) +{ + struct pci_epf_vnet *vnet = netdev_priv(netdev); + struct pci_ep_queue *queue = vnet->queue; + struct ep_queue *tx_queue = &queue->ep_tx_queue; + + return tx_queue->flags & PCI_EP_VNET_FLAGS_BIT(RC_LINK_ACTIVE); +} + +static const char vnet_gstrings_stats[][ETH_GSTRING_LEN] = { + "rx_packets", "tx_packets", "rx_bytes", + "tx_bytes", "rx_errors", "tx_errors", + "rx_dropped", "tx_dropped", "multicast", + "collisions", "rx_length_errors", "rx_over_errors", + "rx_crc_errors", "rx_frame_errors", "rx_fifo_errors", + "rx_missed_errors", "tx_aborted_errors", "tx_carrier_errors", + "tx_fifo_errors", "tx_heartbeat_errors", "tx_window_errors", +}; + +#define VNET_NET_STATS_LEN ARRAY_SIZE(vnet_gstrings_stats) + +static int vnet_get_sset_count(struct net_device *netdev, int sset) +{ + switch (sset) { + case ETH_SS_STATS: + return VNET_NET_STATS_LEN; + default: + return -EOPNOTSUPP; + } +} + +static void vnet_get_ethtool_stats(struct net_device *netdev, + struct ethtool_stats *stats, u64 *data) +{ + int i; + + for (i = 0; i < VNET_NET_STATS_LEN; i++) + data[i] = ((unsigned long *)&netdev->stats)[i]; +} + +static void vnet_get_strings(struct net_device *netdev, u32 stringset, u8 *data) +{ + switch (stringset) { + case ETH_SS_STATS: + memcpy(data, *vnet_gstrings_stats, sizeof(vnet_gstrings_stats)); + break; + } +} + +static const struct ethtool_ops vnet_ethtool_ops = { + .get_drvinfo = vnet_get_drvinfo, + .get_msglevel = vnet_get_msglevel, + .set_msglevel = vnet_set_msglevel, + .get_link = vnet_get_link, + .get_strings = vnet_get_strings, + .get_ethtool_stats = vnet_get_ethtool_stats, + .get_sset_count = vnet_get_sset_count, +}; + +static void pci_ep_vnet_get_defaults(struct pci_epf_vnet *vnet) +{ + struct param_range rxd_size = { .min = VNET_MIN_RXD, + .max = VNET_MAX_RXD, + .count = VNET_DEFAULT_RXD }; + struct param_range txd_size = { .min = VNET_MIN_TXD, + .max = VNET_MAX_TXD, + .count = VNET_DEFAULT_TXD }; + + vnet->rx_refill_start = 0; + vnet->rx_refill_num = 0; + + /* RX buffers initialization */ + vnet->rx_buffer_size = VNET_RXBUFFER_2048; + + vnet->rx_ring_size = rxd_size.count; + vnet->tx_ring_size = txd_size.count; + + vnet->rxd_size = rxd_size; + vnet->txd_size = txd_size; +} + +static bool ep_can_reuse_rx_page(struct ep_rx_buffer *rx_buffer) +{ + unsigned int pagecnt_bias = rx_buffer->pagecnt_bias; + struct page *page = rx_buffer->page; + + /* avoid re-using remote and pfmemalloc pages */ + if (!dev_page_is_reusable(page)) + return false; + +#if (PAGE_SIZE < 8192) + /* if we are only owner of page we can reuse it */ + if (unlikely((page_ref_count(page) - pagecnt_bias) > 1)) + return false; +#else +#define EP_LAST_OFFSET \ + (SKB_WITH_OVERHEAD(PAGE_SIZE) - EP_RXBUFFER_2048) + + if (rx_buffer->page_offset > EP_LAST_OFFSET) + return false; +#endif + + /* If we have drained the page fragment pool we need to update + * the pagecnt_bias and page count so that we fully restock the + * number of references the driver holds. + */ + if (unlikely(!pagecnt_bias)) { + page_ref_add(page, USHRT_MAX); + rx_buffer->pagecnt_bias = USHRT_MAX; + } + + return true; +} + +static void ep_reuse_rx_page(struct pci_epf_vnet *vnet, + struct ep_rx_buffer *old_buff) +{ + struct ep_rx_buffer *new_buff; + u16 nta = vnet->rx_next_to_alloc; + + new_buff = &vnet->rx_buffer_info[nta & (vnet->rx_ring_size - 1)]; + + /* update, and store next to alloc */ + nta++; + vnet->rx_next_to_alloc = (nta < vnet->rx_ring_size) ? nta : 0; + + /* Transfer page from old buffer to new buffer. + * Move each member individually to avoid possible store + * forwarding stalls. + */ + new_buff->addr = old_buff->addr; + new_buff->page = old_buff->page; + new_buff->page_offset = old_buff->page_offset; + new_buff->pagecnt_bias = old_buff->pagecnt_bias; +} + +static void ep_put_rx_buffer(struct pci_epf_vnet *vnet, + struct ep_rx_buffer *rx_buffer_info) +{ + struct pci_epf *epf = vnet->epf; + struct pci_epc *epc = epf->epc; + + if (ep_can_reuse_rx_page(rx_buffer_info)) { + /* hand second half of page back to the ring */ + ep_reuse_rx_page(vnet, rx_buffer_info); + } else { + dma_unmap_page_attrs(epc->dev.parent, rx_buffer_info->addr, + EP_RX_PAGE_SIZE, DMA_FROM_DEVICE, EP_DMA_ATTR); + + __page_frag_cache_drain(rx_buffer_info->page, + rx_buffer_info->pagecnt_bias); + } + + /* clear contents of rx_buffer */ + rx_buffer_info->page = NULL; +} + +static int pci_epf_vnet_rx(struct pci_epf_vnet *vnet, int budget) +{ + struct pci_epf *epf = vnet->epf; + struct pci_epc *epc = epf->epc; + struct ep_queue *rx_queue = &vnet->queue->ep_rx_queue; + u64 src_addr, dst_addr; + struct sk_buff *skb; + unsigned int len; + struct pci_ep_dma_desc *desc; + int count; + u32 desc_crc, pkt_crc; + void *pkt_data; + unsigned int entry; + int ret = 0; + int timeout; + struct ep_rx_buffer *rx_buffer_info; + u32 rx_head, rx_tail; + + for (count = 0; count < budget; count++) { + rx_head = READ_ONCE(rx_queue->head); + rx_tail = READ_ONCE(rx_queue->tail); + /* Obtain descriptor index */ + entry = vnet_rx_ring_wrap(vnet, rx_head); + + /* Obtain descriptors for bar space */ + desc = vnet_rx_desc(vnet, entry); + + /* No package to receive */ + if (rx_head == rx_tail) + break; + + /* Make hw descriptor updates visible to CPU */ + dma_rmb(); + /* Determine if it is a valid package */ + if (!(desc->ctrl & PCI_EP_DMA_DESC_CTRL_FIELD_MASK(USED))) + break; + + rx_buffer_info = &vnet->rx_buffer_info[entry & (vnet->rx_ring_size - 1)]; + if (!rx_buffer_info->page) { + netdev_err(vnet->netdev, "%s:The page used is null!\n", __func__); + break; + } + + rx_buffer_info->pagecnt_bias--; + + /* Obtain packet length from descriptor */ + len = PCI_EP_DMA_DESC_FIELD_GET(LEN, desc->ctrl); + + /* Obtain the DMA address of the package in RC from the descriptor */ + src_addr = desc->addr; + dst_addr = rx_buffer_info->addr + rx_buffer_info->page_offset; + + spin_lock(&vnet->rx_lock); + + /* Start DMA transmission */ + ret = pci_epf_vnet_rc2ep_dma(vnet, dst_addr, + src_addr, mul32(len)); + if (ret) { + spin_unlock(&vnet->rx_lock); + netdev_err(vnet->netdev, + "dma transfer failure(err %d)\n", ret); + vnet->netdev->stats.rx_dropped++; + rx_buffer_info->pagecnt_bias++; + ep_put_rx_buffer(vnet, rx_buffer_info); + + desc->ctrl |= PCI_EP_DMA_DESC_CTRL_PKT_ERR; + desc->ctrl &= ~PCI_EP_DMA_DESC_CTRL_PKT_USED; + dma_wmb(); + goto next_cycle; + } + + timeout = (mul32(len) / DIVISOR + 1) * 10 * 8; + while (!(pci_epc_dma_status(epc, epf->func_no, DMA_READ) & + DMA_STATUS_DONE)) { + timeout -= 5; + udelay(1); + if (timeout <= 0) { + spin_unlock(&vnet->rx_lock); + netdev_err(vnet->netdev, + "%s:dma transfer timeout!\n", + __func__); + vnet->netdev->stats.rx_dropped++; + rx_buffer_info->pagecnt_bias++; + ep_put_rx_buffer(vnet, rx_buffer_info); + + desc->ctrl |= PCI_EP_DMA_DESC_CTRL_PKT_ERR; + desc->ctrl &= ~PCI_EP_DMA_DESC_CTRL_PKT_USED; + dma_wmb(); + goto next_cycle; + } + } + + spin_unlock(&vnet->rx_lock); + + dma_sync_single_range_for_cpu(epc->dev.parent, + rx_buffer_info->addr, + rx_buffer_info->page_offset, + len, DMA_FROM_DEVICE); + + pkt_data = page_address(rx_buffer_info->page) + + rx_buffer_info->page_offset - EP_SKB_PAD; + pkt_crc = crc32_le(~0, (unsigned char *)pkt_data, len); + desc_crc = PCI_EP_DMA_DESC_FIELD_GET(CHECKSUM, desc->ctrl); + + /* Compare the CRC of RC and EP */ + if (pkt_crc != desc_crc) { + vnet->netdev->stats.rx_errors++; + rx_buffer_info->pagecnt_bias++; + ep_put_rx_buffer(vnet, rx_buffer_info); + + desc->ctrl |= PCI_EP_DMA_DESC_CTRL_PKT_ERR; + desc->ctrl &= ~PCI_EP_DMA_DESC_CTRL_PKT_USED; + dma_wmb(); + goto next_cycle; + } + + skb = ep_build_skb(rx_buffer_info, len); + if (unlikely(!skb)) { + netdev_err(vnet->netdev, "rx build skb failed\n"); + vnet->netdev->stats.rx_dropped++; + rx_buffer_info->pagecnt_bias++; + break; + } + + ep_put_rx_buffer(vnet, rx_buffer_info); + + /* Modify the Ctrl key in the descriptor */ + desc->ctrl &= ~PCI_EP_DMA_DESC_CTRL_PKT_USED; + dma_wmb(); + + skb->protocol = eth_type_trans(skb, vnet->netdev); + skb->ip_summed = CHECKSUM_UNNECESSARY; + + /* Upload protocol stack */ + netif_receive_skb(skb); + + vnet->netdev->stats.rx_packets++; + vnet->netdev->stats.rx_bytes += skb->len; + +next_cycle: + /* Move the SKB position in the ring backwards */ + rx_head = (rx_head + 1) % vnet->rx_ring_size; + WRITE_ONCE(rx_queue->head, rx_head); + } + + vnet_rx_refill(vnet, false); + + return count; +} + +int pci_epf_vnet_clean_ep2rc_irq(struct pci_epf_vnet *vnet, int budget) +{ + int ret = 0; + + ret = pci_epf_vnet_rx(vnet, budget); + return ret; +} + +void pci_epf_vnet_set_irq(struct phytium_pcie_ep *priv, struct pci_epf_vnet *vnet) +{ + u32 bar1_lo_addr, bar1_hi_addr; + struct pci_epf *epf = vnet->epf; + + /* Use BAR1 as the area to trigger the EP interrupt */ + bar1_lo_addr = readl(priv->reg_base + + PHYTIUM_PCIE_FUNC_BASE(epf->func_no) + + PHYTIUM_PCI_WIN0_TRSL_ADDR0(BAR_1)); + bar1_hi_addr = readl(priv->reg_base + + PHYTIUM_PCIE_FUNC_BASE(epf->func_no) + + PHYTIUM_PCI_WIN0_TRSL_ADDR1(BAR_1)); + + writel(bar1_lo_addr, priv->hpb_base + PHYTIUM_HPB_REG_MSI64_LO_ADDR); + writel(bar1_hi_addr, priv->hpb_base + PHYTIUM_HPB_REG_MSI64_HI_ADDR); + + /* Enable Interrupt */ + writel(HPB_ERR_EVT_LOG_EN, priv->hpb_base + + PHYTIUM_HPB_REG_ERR_EVT_LOG_EN); + writel(HPB_ERR_EVT_INT_EN, priv->hpb_base + + PHYTIUM_HPB_REG_ERR_EVT_INT_EN); + writel(HPB_MSI_EN, priv->hpb_base + PHYTIUM_HPB_REG_MSI_EN); + writel(HPB_NS_MSI_SPI_EN, priv->hpb_base + + PHYTIUM_HPB_REG_NS_MSI_SPI_EN); +} + +void pci_epf_vnet_enable_irq(struct phytium_pcie_ep *priv) +{ + /* Enable interrupt reporting */ + writel(HPB_ERR_EVT_INT_EN, priv->hpb_base + + PHYTIUM_HPB_REG_ERR_EVT_INT_EN); +} + +void pci_epf_vnet_disable_irq(struct phytium_pcie_ep *priv) +{ + /* Close interrupt reporting */ + writel(HPB_ERR_EVT_INT_CLR, priv->hpb_base + + PHYTIUM_HPB_REG_ERR_EVT_INT_EN); +} + +static int pci_epf_vnet_rx_poll(struct napi_struct *napi, int budget) +{ + int work_done; + struct pci_epf_vnet *vnet = container_of(napi, + struct pci_epf_vnet, + rx_napi); + struct pci_epf *epf = vnet->epf; + struct pci_epc *epc = epf->epc; + struct phytium_pcie_ep *priv = epc_get_drvdata(epc); + + work_done = vnet->clean_rc2ep(vnet, budget); + if (work_done < budget) { + napi_complete_done(napi, work_done); + /* Enable interrupt */ + pci_epf_vnet_enable_irq(priv); + } + return work_done; +} + +static irqreturn_t pci_epf_vnet_rx_irqhandler(int irq, void *data) +{ + struct pci_epf_vnet *vnet = (struct pci_epf_vnet *)data; + struct pci_epf *epf = vnet->epf; + struct pci_epc *epc = epf->epc; + struct phytium_pcie_ep *priv = epc_get_drvdata(epc); + + if (napi_schedule_prep(&vnet->rx_napi)) { + /* Disable interrupt */ + pci_epf_vnet_disable_irq(priv); + __napi_schedule(&vnet->rx_napi); + } + + readl(priv->hpb_base + PHYTIUM_HPB_REG_NS_MSI_DATA); + return IRQ_HANDLED; +} + +static netdev_tx_t vnet_start_xmit(struct sk_buff *skb, + struct net_device *netdev) +{ + u32 tail; + u32 crc32; + int timeout; + int len; + char *data; + dma_addr_t src_addr, dst_addr = 0; + int ret; + unsigned long tx_lock_flags; + struct pci_ep_dma_desc *desc; + u64 ctrl = 0; + u32 tx_flags; + + struct pci_epf_vnet *vnet = netdev_priv(netdev); + struct pci_ep_queue *queue = vnet->queue; + struct pci_epf *epf = vnet->epf; + struct pci_epc *epc = epf->epc; + struct ep_queue *tx_queue = &queue->ep_tx_queue; + + data = skb->data; + len = skb->len; + + /* Check if RC is online */ + tx_flags = READ_ONCE(tx_queue->flags); + if (!(tx_flags & PCI_EP_VNET_FLAGS_BIT(RC_LINK_ACTIVE))) + return NETDEV_TX_BUSY; + + /* Calculate the CRC of the package */ + crc32 = crc32_le(~0, data, len); + + spin_lock_irqsave(&vnet->tx_lock, tx_lock_flags); + + /* Check if the bar space descriptor ring is full */ + if (vnet_tx_desc_queue_full(vnet)) { + spin_unlock_irqrestore(&vnet->tx_lock, tx_lock_flags); + netdev_err(vnet->netdev, "tx desc queue is full!!!\n"); + return NETDEV_TX_BUSY; + } + + /* Obtain the tail index of the bar descriptor ring */ + tail = vnet_get_tx_desc_tail_idx(vnet); + + desc = vnet_tx_desc(vnet, tail); + + /* Obtain the DMA address of the buffer on the RC end */ + dst_addr = desc->addr; + if (!dst_addr) { + spin_unlock_irqrestore(&vnet->tx_lock, tx_lock_flags); + return NETDEV_TX_BUSY; + } + + memcpy(vnet->tx_buffer_info[tail].vaddr, data, len); + + src_addr = vnet->tx_buffer_info[tail].addr; + + ctrl |= PCI_EP_DMA_DESC_CTRL_PKT_USED | + PCI_EP_DMA_DESC_FIELD_SET(LEN, len) | + PCI_EP_DMA_DESC_FIELD_SET(CHECKSUM, (u64)crc32); + + /* Start DMA transmission */ + ret = pci_epf_vnet_ep2rc_dma(vnet, src_addr, dst_addr, mul32(len), + DMA_TO_DEVICE); + if (ret) { + spin_unlock_irqrestore(&vnet->tx_lock, tx_lock_flags); + dev_kfree_skb_any(skb); + /* Packet loss statistics */ + vnet->netdev->stats.tx_dropped++; + netdev_err(vnet->netdev, "dma failed(%d)!\n", ret); + return NETDEV_TX_OK; + } + + timeout = (mul32(len) / DIVISOR + 1) * 10 * 8; + while (!(pci_epc_dma_status(epc, epf->func_no, DMA_WRITE) & + DMA_STATUS_DONE)) { + timeout -= 5; + udelay(1); + if (timeout <= 0) { + netdev_err(vnet->netdev, "%s:dma transfer timeout!\n", __func__); + spin_unlock_irqrestore(&vnet->tx_lock, tx_lock_flags); + dev_kfree_skb_any(skb); + vnet->netdev->stats.tx_dropped++; + return NETDEV_TX_OK; + } + } + + /* Move the tail pointer of the ep2rc descriptor */ + vnet_adj_tx_desc_tail_idx(vnet); + + /* Modify the Ctrl key in the descriptor */ + desc->ctrl = ctrl; + + /* Make descriptor updates visible to device */ + dma_wmb(); + + spin_unlock_irqrestore(&vnet->tx_lock, tx_lock_flags); + + vnet->netdev->stats.tx_packets++; + vnet->netdev->stats.tx_bytes += skb->len; + + dev_kfree_skb_any(skb); + + return NETDEV_TX_OK; +} + +static void pci_epf_vnet_link_detect_handle(struct timer_list *timer) +{ + struct pci_epf_vnet *vnet = container_of(timer, struct pci_epf_vnet, + link_detect_timer); + struct ep_queue *tx_queue = &vnet->queue->ep_tx_queue; + struct net_device *netdev = vnet->netdev; + u32 tx_flags = READ_ONCE(tx_queue->flags); + + /* Link detection */ + if ((tx_flags & PCI_EP_VNET_FLAGS_BIT(RC_LINK_ACTIVE)) && + !(netif_carrier_ok(netdev))) { + netif_carrier_on(netdev); + } else if (!(tx_flags & PCI_EP_VNET_FLAGS_BIT(RC_LINK_ACTIVE)) && + (netif_carrier_ok(netdev))) { + netif_carrier_off(netdev); + } + + mod_timer(timer, jiffies + msecs_to_jiffies(LINK_DETECT_PERIOD)); +} + +static int vnet_close(struct net_device *netdev) +{ + struct pci_epf_vnet *vnet = netdev_priv(netdev); + struct ep_queue *rx_queue; + struct ep_queue *tx_queue; + struct pci_epf *epf = vnet->epf; + struct pci_epc *epc = epf->epc; + struct phytium_pcie_ep *priv = epc_get_drvdata(epc); + u32 rx_flags; + + rx_queue = &vnet->queue->ep_rx_queue; + tx_queue = &vnet->queue->ep_tx_queue; + + dma_unmap_page_attrs(epc->dev.parent, epf_cdev_mdata.phys_addr, + MEM_MAX, DMA_FROM_DEVICE, EP_DMA_ATTR); + + pci_epf_vnet_disable_irq(priv); + /* Set EP end offline */ + rx_flags = READ_ONCE(rx_queue->flags); + rx_flags &= ~PCI_EP_VNET_FLAGS_BIT(EP_LINK_ACTIVE); + WRITE_ONCE(rx_queue->flags, rx_flags); + + del_timer(&vnet->link_detect_timer); + + vnet_down(netdev); + + devm_free_irq(&priv->pdev->dev, vnet->rx_irq, vnet); + + return 0; +} + +static int vnet_open(struct net_device *netdev) +{ + struct pci_epf_vnet *vnet = netdev_priv(netdev); + struct pci_epf *epf = vnet->epf; + struct pci_epc *epc; + struct phytium_pcie_ep *priv; + struct ep_queue *rx_queue; + u32 rx_flags; + int ret; + + /* Prevent opening network card devices before binding EPF and EPC */ + if (!epf->epc) + return -ENXIO; + if (!vnet->bind_success) + return -ENXIO; + + epc = epf->epc; + priv = epc_get_drvdata(epc); + + rx_queue = &vnet->queue->ep_rx_queue; + + /* Obtain interrupt number */ + vnet->rx_irq = platform_get_irq(priv->pdev, 1); + if (vnet->rx_irq < 0) { + dev_err(&priv->pdev->dev, + "Failed to obtain interrupt from device tree!\n"); + return vnet->rx_irq; + } + + /* Apply for interruption */ + ret = devm_request_irq(&priv->pdev->dev, vnet->rx_irq, + pci_epf_vnet_rx_irqhandler, IRQF_SHARED, + "pci_ep_vnet", vnet); + if (ret < 0) { + dev_err(&priv->pdev->dev, + "The EP interruption application failed\n"); + return ret; + } + + netif_carrier_off(netdev); + + ret = vnet_up(netdev); + if (ret) { + netdev_err(netdev, "pci_ep_vnet: Cannot open interface, aborting"); + return ret; + } + + timer_setup(&vnet->link_detect_timer, pci_epf_vnet_link_detect_handle, 0); + add_timer(&vnet->link_detect_timer); + + /* Set EP end online */ + rx_flags = READ_ONCE(rx_queue->flags); + rx_flags |= PCI_EP_VNET_FLAGS_BIT(EP_LINK_ACTIVE); + WRITE_ONCE(rx_queue->flags, rx_flags); + + pci_epf_vnet_set_irq(priv, vnet); + + epf_cdev_mdata.phys_addr = dma_map_page_attrs(epc->dev.parent, epf_cdev_mdata.page, 0, + MEM_MAX, DMA_FROM_DEVICE, EP_DMA_ATTR); + + return 0; +} + +static int vnet_init(struct net_device *netdev) +{ + struct pci_epf_vnet *vnet = netdev_priv(netdev); + + dev_info(vnet_to_dev(vnet), "virtual net device initialized.\n"); + return 0; +} + +static const struct net_device_ops vnet_ops = { + .ndo_init = vnet_init, + .ndo_open = vnet_open, + .ndo_stop = vnet_close, + .ndo_start_xmit = vnet_start_xmit, +}; + +static const struct pci_epf_device_id pci_epf_vnet_ids[] = { + { + .name = "pci_epf_vnet", + }, + {}, +}; + +static int epf_cdev_mmap(struct file *filp, struct vm_area_struct *vma) +{ + unsigned long pfn = epf_cdev_mdata.phys_addr >> PAGE_SHIFT; + unsigned long size = vma->vm_end - vma->vm_start; + + if (remap_pfn_range(vma, vma->vm_start, pfn, size, vma->vm_page_prot)) + return -EAGAIN; + + return 0; +} + +static int epf_cdev_open(struct inode *inode, struct file *filp) +{ + return 0; +} + +static int epf_read_rc_mem(u64 rc_addr, u64 len) +{ + int ret; + int timeout; + unsigned long lock_flags; + + if (len < 0 || len > MEM_MAX) + return -EINVAL; + + spin_lock_irqsave(&epf_cdev_mdata.cdev_vnet->rx_lock, lock_flags); + + ret = pci_epf_vnet_rc2ep_dma(epf_cdev_mdata.cdev_vnet, epf_cdev_mdata.phys_addr, + rc_addr, len); + if (ret) { + spin_unlock_irqrestore(&epf_cdev_mdata.cdev_vnet->rx_lock, lock_flags); + netdev_err(epf_cdev_mdata.cdev_vnet->netdev, + "dma transfer failure(err %d)\n", ret); + return ret; + } + + timeout = (mul32(len) / DIVISOR + 1) * 20 * 8; + while (!(pci_epc_dma_status(epf_cdev_mdata.cdev_vnet->epf->epc, + epf_cdev_mdata.cdev_vnet->epf->func_no, DMA_READ) & + DMA_STATUS_DONE)) { + timeout -= 5; + udelay(1); + if (timeout <= 0) { + spin_unlock_irqrestore(&epf_cdev_mdata.cdev_vnet->rx_lock, lock_flags); + netdev_err(epf_cdev_mdata.cdev_vnet->netdev, + "%s:dma transfer timeout!\n", + __func__); + return -ETIME; + } + } + + spin_unlock_irqrestore(&epf_cdev_mdata.cdev_vnet->rx_lock, lock_flags); + + return 0; +} + +int measure_ddr(phys_addr_t hostddr, uint8_t *bmcaddr, u32 size) +{ + int ret = 0; + u32 offset = hostddr & (PAGE_SIZE - 1); + u64 new_base = hostddr - offset; + u64 new_len = size + offset; + u64 read_len = 0; + u8 *ori_addr = bmcaddr; + void *vaddr = memremap(epf_cdev_mdata.phys_addr, MEM_MAX, MEMREMAP_WB); + + while (new_len) { + if (new_len >= MEM_MAX) + read_len = MEM_MAX; + else + read_len = new_len; + + ret = epf_read_rc_mem(new_base, read_len); + if (ret) + return ret; + + new_len -= read_len; + new_base += read_len; + + memcpy(ori_addr, (u8 *)vaddr + offset, read_len - offset); + ori_addr += read_len - offset; + + if (offset) + offset = 0; + } + + memunmap(vaddr); + + return 0; +} +EXPORT_SYMBOL(measure_ddr); + +static long epf_cdev_ioctl(struct file *file, unsigned int cmd, unsigned long arg) +{ + switch (cmd) { + case IOCTL_EPF_READ_MEM: + if (copy_from_user(&ioctl_data, (struct epf_ioctl_data __user *)arg, + sizeof(struct epf_ioctl_data))) + return -EFAULT; + + pr_debug("Received ioctl data: phy_addr=%llx, len=%d\n", + ioctl_data.ktext_addr, ioctl_data.ktext_len); + return epf_read_rc_mem(ioctl_data.ktext_addr, ioctl_data.ktext_len); + default: + return -EINVAL; + } + return 0; +} + +static int epf_cdev_release(struct inode *inode, struct file *file) +{ + pr_debug("epf_char_release\n"); + return 0; +} + +static const struct file_operations epf_fops = { + .owner = THIS_MODULE, + .open = epf_cdev_open, + .release = epf_cdev_release, + .unlocked_ioctl = epf_cdev_ioctl, + .mmap = epf_cdev_mmap +}; + +static int pci_epf_vnet_probe(struct pci_epf *epf, + const struct pci_epf_device_id *id) +{ + struct pci_epf_vnet *vnet; + struct net_device *netdev; + struct device *dev = &epf->dev; + int err; + + netdev = devm_alloc_etherdev(dev, sizeof(*vnet)); + if (!netdev) + return -ENOMEM; + + netdev->netdev_ops = &vnet_ops; + netdev->ethtool_ops = &vnet_ethtool_ops; + + dev_addr_set(netdev, dev_addr); + + /* MTU range: 46 - 3982 */ + netdev->min_mtu = ETH_ZLEN - ETH_HLEN; + netdev->max_mtu = MAX_JUMBO_FRAME_SIZE - (ETH_HLEN + ETH_FCS_LEN); + + vnet = netdev_priv(netdev); + + epf->header = &vnet_header; + vnet->epf = epf; + vnet->netdev = netdev; + + vnet->msg_enable = (1 << debug) - 1; + vnet->link_detect_timer.expires = jiffies + msecs_to_jiffies(LINK_DETECT_PERIOD); + vnet->bind_success = false; + + epf_set_drvdata(epf, vnet); + + pci_ep_vnet_get_defaults(vnet); + + spin_lock_init(&vnet->tx_lock); + spin_lock_init(&vnet->rx_lock); + + err = register_netdev(netdev); + if (err) { + dev_err(&epf->dev, "Cannot register net device, aborting.\n"); + goto netdev_reg_failed; + } + + epf_cdev_mdata.cdev_vnet = vnet; + + netif_napi_add(netdev, &vnet->rx_napi, pci_epf_vnet_rx_poll); + vnet->clean_rc2ep = pci_epf_vnet_clean_ep2rc_irq; + + /* cdev register */ + err = alloc_chrdev_region(&epf_cdev_mdata.dev_num, 0, 1, DEVICE_NAME); + if (err < 0) + goto err_unregister_netdev; + + epf_cdev_mdata.epf_cdev = cdev_alloc(); + if (!epf_cdev_mdata.epf_cdev) { + dev_err(&epf->dev, "Failed to allocate cdev\n"); + unregister_chrdev_region(epf_cdev_mdata.dev_num, 1); + err = -ENOMEM; + goto err_unregister_chrdev; + } + + epf_cdev_mdata.epf_cdev->owner = THIS_MODULE; + + cdev_init(epf_cdev_mdata.epf_cdev, &epf_fops); + if (cdev_add(epf_cdev_mdata.epf_cdev, epf_cdev_mdata.dev_num, 1) == -1) + goto err_cdev_del; + + epf_cdev_mdata.epf_class = class_create("epf_class"); + if (IS_ERR(epf_cdev_mdata.epf_class)) { + err = PTR_ERR(epf_cdev_mdata.epf_class); + goto err_cdev_del; + } + + epf_cdev_mdata.epf_device = device_create(epf_cdev_mdata.epf_class, + NULL, epf_cdev_mdata.dev_num, + NULL, DEVICE_NAME); + if (IS_ERR(epf_cdev_mdata.epf_device)) { + err = PTR_ERR(epf_cdev_mdata.epf_device); + goto err_class_destroy; + } + + epf_cdev_mdata.page = alloc_pages(GFP_KERNEL | __GFP_DMA32, EP_CDEV_PAGE_ORDER); + if (!epf_cdev_mdata.page) { + dev_err(&epf->dev, "failed to allocate %d pages\n", EP_CDEV_PAGE_ORDER); + goto err_device_destroy; + } + + return 0; + +err_device_destroy: + device_destroy(epf_cdev_mdata.epf_class, epf_cdev_mdata.dev_num); +err_class_destroy: + class_destroy(epf_cdev_mdata.epf_class); +err_cdev_del: + cdev_del(epf_cdev_mdata.epf_cdev); +err_unregister_chrdev: + unregister_chrdev_region(epf_cdev_mdata.dev_num, 1); +err_unregister_netdev: + unregister_netdev(netdev); +netdev_reg_failed: + free_netdev(netdev); + return err; +} + +static void pci_epf_vnet_remove(struct pci_epf *epf) +{ + struct pci_epf_vnet *vnet = epf_get_drvdata(epf); + struct net_device *netdev = vnet->netdev; + + netif_napi_del(&vnet->rx_napi); + + unregister_netdev(netdev); + free_netdev(netdev); + + device_destroy(epf_cdev_mdata.epf_class, epf_cdev_mdata.dev_num); + class_destroy(epf_cdev_mdata.epf_class); + cdev_del(epf_cdev_mdata.epf_cdev); + unregister_chrdev_region(epf_cdev_mdata.dev_num, 1); + + __free_pages(epf_cdev_mdata.page, EP_CDEV_PAGE_ORDER); +} + +static int pci_epf_vnet_set_bar(struct pci_epf *epf) +{ + int ret; + struct pci_epf_bar *epf_bar; + struct pci_epc *epc = epf->epc; + struct device *dev = &epf->dev; + struct pci_epf_vnet *vnet = epf_get_drvdata(epf); + enum pci_barno vnet_reg_barno = vnet->vnet_reg_barno; + + /* Set BAR1 */ + epf_bar = &epf->bar[BAR_1]; + + ret = pci_epc_set_bar(epc, epf->func_no, epf->vfunc_no, epf_bar); + if (ret) { + pci_epf_free_space(epf, vnet->reg[vnet_reg_barno], + vnet_reg_barno, PRIMARY_INTERFACE); + dev_err(dev, "Failed to set BAR%d\n", vnet_reg_barno); + } + + /* Set BAR2 */ + epf_bar = &epf->bar[vnet_reg_barno]; + + epf_bar->flags |= upper_32_bits(epf_bar->size) ? + PCI_BASE_ADDRESS_MEM_TYPE_64 : + PCI_BASE_ADDRESS_MEM_TYPE_32; + + ret = pci_epc_set_bar(epc, epf->func_no, epf->vfunc_no, epf_bar); + if (ret) { + pci_epf_free_space(epf, vnet->reg[vnet_reg_barno], + vnet_reg_barno, PRIMARY_INTERFACE); + dev_err(dev, "Failed to set BAR%d\n", vnet_reg_barno); + } + + return 0; +} + +static int pci_epf_vnet_alloc_space(struct pci_epf *epf) +{ + struct pci_epf_vnet *vnet = epf_get_drvdata(epf); + struct device *dev = &epf->dev; + void *base; + enum pci_barno vnet_reg_barno = vnet->vnet_reg_barno; + u64 offset; + size_t alloc_size = 64 * 1024; + + /* Allocate BAR1 space */ + base = pci_epf_alloc_space(epf, alloc_size, BAR_1, + 0, PRIMARY_INTERFACE); + if (!base) { + dev_err(dev, "Failed to allocated register BAR%d space\n", BAR_1); + return -ENOMEM; + } + vnet->reg[BAR_1] = base; + + /* Allocate BAR2 space */ + alloc_size = sizeof(struct pci_ep_queue) % PAGE_SIZE ? + (sizeof(struct pci_ep_queue) / PAGE_SIZE * 4 * PAGE_SIZE) : + (sizeof(struct pci_ep_queue) * 2); + base = pci_epf_alloc_space(epf, alloc_size, vnet_reg_barno, + 0, PRIMARY_INTERFACE); + if (!base) { + dev_err(dev, "Failed to allocated register BAR%d space\n", + vnet_reg_barno); + return -ENOMEM; + } + offset = epf->bar[vnet_reg_barno].phys_addr + & (epf->bar[vnet_reg_barno].size / 2 - 1); + if (offset) + offset = epf->bar[vnet_reg_barno].size / 2 - offset; + epf->bar[vnet_reg_barno].phys_addr = + epf->bar[vnet_reg_barno].phys_addr + offset; + vnet->reg[vnet_reg_barno] = (void *)((u64)base + offset); + + return 0; +} + +static int pci_epf_vnet_bind(struct pci_epf *epf) +{ + int ret; + struct pci_epf_vnet *vnet = epf_get_drvdata(epf); + struct pci_epf_header *header = epf->header; + struct pci_epc *epc = epf->epc; + struct phytium_pcie_ep *priv = epc_get_drvdata(epc); + struct device *dev = &epf->dev; + struct pci_ep_queue *queue; + struct ep_queue *rx_queue; + struct ep_queue *tx_queue; + + if (WARN_ON_ONCE(!epc)) + return -EINVAL; + + if (!vnet) + return -EINVAL; + + vnet->vnet_reg_barno = BAR_2; + + ret = pci_epc_write_header(epc, epf->func_no, epf->vfunc_no, header); + if (ret) { + dev_err(dev, "Configuration header write failed\n"); + return ret; + } + + ret = pci_epf_vnet_alloc_space(epf); + if (ret) + return ret; + + ret = pci_epf_vnet_set_bar(epf); + if (ret) + return ret; + + queue = vnet->reg[vnet->vnet_reg_barno]; + vnet->queue = queue; + rx_queue = &queue->ep_rx_queue; + tx_queue = &queue->ep_tx_queue; + + /* Initialize variables in the bar space */ + WRITE_ONCE(rx_queue->head, 0); + WRITE_ONCE(rx_queue->tail, 0); + WRITE_ONCE(rx_queue->flags, 0); + rx_queue->nb_desc = vnet->rx_ring_size; + + WRITE_ONCE(tx_queue->head, 0); + WRITE_ONCE(tx_queue->tail, 0); + WRITE_ONCE(tx_queue->flags, 0); + tx_queue->nb_desc = vnet->tx_ring_size; + vnet->bind_success = true; + + /* Used to apply for interruption */ + if (!priv->pdev) { + dev_err(dev, "Failed to obtain pdev!\n"); + return -ENODEV; + } + + return 0; +} + +static void pci_epf_vnet_unbind(struct pci_epf *epf) +{ + struct pci_epf_vnet *vnet = epf_get_drvdata(epf); + struct pci_epc *epc = epf->epc; + struct pci_epf_bar *epf_bar; + int bar = vnet->vnet_reg_barno; + + pci_epc_stop(epc); + epf_bar = &epf->bar[bar]; + + if (vnet->reg[bar]) { + pci_epf_free_space(epf, vnet->reg[bar], bar, PRIMARY_INTERFACE); + pci_epc_clear_bar(epc, epf->func_no, epf->vfunc_no, epf_bar); + } + + bar = BAR_1; + epf_bar = &epf->bar[bar]; + + if (vnet->reg[bar]) { + pci_epf_free_space(epf, vnet->reg[bar], bar, PRIMARY_INTERFACE); + pci_epc_clear_bar(epc, epf->func_no, epf->vfunc_no, epf_bar); + } +} + +static struct pci_epf_ops vnet_epf_ops = { + .unbind = pci_epf_vnet_unbind, + .bind = pci_epf_vnet_bind, +}; + +static struct pci_epf_driver pci_epf_vnet = { + .driver.name = "pci_epf_vnet", + .probe = pci_epf_vnet_probe, + .remove = pci_epf_vnet_remove, + .id_table = pci_epf_vnet_ids, + .ops = &vnet_epf_ops, + .owner = THIS_MODULE, +}; + +static int __init pci_epf_vnet_init(void) +{ + int ret; + + ret = pci_epf_register_driver(&pci_epf_vnet); + if (ret) { + pr_err("Failed to register pci epf test driver --> %d\n", ret); + return ret; + } + + return 0; +} +module_init(pci_epf_vnet_init); + +static void __exit pci_epf_vnet_exit(void) +{ + pci_epf_unregister_driver(&pci_epf_vnet); +} +module_exit(pci_epf_vnet_exit); + +MODULE_DESCRIPTION("Virtual network driver based on the pcie interface in EP"); +MODULE_AUTHOR("litongfeng1497@phytium.com.cn"); +MODULE_LICENSE("GPL"); +MODULE_VERSION(DRV_VERSION); diff --git a/drivers/net/ethernet/phytium-pci-vnet/pci-ep-net/pci_epf_vnet.h b/drivers/net/ethernet/phytium-pci-vnet/pci-ep-net/pci_epf_vnet.h new file mode 100644 index 0000000000000..bd9fd14982f63 --- /dev/null +++ b/drivers/net/ethernet/phytium-pci-vnet/pci-ep-net/pci_epf_vnet.h @@ -0,0 +1,282 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * phytium pcie ep vnet Ethernet Controller driver + * + * Copyright (C) 2023-2023 PHYTIUM Corporation + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#ifndef __PCI_EPF_VNET_H__ +#define __PCI_EPF_VNET_H__ + +#include +#include +#include + +#define DRV_MODULE_NAME "pci_epf_vnet" +#define DRV_VERSION "1.0.0" + +/* IRQ type */ +#define IRQ_TYPE_LEGACY 0 +#define IRQ_TYPE_MSI 1 +#define IRQ_TYPE_MSIX 2 + +/* DMA transfer state */ +#define DMA_STATUS_DONE BIT(0) +#define DMA_STATUS_ERROR 0xfffffffe + +/* DMA transmission direction */ +#define DMA_READ 0 +#define DMA_WRITE 1 + +/* Supported Rx Buffer Sizes */ +#define VNET_RXBUFFER_1024 1024 +#define VNET_RXBUFFER_2048 2048 +#define VNET_RXBUFFER_4096 4096 +#define VNET_RXBUFFER_8192 8192 +#define VNET_RXBUFFER_16384 16384 + +#define MAX_JUMBO_FRAME_SIZE 4000 + +/* Ctrl in the bar space descriptor*/ +#define PCI_EP_DMA_DESC_CTRL_PKT_USED_OFFSET 0 /* 1=Valid packet */ + /*0=Invalid packet*/ +#define PCI_EP_DMA_DESC_CTRL_PKT_USED_SIZE 1 +#define PCI_EP_DMA_DESC_CTRL_PKT_ERR_OFFSET 1 /* 1=DMA transfer successful */ + /* 0=DMA transfer failure */ +#define PCI_EP_DMA_DESC_CTRL_PKT_ERR_SIZE 1 +#define PCI_EP_DMA_DESC_CTRL_PKT_ADDR_DIFF_OFFSET 2 /* 1=Move the receiving */ + /* address to higher position */ + /* 0=Move receiving address to lower position */ +#define PCI_EP_DMA_DESC_CTRL_PKT_ADDR_DIFF_SIZE 1 +#define PCI_EP_DMA_DESC_CTRL_PKT_MISALIGN_OFFSET 3 +#define PCI_EP_DMA_DESC_CTRL_PKT_MISALIGN_SIZE 5 +#define PCI_EP_DMA_DESC_CTRL_PKT_RESERVE_OFFSET 8 +#define PCI_EP_DMA_DESC_CTRL_PKT_RESERVE_SIZE 8 +#define PCI_EP_DMA_DESC_CTRL_PKT_LEN_OFFSET 16 /* Packet length */ +#define PCI_EP_DMA_DESC_CTRL_PKT_LEN_SIZE 16 +#define PCI_EP_DMA_DESC_CTRL_PKT_CHECKSUM_OFFSET 32 /* Checksum bit */ +#define PCI_EP_DMA_DESC_CTRL_PKT_CHECKSUM_SIZE 32 + +#define PCI_EP_DMA_DESC_CTRL_PKT_USED BIT(0) +#define PCI_EP_DMA_DESC_CTRL_PKT_ERR BIT(1) +#define PCI_EP_DMA_DESC_CTRL_PKT_ADDR_DIFF BIT(10) + +#define PCI_EP_DMA_DESC_CTRL_FIELD_MASK(name) \ + (((1 << PCI_EP_DMA_DESC_CTRL_PKT_##name##_SIZE) - 1) \ + << PCI_EP_DMA_DESC_CTRL_PKT_##name##_OFFSET) + +#define PCI_EP_DMA_DESC_FIELD_SET(name, value) \ + ((((u64)value) & ((1UL << PCI_EP_DMA_DESC_CTRL_PKT_##name##_SIZE) - 1UL)) \ + << PCI_EP_DMA_DESC_CTRL_PKT_##name##_OFFSET) + +#define PCI_EP_DMA_DESC_FIELD_GET(name, desc) \ + (((desc) >> PCI_EP_DMA_DESC_CTRL_PKT_##name##_OFFSET) \ + & ((1UL << PCI_EP_DMA_DESC_CTRL_PKT_##name##_SIZE) - 1)) + +/* Flags in the bar space */ +#define PCI_EP_VNET_FLAGS_EP_LINK_ACTIVE_OFFSET 0 +#define PCI_EP_VNET_FLAGS_RC_LINK_ACTIVE_OFFSET 1 + +#define PCI_EP_VNET_FLAGS_BIT(name) \ + (1 << PCI_EP_VNET_FLAGS_##name##_OFFSET) + +/* TX/RX descriptor defines */ +#define VNET_DEFAULT_TXD 1024 +#define VNET_MAX_TXD 4096 +#define VNET_MIN_TXD 48 + +#define VNET_DEFAULT_RXD 1024 +#define VNET_MAX_RXD 4096 +#define VNET_MIN_RXD 48 + +#define DIVISOR 0x1000 + +#define MAX_BAR_NUM 6 + +#define DMA_ALIGN_SIZE 16 /* Address alignment value of DMA */ + +#define RX_IRQ 1 + +#define PHYTIUM_PCIE_FUNC_BASE(fn) (((fn) << 14) & GENMASK(16, 14)) +#define PHYTIUM_PCI_WIN0_BASE 0x600 +#define PHYTIUM_PCI_WIN0_TRSL_ADDR0(table) (PHYTIUM_PCI_WIN0_BASE + 0X20 * (table) + 0x8) +#define PHYTIUM_PCI_WIN0_TRSL_ADDR1(table) (PHYTIUM_PCI_WIN0_BASE + 0X20 * (table) + 0xc) + +#define HPB_ERR_EVT_LOG_EN BIT(6) +#define HPB_ERR_EVT_INT_EN BIT(6) +#define HPB_ERR_EVT_INT_CLR 0x0 +#define HPB_MSI_EN 0xF +#define HPB_NS_MSI_SPI_EN 0x1 + +#define PHYTIUM_HPB_REG_ERR_EVT_LOG_EN 0x0C4 +#define PHYTIUM_HPB_REG_ERR_EVT_INT_EN 0x0C8 +#define PHYTIUM_HPB_REG_ERR_EVT_WIC 0x0D0 +#define PHYTIUM_HPB_REG_MSI_EN 0x200 +#define PHYTIUM_HPB_REG_MSI64_HI_ADDR 0x208 +#define PHYTIUM_HPB_REG_MSI64_LO_ADDR 0x20C +#define PHYTIUM_HPB_REG_NS_MSI_SPI_EN 0x608 +#define PHYTIUM_HPB_REG_NS_MSI_DATA 0x60c + +#define LINK_DETECT_PERIOD 500 + +#define EP_SKB_PAD (0) + +#define EP_DMA_ATTR \ + (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING) + +#define EP_GFP_FLAGS \ + (GFP_ATOMIC | __GFP_NOWARN | __GFP_DMA32) + +#define EP_RX_PAGE_ORDER 0 +#define EP_RX_PAGE_SIZE (PAGE_SIZE << EP_RX_PAGE_ORDER) + +#define EP_TX_PAGE_ORDER 0 +#define EP_TX_PAGE_SIZE (PAGE_SIZE << EP_TX_PAGE_ORDER) +#define EP_RXBUFFER_2048 2048 +#define RX_BUFFER_MULTIPLE 64 +#define EP_MAX_FRAME_BUILD_SKB \ + (SKB_WITH_OVERHEAD(EP_RXBUFFER_2048) - EP_SKB_PAD) +struct param_range { + u32 min; + u32 max; + u32 count; +}; + +/* Private structure of EPC */ +struct phytium_pcie_ep { + void __iomem *reg_base; + struct resource *mem_res; + void __iomem *hpb_base; + unsigned int max_regions; + unsigned long ob_region_map; + phys_addr_t *ob_addr; + phys_addr_t irq_phys_addr; + void __iomem *irq_cpu_addr; + unsigned long irq_pci_addr; + u8 irq_pci_fn; + struct pci_epc *epc; + + struct platform_device *pdev; +}; + +/* Shared descriptor */ +struct pci_ep_dma_desc { + u64 addr; + u64 ctrl; +}; + +struct ep_queue { + u32 irq_num; + u32 tail; + u32 head; + u32 nb_desc; + u32 flags; + struct pci_ep_dma_desc dma_desc_base[VNET_MAX_RXD]; +}; + +/* bar space */ +struct pci_ep_queue { + struct ep_queue ep_rx_queue; + struct ep_queue ep_tx_queue; +} __packed; + +struct ep_rx_buffer { + dma_addr_t addr; + struct page *page; + __u16 page_offset; + __u16 pagecnt_bias; +}; + +struct ep_tx_buffer { + dma_addr_t addr; + void *vaddr; + struct page *page; + __u16 page_offset; + __u16 pagecnt_bias; +}; + +/* Structure containing variables used by the shared code */ +struct pci_epf_vnet { + u32 msg_enable ____cacheline_aligned; + + void *reg[MAX_BAR_NUM]; + struct pci_epf *epf; + enum pci_barno vnet_reg_barno; + struct hrtimer timer; + struct timer_list link_detect_timer; + + u32 rx_refill_start; + unsigned int rx_refill_num; + u32 rx_ring_size; /* Shared descriptor size */ + u32 rx_buffer_size; + struct sk_buff **rx_skb; + struct ep_rx_buffer *rx_buffer_info; + int rx_buffer_len; + u32 rx_next_to_alloc; + struct param_range rxd_size; /* Records rx descriptor ring size range */ + + u32 tx_head, tx_tail; + u32 tx_ring_size; + struct param_range txd_size; + struct ep_tx_buffer *tx_buffer_info; + + struct net_device *netdev; + + /* Lock to protect tx */ + spinlock_t tx_lock; + /* Lock to protect rx */ + spinlock_t rx_lock; + + struct napi_struct rx_napi; + + struct pci_ep_queue *queue; + + int (*clean_rc2ep)(struct pci_epf_vnet *vnet, int budget); + int rx_irq; + + bool bind_success; +}; + +enum vnet_state_t { + __VNET_TESTING, + __VNET_RESETTING, + __VNET_DOWN, + __VNET_DISABLED +}; + +static struct pci_epf_header vnet_header = { + .vendorid = PCI_ANY_ID, + .deviceid = PCI_ANY_ID, + .baseclass_code = PCI_CLASS_OTHERS, + .interrupt_pin = PCI_INTERRUPT_INTA, +}; + +struct pci_epf_vnet_data { + enum pci_barno vnet_reg_barno; +}; + +/* dma read module */ +struct epf_ioctl_data { + u64 ktext_addr; + u32 ktext_len; +}; + +struct cdev_mdata { + dev_t dev_num; + struct page *page; + phys_addr_t phys_addr; + struct cdev *epf_cdev; + struct class *epf_class; + struct device *epf_device; + struct pci_epf_vnet *cdev_vnet; +}; + +#define IOCTL_EPF_READ_MEM _IOW('a', 1, struct epf_ioctl_data) +#define EP_CDEV_PAGE_ORDER 7 +#define MEM_MAX (EP_RX_PAGE_SIZE << EP_CDEV_PAGE_ORDER) +#define DEVICE_NAME "epf_cdev" +#endif /* __PCI_EPF_VNET_H__ */ diff --git a/drivers/net/ethernet/phytium-pci-vnet/pci-rc-net/Kconfig b/drivers/net/ethernet/phytium-pci-vnet/pci-rc-net/Kconfig new file mode 100644 index 0000000000000..a0a78bc37a5d4 --- /dev/null +++ b/drivers/net/ethernet/phytium-pci-vnet/pci-rc-net/Kconfig @@ -0,0 +1,18 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Phytium PCI EP virtual network device configuration +# + +config PHYTIUM_PCI_RC_VNET + tristate "Phytium PCI Root Complex Virtual Ethernet Device" + depends on PCI + help + This driver provides support for a virtual Ethernet interface on the + Phytium PCIe Root Complex side. It is designed to work together with + a matching PCIe Endpoint function (such as PHYTIUM_PCI_EPF_VNET) to + enable Ethernet communication over PCIe links. + + If you want to use the Root Complex side of the virtual network interface + over PCIe, say Y or M. + + If unsure, say N. diff --git a/drivers/net/ethernet/phytium-pci-vnet/pci-rc-net/Makefile b/drivers/net/ethernet/phytium-pci-vnet/pci-rc-net/Makefile new file mode 100644 index 0000000000000..a0fb493975c82 --- /dev/null +++ b/drivers/net/ethernet/phytium-pci-vnet/pci-rc-net/Makefile @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0 +# Copyright(c) 2022 - 2025 Phytium Technology Co., Ltd. +# +# Makefile for the Phytium network device drivers. + +obj-$(CONFIG_PHYTIUM_PCI_RC_VNET) += pci_rc_vnet.o diff --git a/drivers/net/ethernet/phytium-pci-vnet/pci-rc-net/pci_rc_vnet.c b/drivers/net/ethernet/phytium-pci-vnet/pci-rc-net/pci_rc_vnet.c new file mode 100644 index 0000000000000..3a1f51677b3a9 --- /dev/null +++ b/drivers/net/ethernet/phytium-pci-vnet/pci-rc-net/pci_rc_vnet.c @@ -0,0 +1,1007 @@ +// SPDX-License-Identifier: GPL-2.0 +/** + * Virtual network driver based on the pcie interface in RC + * + * Copyright (C) 2023 Phytium Corporation + * Author: netgroup@phytium.com.cn + */ + +#include "pci_rc_vnet.h" + +static int debug = 3; +static const char dev_addr[6] = {0x04, 0x0c, 0x00, 0x00, 0x0d, 0x05}; + +static const char pci_rc_vnet_gstrings_stats[][ETH_GSTRING_LEN] = { + "rx_packets", "tx_packets", "rx_bytes", "tx_bytes", "rx_errors", + "tx_errors", "rx_dropped", "tx_dropped", "multicast", "collisions", + "rx_length_errors", "rx_over_errors", "rx_crc_errors", + "rx_frame_errors", "rx_fifo_errors", "rx_missed_errors", + "tx_aborted_errors", "tx_carrier_errors", "tx_fifo_errors", + "tx_heartbeat_errors", "tx_window_errors", +}; + +#define PCI_RC_VNET_NET_STATS_LEN 21 +#define PCI_RC_VNET_STATS_LEN ARRAY_SIZE(pci_rc_vnet_gstrings_stats) + +static const char pci_rc_vnet_gstrings_test[][ETH_GSTRING_LEN] = { +}; + +#define PCI_RC_VNET_TEST_LEN ARRAY_SIZE(pci_rc_vnet_gstrings_test) + +static inline int mul32(int size) +{ + int ret = 0; + + if (size % 32) + ret = size + (32 - size % 32); + else + ret = size; + + return ret; +} + +static struct sk_buff *rc_build_skb(struct rc_rx_buffer *rx_buffer, + unsigned int size) +{ + struct sk_buff *skb; + unsigned int truesize; + void *va; + +#if (PAGE_SIZE < 8192) + truesize = RC_RX_PAGE_SIZE / 2; +#else + truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) + + SKB_DATA_ALIGN(RC_SKB_PAD + size); +#endif + + va = page_address(rx_buffer->page) + rx_buffer->page_offset; + /* prefetch first cache line of first page */ + prefetch(va); + + /* build an skb around the page buffer */ + skb = build_skb(va - RC_SKB_PAD, truesize); + if (unlikely(!skb)) + return NULL; + + /* update pointers within the skb to store the data */ + skb_reserve(skb, RC_SKB_PAD); + __skb_put(skb, size); + + /* update buffer offset */ +#if (PAGE_SIZE < 8192) + rx_buffer->page_offset ^= truesize; +#else + rx_buffer->page_offset += truesize; +#endif + + return skb; +} + +static int pci_rc_vnet_tx_poll(struct pci_rc_vnet_private *tp, int budget) +{ + int len; + int work_done = 0; + u64 ctrl; + int tx_reclaim_start = 0; + int tx_reclaim_num = 0; + struct pci_rc_vnet_dma_desc *tx_desc_ring; + unsigned long tx_reclaim_lock_flags; + + struct pci_dev *pdev = tp->pci_dev; + struct device *dev = &pdev->dev; + int ring_size = tp->params.tx_ring.count; + + if (!(READ_ONCE(tp->tx_queue->flags) & PCI_RC_VNET_FLAGS_EP_LINK)) { + netdev_info(tp->netdev, "EP Device not online!\n"); + return 0; + } + tx_desc_ring = tp->tx_queue->desc_ring; + spin_lock_irqsave(&tp->tx_reclaim_lock, tx_reclaim_lock_flags); + tx_reclaim_start = tp->tx_reclaim_start; + tx_reclaim_num = tp->tx_reclaim_num; + + /* Ensure ctrl is at least as up-to-date as used */ + dma_rmb(); + ctrl = tx_desc_ring[tx_reclaim_start].ctrl; + while ((work_done < budget) && (work_done < tx_reclaim_num) && + (!(ctrl & PCI_RC_VNET_DMA_DESC_CTRL_FIELD_MASK(USED)))) { + len = PCI_RC_VNET_DMA_DESC_FIELD_GET(PKT_LEN, ctrl); + dma_unmap_single(dev, tp->tx_phys_addr_list[tx_reclaim_start], + mul32(len), DMA_TO_DEVICE); + dev_kfree_skb_any(tp->tx_skbuff[tx_reclaim_start]); + tp->tx_skbuff[tx_reclaim_start] = NULL; + work_done++; + tx_reclaim_start++; + if (tx_reclaim_start == ring_size) + tx_reclaim_start = 0; + + if (ctrl & PCI_RC_VNET_DMA_DESC_CTRL_FIELD_MASK(ERR)) { + tp->netdev->stats.tx_dropped++; + } else { + tp->netdev->stats.tx_packets++; + tp->netdev->stats.tx_bytes += len; + } + dma_rmb(); + ctrl = tx_desc_ring[tx_reclaim_start].ctrl; + } + + tp->tx_reclaim_start = tx_reclaim_start; + tp->tx_reclaim_num -= work_done; + spin_unlock_irqrestore(&tp->tx_reclaim_lock, tx_reclaim_lock_flags); + + if (netif_queue_stopped(tp->netdev) && (tp->tx_queue->nb_desc - tp->tx_reclaim_num >= + PCI_RC_VNET_TX_QUEUE_RESTART_THRESHOLD)) { + netif_wake_queue(tp->netdev); + } + + return work_done; +} + +static netdev_tx_t pci_rc_vnet_start_xmit(struct sk_buff *skb, struct net_device *netdev) +{ + u32 tail, head; + u32 crc32; + u64 ctrl; + int len; + char *data; + unsigned long tx_lock_flags; + unsigned long tx_reclaim_lock_flags; + dma_addr_t phys_addr; + struct pci_rc_vnet_private *tp = netdev_priv(netdev); + struct pci_rc_vnet_tx_queue *tx_queue = tp->tx_queue; + struct pci_dev *pdev = tp->pci_dev; + struct device *dev = &pdev->dev; + int ring_size = tp->params.tx_ring.count; + + if (!(READ_ONCE(tp->tx_queue->flags) & PCI_RC_VNET_FLAGS_EP_LINK)) { + netdev_info(tp->netdev, "EP Device not online!\n"); + return NETDEV_TX_BUSY; + } + + data = skb->data; + len = skb->len; + + if (unlikely(tx_queue->nb_desc - tp->tx_reclaim_num <= + PCI_RC_VNET_TX_QUEUE_STOP_THRESHOLD)) { + netif_stop_queue(tp->netdev); + netdev_info(tp->netdev, + "desc queue reclaim is not finished\n"); + return NETDEV_TX_BUSY; + } + + spin_lock_irqsave(&tp->tx_lock, tx_lock_flags); + + tail = READ_ONCE(tx_queue->tail); + head = READ_ONCE(tx_queue->head); + + crc32 = crc32_le(~0, data, len); + + phys_addr = dma_map_single(dev, data, mul32(len), DMA_TO_DEVICE); + if (dma_mapping_error(dev, phys_addr)) { + spin_unlock_irqrestore(&tp->tx_lock, tx_lock_flags); + netdev_err(tp->netdev, "tx map failed\n"); + return NETDEV_TX_BUSY; + } + + dma_sync_single_range_for_device(dev, phys_addr, 0, mul32(len), DMA_TO_DEVICE); + + ctrl = PCI_RC_VNET_DMA_DESC_SET(USED, 1) | + PCI_RC_VNET_DMA_DESC_SET(PKT_LEN, len) | + PCI_RC_VNET_DMA_DESC_SET(CHECKSUM, crc32); + + + if (unlikely(head == (tail + 1) % tx_queue->nb_desc)) { + spin_unlock_irqrestore(&tp->tx_lock, tx_lock_flags); + netif_stop_queue(tp->netdev); + dma_unmap_single(dev, phys_addr, mul32(len), DMA_TO_DEVICE); + netdev_info(tp->netdev, "tx_desc_queue_is_full\n"); + return NETDEV_TX_BUSY; + } + + tx_queue->desc_ring[tail].addr = phys_addr; + tx_queue->desc_ring[tail].ctrl = ctrl; + + /* Make descriptor updates visible to device */ + wmb(); + + spin_lock_irqsave(&tp->tx_reclaim_lock, tx_reclaim_lock_flags); + tp->tx_reclaim_num++; + spin_unlock_irqrestore(&tp->tx_reclaim_lock, tx_reclaim_lock_flags); + + tp->tx_skbuff[tail] = skb; + tp->tx_phys_addr_list[tail] = phys_addr; + + tail = (tail + 1) % ring_size; + WRITE_ONCE(tx_queue->tail, tail); + + writel(SEND_MSI_IRQ, tp->msi_irq_addr); + + spin_unlock_irqrestore(&tp->tx_lock, tx_lock_flags); + + return NETDEV_TX_OK; +} + +static bool rc_alloc_mapped_page(struct pci_rc_vnet_private *tp, + struct rc_rx_buffer *rx_buffer_info) +{ + dma_addr_t paddr; + struct page *page = rx_buffer_info->page; + struct pci_rc_vnet_dma_desc *rx_desc_ring; + + rx_desc_ring = tp->rx_queue->desc_ring; + + /* since we are recycling buffers we should seldom need to alloc */ + if (likely(page)) + return true; + + page = __dev_alloc_pages(RC_GFP_FLAGS, RC_RX_PAGE_ORDER); + if (unlikely(!page)) { + netdev_err(tp->netdev, "rx alloc page failed\n"); + rx_buffer_info->page = NULL; + return false; + } + + paddr = dma_map_page_attrs(&tp->pci_dev->dev, page, 0, + RC_RX_PAGE_SIZE, + DMA_FROM_DEVICE, RC_RX_DMA_ATTR); + if (dma_mapping_error(&tp->pci_dev->dev, paddr)) { + __free_pages(page, RC_RX_PAGE_ORDER); + rx_buffer_info->page = NULL; + return false; + } + + rx_buffer_info->addr = paddr; + rx_buffer_info->page = page; + rx_buffer_info->pagecnt_bias = 1; + rx_buffer_info->page_offset = RC_SKB_PAD; + + return true; +} + +static inline int rc_calc_rx_buf_len(void) +{ +#if (PAGE_SIZE < 8192) + return rounddown(RC_MAX_FRAME_BUILD_SKB, RX_BUFFER_MULTIPLE); +#endif + return rounddown(RC_RXBUFFER_2048, RX_BUFFER_MULTIPLE); +} + +static void pci_rc_vnet_rx_refill(struct pci_rc_vnet_private *tp, bool init) +{ + unsigned int entry, space; + struct rc_rx_buffer *rx_buffer_info; + struct pci_rc_vnet_dma_desc *rx_desc_ring; + int ring_size = tp->params.rx_ring.count; + + rx_desc_ring = tp->rx_queue->desc_ring; + + if (init) + space = ring_size - 1; + else + space = CIRC_SPACE(tp->rx_refill_start, + READ_ONCE(tp->queue->rx_queue.head), + ring_size); + + while (space > 0) { + entry = tp->rx_refill_start & (ring_size - 1); + rx_buffer_info = &tp->rx_buffer_info[entry]; + if (!rc_alloc_mapped_page(tp, rx_buffer_info)) + break; + /* sync the buffer for use by the device */ + dma_sync_single_range_for_device(&tp->pci_dev->dev, rx_buffer_info->addr, + rx_buffer_info->page_offset, + tp->rx_buffer_len, DMA_FROM_DEVICE); + rx_desc_ring[tp->rx_refill_start].addr = + rx_buffer_info->addr + rx_buffer_info->page_offset; + rx_desc_ring[tp->rx_refill_start].ctrl = 0; + + dma_wmb(); + + tp->rx_refill_start = (tp->rx_refill_start + 1) & (ring_size - 1); + space--; + } + + tp->rx_next_to_alloc = tp->rx_refill_start; +} + +static int pci_rc_vnet_rx_buff_alloc(struct pci_rc_vnet_private *tp) +{ + int ring_size = tp->params.rx_ring.count; + struct pci_rc_vnet_dma_desc *desc_ring; + + tp->rx_buffer_len = rc_calc_rx_buf_len(); + desc_ring = tp->rx_queue->desc_ring; + tp->rx_buffer_info = vzalloc(ring_size * sizeof(struct rc_rx_buffer)); + if (!tp->rx_buffer_info) { + netdev_err(tp->netdev, "Unable to allocate rx_buffer_info memory\n"); + return -ENOMEM; + } + + /* Pre allocated rx skb */ + pci_rc_vnet_rx_refill(tp, true); + + return 0; +} + +static void pci_rc_vnet_rx_buff_free(struct pci_rc_vnet_private *tp) +{ + int i; + int ring_size = tp->params.rx_ring.count; + + for (i = 0; i < ring_size; i++) { + if (tp->rx_buffer_info) { + if (tp->rx_buffer_info[i].page) { + dma_sync_single_range_for_cpu(&tp->pci_dev->dev, + tp->rx_buffer_info[i].addr, + 0, RC_RX_PAGE_SIZE, + DMA_FROM_DEVICE); + /* free resources associated with mapping */ + dma_unmap_page_attrs(&tp->pci_dev->dev, + tp->rx_buffer_info[i].addr, + RC_RX_PAGE_SIZE, + DMA_FROM_DEVICE, + RC_RX_DMA_ATTR); + __page_frag_cache_drain(tp->rx_buffer_info[i].page, + tp->rx_buffer_info[i].pagecnt_bias); + } + } + } + + vfree(tp->rx_buffer_info); + tp->rx_buffer_info = NULL; +} + +static int pci_rc_vnet_tx_buff_alloc(struct pci_rc_vnet_private *tp) +{ + int ret = 0, i; + int ring_size = tp->params.tx_ring.count; + dma_addr_t *tx_phys_addr_list; + struct sk_buff **tx_skbuff; + + tx_skbuff = kcalloc(ring_size, sizeof(struct sk_buff *), GFP_KERNEL); + if (!tx_skbuff) { + ret = -ENOMEM; + goto err_failed_alloc_skbuff; + } + for (i = 0; i < ring_size; i++) + tx_skbuff[i] = NULL; + + tp->tx_skbuff = tx_skbuff; + + tx_phys_addr_list = kmalloc_array(ring_size, sizeof(dma_addr_t), GFP_KERNEL); + if (!tx_phys_addr_list) { + ret = -ENOMEM; + goto err_free_skbuff; + } + tp->tx_phys_addr_list = tx_phys_addr_list; + return 0; + +err_free_skbuff: + kfree(tp->tx_skbuff); +err_failed_alloc_skbuff: + return ret; +} + +static void pci_rc_vnet_tx_buff_clean(struct pci_rc_vnet_private *tp) +{ + int len, i; + int ring_size = tp->params.tx_ring.count; + struct pci_dev *pdev = tp->pci_dev; + struct device *dev = &pdev->dev; + unsigned long tx_lock_flag; + unsigned long tx_reclaim_lock_flag; + u32 tx_tail; + + spin_lock_irqsave(&tp->tx_lock, tx_lock_flag); + spin_lock_irqsave(&tp->tx_reclaim_lock, tx_reclaim_lock_flag); + + for (i = 0; i < ring_size; i++) { + if (tp->tx_skbuff[i]) { + len = tp->tx_skbuff[i]->len; + dma_unmap_single(dev, tp->tx_phys_addr_list[i], + mul32(len), DMA_TO_DEVICE); + tp->tx_queue->desc_ring[i].ctrl = 0; + dev_kfree_skb_any(tp->tx_skbuff[i]); + } + } + tp->tx_reclaim_num = 0; + tx_tail = READ_ONCE(tp->tx_queue->tail); + tp->tx_reclaim_start = tx_tail; + WRITE_ONCE(tp->tx_queue->head, tx_tail); + + spin_unlock_irqrestore(&tp->tx_reclaim_lock, tx_reclaim_lock_flag); + spin_unlock_irqrestore(&tp->tx_lock, tx_lock_flag); +} + +static void pci_rc_vnet_tx_buff_free(struct pci_rc_vnet_private *tp) +{ + kfree(tp->tx_skbuff); + tp->tx_skbuff = NULL; + kfree(tp->tx_phys_addr_list); + tp->tx_phys_addr_list = NULL; +} + +static int pci_rc_vnet_up(struct pci_rc_vnet_private *tp) +{ + int err = 0; + + /* Apply for DMA buffer */ + if (pci_rc_vnet_tx_buff_alloc(tp)) { + err = -ENOMEM; + netdev_err(tp->netdev, "Failed to allocate tx buffer\n"); + goto err_failed_alloc_tx_buff; + } + if (pci_rc_vnet_rx_buff_alloc(tp)) { + err = -ENOMEM; + netdev_err(tp->netdev, "Failed to allocate rx buffer\n"); + goto err_free_tx_buff; + } + + tp->netdev->flags |= IFF_UP; + + netif_wake_queue(tp->netdev); + return err; + +err_free_tx_buff: + pci_rc_vnet_tx_buff_free(tp); + +err_failed_alloc_tx_buff: + return err; +} + +static void pci_rc_vnet_down(struct pci_rc_vnet_private *tp) +{ + netif_stop_queue(tp->netdev); + + netif_carrier_off(tp->netdev); + + msleep(200); + pci_rc_vnet_tx_buff_clean(tp); + pci_rc_vnet_tx_buff_free(tp); + pci_rc_vnet_rx_buff_free(tp); +} + +static bool rc_can_reuse_rx_page(struct rc_rx_buffer *rx_buffer) +{ + unsigned int pagecnt_bias = rx_buffer->pagecnt_bias; + struct page *page = rx_buffer->page; + + /* avoid re-using remote and pfmemalloc pages */ + if (!dev_page_is_reusable(page)) + return false; + +#if (PAGE_SIZE < 8192) + /* if we are only owner of page we can reuse it */ + if (unlikely((page_ref_count(page) - pagecnt_bias) > 1)) + return false; +#else +#define RC_LAST_OFFSET \ + (SKB_WITH_OVERHEAD(PAGE_SIZE) - RC_RXBUFFER_2048) + + if (rx_buffer->page_offset > RC_LAST_OFFSET) + return false; +#endif + + /* If we have drained the page fragment pool we need to update + * the pagecnt_bias and page count so that we fully restock the + * number of references the driver holds. + */ + if (unlikely(!pagecnt_bias)) { + page_ref_add(page, USHRT_MAX); + rx_buffer->pagecnt_bias = USHRT_MAX; + } + + return true; +} + +static void rc_reuse_rx_page(struct pci_rc_vnet_private *tp, + struct rc_rx_buffer *old_buff) +{ + struct rc_rx_buffer *new_buff; + u16 nta = tp->rx_next_to_alloc; + int ring_size = tp->params.rx_ring.count; + struct pci_rc_vnet_dma_desc *rx_desc_ring; + + rx_desc_ring = tp->rx_queue->desc_ring; + + new_buff = &tp->rx_buffer_info[nta & (ring_size - 1)]; + + /* update, and store next to alloc */ + nta++; + tp->rx_next_to_alloc = (nta < ring_size) ? nta : 0; + + /* Transfer page from old buffer to new buffer. + * Move each member individually to avoid possible store + * forwarding stalls. + */ + new_buff->addr = old_buff->addr; + new_buff->page = old_buff->page; + new_buff->page_offset = old_buff->page_offset; + new_buff->pagecnt_bias = old_buff->pagecnt_bias; +} + +static void rc_put_rx_buffer(struct pci_rc_vnet_private *tp, + struct rc_rx_buffer *rx_buffer_info) +{ + if (rc_can_reuse_rx_page(rx_buffer_info)) { + /* hand second half of page back to the ring */ + rc_reuse_rx_page(tp, rx_buffer_info); + } else { + dma_unmap_page_attrs(&tp->pci_dev->dev, + rx_buffer_info->addr, + RC_RX_PAGE_SIZE, + DMA_FROM_DEVICE, + RC_RX_DMA_ATTR); + __page_frag_cache_drain(rx_buffer_info->page, + rx_buffer_info->pagecnt_bias); + } + + /* clear contents of rx_buffer */ + rx_buffer_info->page = NULL; +} + +static int pci_rc_vnet_rx(struct pci_rc_vnet_private *tp, u32 head, u32 tail) +{ + u32 desc_crc, pkt_crc; + int used; + u64 ctrl; + int pkt_len; + void *pkt_data; + + struct rc_rx_buffer *rx_buffer_info; + struct pci_rc_vnet_rx_queue *rx_queue; + struct pci_rc_vnet_dma_desc *rx_desc_ring; + struct sk_buff *skb; + int ring_size = tp->params.tx_ring.count; + unsigned int pkt_num, count = 0; + + rx_queue = tp->rx_queue; + rx_desc_ring = rx_queue->desc_ring; + + if (tail >= head) + pkt_num = tail - head; + else + pkt_num = ring_size - head + tail; + + for (count = 0; count < pkt_num; count++) { + /* Ensure ctrl is at least as up-to-date as rxused */ + dma_rmb(); + + ctrl = rx_desc_ring[head].ctrl; + used = PCI_RC_VNET_DMA_DESC_FIELD_GET(USED, ctrl); + + pkt_len = PCI_RC_VNET_DMA_DESC_FIELD_GET(PKT_LEN, ctrl); + + /* No package to receive */ + if (!used) + break; + + /* get rx buffer */ + rx_buffer_info = &tp->rx_buffer_info[head & (ring_size - 1)]; + if (!rx_buffer_info->page) { + netdev_err(tp->netdev, + "%s: The page used is null!\n", __func__); + break; + } + rx_buffer_info->pagecnt_bias--; + + dma_sync_single_range_for_cpu(&tp->pci_dev->dev, rx_buffer_info->addr, + rx_buffer_info->page_offset, + pkt_len, DMA_FROM_DEVICE); + + pkt_data = page_address(rx_buffer_info->page) + + rx_buffer_info->page_offset - RC_SKB_PAD; + pkt_crc = crc32_le(~0, (unsigned char *)pkt_data, pkt_len); + desc_crc = PCI_RC_VNET_DMA_DESC_FIELD_GET(CHECKSUM, ctrl); + if (desc_crc != pkt_crc) { + tp->netdev->stats.rx_errors++; + rx_buffer_info->pagecnt_bias++; + rc_put_rx_buffer(tp, rx_buffer_info); + goto refresh; + } + + skb = rc_build_skb(rx_buffer_info, pkt_len); + if (unlikely(!skb)) { + netdev_err(tp->netdev, "rx build skb failed\n"); + tp->netdev->stats.rx_dropped++; + rx_buffer_info->pagecnt_bias++; + break; + } + + rc_put_rx_buffer(tp, rx_buffer_info); + + skb->protocol = eth_type_trans(skb, tp->netdev); + /* Upload protocol stack */ + netif_receive_skb(skb); + + tp->netdev->stats.rx_packets++; + tp->netdev->stats.rx_bytes += pkt_len; +refresh: + head = (head + 1) % ring_size; + } + + WRITE_ONCE(rx_queue->head, head); + + pci_rc_vnet_rx_refill(tp, false); + + return count; +} + +static void pci_rc_vnet_get_drvinfo(struct net_device *netdev, struct ethtool_drvinfo *info) +{ + struct pci_rc_vnet_private *tp = netdev_priv(netdev); + + strscpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver)); + strscpy(info->version, DRV_VERSION, sizeof(info->version)); + strscpy(info->bus_info, pci_name(tp->pci_dev), + sizeof(info->bus_info)); +} + +static u32 pci_rc_vnet_get_msglevel(struct net_device *netdev) +{ + struct pci_rc_vnet_private *tp = netdev_priv(netdev); + + return tp->msg_enable; +} + +static void pci_rc_vnet_set_msglevel(struct net_device *netdev, u32 value) +{ + struct pci_rc_vnet_private *tp = netdev_priv(netdev); + + tp->msg_enable = value; +} + +static u32 pci_rc_vnet_get_link(struct net_device *netdev) +{ + struct pci_rc_vnet_private *tp = netdev_priv(netdev); + u32 tx_flags; + + tx_flags = READ_ONCE(tp->tx_queue->flags); + return tx_flags & PCI_RC_VNET_FLAGS_EP_LINK; +} + +static void pci_rc_vnet_get_strings(struct net_device *netdev, u32 stringset, u8 *data) +{ + switch (stringset) { + case ETH_SS_TEST: + memcpy(data, *pci_rc_vnet_gstrings_test, sizeof(pci_rc_vnet_gstrings_test)); + break; + case ETH_SS_STATS: + memcpy(data, *pci_rc_vnet_gstrings_stats, sizeof(pci_rc_vnet_gstrings_stats)); + break; + } +} + +static int pci_rc_vnet_get_sset_count(struct net_device *netdev, int sset) +{ + switch (sset) { + case ETH_SS_TEST: + return PCI_RC_VNET_TEST_LEN; + case ETH_SS_STATS: + return PCI_RC_VNET_STATS_LEN; + default: + return -EOPNOTSUPP; + } +} + +static void pci_rc_vnet_get_ethtool_stats(struct net_device *netdev, + struct ethtool_stats *stats, u64 *data) +{ + int i; + + for (i = 0; i < PCI_RC_VNET_NET_STATS_LEN; i++) + data[i] = ((unsigned long *)&netdev->stats)[i]; +} + +static const struct ethtool_ops pci_rc_vnet_ethtool_ops = { + .get_drvinfo = pci_rc_vnet_get_drvinfo, + .get_msglevel = pci_rc_vnet_get_msglevel, + .set_msglevel = pci_rc_vnet_set_msglevel, + .get_link = pci_rc_vnet_get_link, + .get_strings = pci_rc_vnet_get_strings, + .get_ethtool_stats = pci_rc_vnet_get_ethtool_stats, + .get_sset_count = pci_rc_vnet_get_sset_count, +}; + +static int pci_rc_vnet_open(struct net_device *dev) +{ + int err; + struct pci_rc_vnet_private *tp = netdev_priv(dev); + struct pci_rc_vnet_rx_queue *rx_queue; + u32 rx_flags; + + rx_queue = tp->rx_queue; + WRITE_ONCE(rx_queue->head, READ_ONCE(rx_queue->tail)); + + netif_carrier_off(dev); + err = pci_rc_vnet_up(tp); + if (err) + goto err; + + hrtimer_start(&tp->link_detect_timer, ns_to_ktime(PCI_RC_VNET_LINK_DETECT_PERIOD_NS), + HRTIMER_MODE_REL_SOFT); + hrtimer_start(&tp->receive_timer, ns_to_ktime(PCI_RC_VNET_RECEIVE_PERIOD_NS), + HRTIMER_MODE_REL_SOFT); + hrtimer_start(&tp->tx_poll_timer, ns_to_ktime(PCI_RC_VNET_TX_POLL_PERIOD_NS), + HRTIMER_MODE_REL_SOFT); + rx_flags |= PCI_RC_VNET_FLAGS_RC_LINK; + WRITE_ONCE(tp->rx_queue->flags, rx_flags); + + return 0; + +err: + return err; +} + +static int pci_rc_vnet_close(struct net_device *netdev) +{ + struct pci_rc_vnet_private *tp = netdev_priv(netdev); + u32 rx_flags; + + rx_flags = READ_ONCE(tp->rx_queue->flags); + rx_flags &= ~PCI_RC_VNET_FLAGS_RC_LINK; + WRITE_ONCE(tp->rx_queue->flags, rx_flags); + + hrtimer_cancel(&tp->link_detect_timer); + hrtimer_cancel(&tp->receive_timer); + hrtimer_cancel(&tp->tx_poll_timer); + + pci_rc_vnet_down(tp); + + return 0; +} + +static const struct net_device_ops pci_rc_netdev_ops = { + .ndo_open = pci_rc_vnet_open, + .ndo_stop = pci_rc_vnet_close, + .ndo_start_xmit = pci_rc_vnet_start_xmit, +}; + +static void pci_rc_vnet_get_defaults(struct pci_rc_vnet_private *tp) +{ + struct param_range rx_ring = { .min = 64, .max = 2048, .count = 512 }; + struct param_range tx_ring = { .min = 64, .max = 2048, .count = 512 }; + + tp->params.rx_ring = rx_ring; + tp->params.tx_ring = tx_ring; + tp->rx_buffer_len = PCI_RC_VNET_RXBUFFER_2048; + + tp->tx_reclaim_start = READ_ONCE(tp->tx_queue->tail); + tp->tx_reclaim_num = 0; + tp->rx_refill_start = READ_ONCE(tp->rx_queue->head); + tp->rx_next_to_alloc = tp->rx_refill_start; + + tp->params.tx_ring.count = tp->tx_queue->nb_desc; + tp->params.rx_ring.count = tp->rx_queue->nb_desc; +} + +static enum hrtimer_restart link_detect_timer_handle(struct hrtimer *timer) +{ + struct pci_rc_vnet_private *tp = + container_of(timer, struct pci_rc_vnet_private, link_detect_timer); + int ret; + + ret = READ_ONCE(tp->tx_queue->flags) & PCI_RC_VNET_FLAGS_EP_LINK; + if (ret && !(netif_carrier_ok(tp->netdev))) + netif_carrier_on(tp->netdev); + else if (!ret && netif_carrier_ok(tp->netdev)) + netif_carrier_off(tp->netdev); + + hrtimer_forward_now(timer, ns_to_ktime(PCI_RC_VNET_LINK_DETECT_PERIOD_NS)); + return HRTIMER_RESTART; +} + +static enum hrtimer_restart pci_epf_vnet_tx_poll_callback(struct hrtimer *timer) +{ + struct pci_rc_vnet_private *tp = + container_of(timer, struct pci_rc_vnet_private, tx_poll_timer); + int tx_reclaim_start, head, budget; + int ring_size = tp->params.tx_ring.count; + + tx_reclaim_start = READ_ONCE(tp->tx_reclaim_start); + head = READ_ONCE(tp->tx_queue->head); + + if (tx_reclaim_start != head) { + if (tx_reclaim_start < head) + budget = head - tx_reclaim_start; + else + budget = ring_size - tx_reclaim_start + head; + + pci_rc_vnet_tx_poll(tp, budget); + } + + hrtimer_forward_now(timer, ns_to_ktime(PCI_RC_VNET_TX_POLL_PERIOD_NS)); + return HRTIMER_RESTART; +} + +static enum hrtimer_restart pci_epf_vnet_ep2rc_callback(struct hrtimer *timer) +{ + u32 head; + u32 tail; + int ret = 0; + struct pci_rc_vnet_private *tp = + container_of(timer, struct pci_rc_vnet_private, receive_timer); + struct pci_rc_vnet_rx_queue *rx_queue; + + rx_queue = tp->rx_queue; + head = READ_ONCE(rx_queue->head); + tail = READ_ONCE(rx_queue->tail); + + /* Receive packets */ + ret = READ_ONCE(tp->tx_queue->flags) & PCI_RC_VNET_FLAGS_EP_LINK; + if (ret) { + if (head != tail) { + ret = pci_rc_vnet_rx(tp, head, tail); + if (!ret) { + netdev_err(tp->netdev, "data packet not received\n"); + goto next_cycle; + } + } + } + +next_cycle: + hrtimer_forward_now(timer, ns_to_ktime(PCI_RC_VNET_RECEIVE_PERIOD_NS)); + return HRTIMER_RESTART; +} + +static int pci_rc_vnet_probe(struct pci_dev *pdev, const struct pci_device_id *ent) +{ + int err; + int rc; + enum pci_barno bar; + struct net_device *netdev; + resource_size_t mmio_start, mmio_len; + struct pci_rc_vnet_private *tp; + uintptr_t msi_irq_addr; + enum pci_barno bar_index = BAR_2; + + if (pci_is_bridge(pdev)) + return -ENODEV; + + err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); + if (err) { + dev_err(&pdev->dev, "dma_set_mask failed\n"); + return err; + } + + netdev = alloc_etherdev(sizeof(struct pci_rc_vnet_private)); + + if (!netdev) + return -ENOMEM; + + mmio_start = pci_resource_start(pdev, 0); + mmio_len = pci_resource_len(pdev, 0); + + SET_NETDEV_DEV(netdev, &pdev->dev); + + netdev->mem_start = mmio_start; + netdev->mem_end = mmio_start + mmio_len; + + netdev->features &= ~NETIF_F_HW_CSUM; + + dev_addr_set(netdev, dev_addr); + + netdev->netdev_ops = &pci_rc_netdev_ops; + netdev->ethtool_ops = &pci_rc_vnet_ethtool_ops; + + netdev->min_mtu = ETH_ZLEN - ETH_HLEN; + netdev->max_mtu = MAX_JUMBO_FRAME_SIZE - (ETH_HLEN + ETH_FCS_LEN); + + tp = netdev_priv(netdev); + tp->pci_dev = pdev; + tp->netdev = netdev; + tp->msg_enable = (1 << debug) - 1; + + err = pci_enable_device(pdev); + if (err) { + netdev_err(tp->netdev, "Cannot enable PCI device\n"); + goto err_free_device; + } + + err = pci_request_regions(pdev, DRV_MODULE_NAME); + if (err) { + netdev_err(tp->netdev, "Cannot obtain PCI resources\n"); + goto err_disable_pdev; + } + + pci_set_master(pdev); + + for (bar = BAR_0; bar <= BAR_5; bar++) { + if (pci_resource_flags(pdev, bar) & IORESOURCE_MEM) + tp->bar[bar] = pci_ioremap_bar(pdev, bar); + } + + tp->queue = tp->bar[bar_index]; + msi_irq_addr = (uintptr_t)tp->bar[BAR_1]; + tp->msi_irq_addr = (void *)msi_irq_addr; + + tp->tx_queue = &tp->queue->tx_queue; + if (!tp->tx_queue) { + err = -ENOMEM; + netdev_err(tp->netdev, + "Cannot perform PCI tp without BAR%d\n", bar_index); + goto err_iounmap; + } + tp->rx_queue = &tp->queue->rx_queue; + if (!tp->rx_queue) { + err = -ENOMEM; + netdev_err(tp->netdev, + "Cannot perform PCI tp without BAR%d\n", bar_index); + goto err_iounmap; + } + + pci_set_drvdata(pdev, tp); + + /* Register as a network device */ + rc = register_netdev(netdev); + + if (rc) { + netdev_err(tp->netdev, "Unable to register_netdev\n"); + goto err_iounmap; + } + pci_rc_vnet_get_defaults(tp); + + spin_lock_init(&tp->tx_lock); + spin_lock_init(&tp->tx_reclaim_lock); + spin_lock_init(&tp->rx_lock); + + hrtimer_init(&tp->link_detect_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_SOFT); + tp->link_detect_timer.function = link_detect_timer_handle; + + hrtimer_init(&tp->receive_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_SOFT); + tp->receive_timer.function = pci_epf_vnet_ep2rc_callback; + + hrtimer_init(&tp->tx_poll_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_SOFT); + tp->tx_poll_timer.function = pci_epf_vnet_tx_poll_callback; + + return 0; + +err_iounmap: + for (bar = BAR_0; bar <= BAR_5; bar++) { + if (tp->bar[bar]) + pci_iounmap(pdev, tp->bar[bar]); + } + +err_disable_pdev: + pci_disable_device(pdev); + +err_free_device: + free_netdev(netdev); + return err; +} + +static void pci_rc_vnet_remove(struct pci_dev *pdev) +{ + enum pci_barno bar; + struct pci_rc_vnet_private *tp = pci_get_drvdata(pdev); + struct net_device *net_dev = tp->netdev; + + unregister_netdev(net_dev); + + for (bar = BAR_0; bar <= BAR_5; bar++) { + if (tp->bar[bar]) + pci_iounmap(pdev, tp->bar[bar]); + } + + pci_release_regions(pdev); + free_netdev(net_dev); + pci_disable_device(pdev); +} + +static const struct pci_device_id pci_rc_vnet_tbl[] = { + { PCI_DEVICE(0x16c3, 0xedda) }, + { } +}; + +static struct pci_driver pci_rc_vnet_driver = { + .name = DRV_MODULE_NAME, + .id_table = pci_rc_vnet_tbl, + .probe = pci_rc_vnet_probe, + .remove = pci_rc_vnet_remove, +}; + +module_pci_driver(pci_rc_vnet_driver); +MODULE_DESCRIPTION("Virtual network driver based on the pcie interface in RC"); +MODULE_AUTHOR("litongfeng1497@phytium.com.cn"); +MODULE_LICENSE("GPL"); +MODULE_VERSION(DRV_VERSION); diff --git a/drivers/net/ethernet/phytium-pci-vnet/pci-rc-net/pci_rc_vnet.h b/drivers/net/ethernet/phytium-pci-vnet/pci-rc-net/pci_rc_vnet.h new file mode 100644 index 0000000000000..55ce55d29ee66 --- /dev/null +++ b/drivers/net/ethernet/phytium-pci-vnet/pci-rc-net/pci_rc_vnet.h @@ -0,0 +1,231 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * phytium pcie ep vnet Ethernet Controller driver + * + * Copyright (C) 2023-2025 PHYTIUM Corporation + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#ifndef __PCI_RC_VNET_H +#define __PCI_RC_VNET_H +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define DRV_MODULE_NAME "pci_rc_vnet" +#define DRV_VERSION "1.0.1" + +#define PCI_ENDPOINT_VNET_IRQ_NUMBER 0x00 +#define PCI_ENDPOINT_VNET_QUEUE_TAIL 0x04 +#define PCI_ENDPOINT_VNET_QUEUE_HEAD 0x08 +#define PCI_ENDPOINT_VNET_DMA_DESC_BASE 0x0c + +#define PCI_ENDPOINT_VNET_DMA_DESC_LOWER_ADDR 0x00 +#define PCI_ENDPOINT_VNET_DMA_DESC_UPPER_ADDR 0x04 +#define PCI_ENDPOINT_VNET_DMA_DESC_CTRL 0x08 + +#define PCI_RC_VNET_DMA_DESC_CTRL_PKT_USED_OFFSET 0 +#define PCI_RC_VNET_DMA_DESC_CTRL_PKT_USED_SIZE 1 +#define PCI_RC_VNET_DMA_DESC_CTRL_PKT_ERR_OFFSET 1 +#define PCI_RC_VNET_DMA_DESC_CTRL_PKT_ERR_SIZE 1 +#define PCI_RC_VNET_DMA_DESC_CTRL_PKT_ADDR_DIFF_OFFSET 2 +#define PCI_RC_VNET_DMA_DESC_CTRL_PKT_ADDR_DIFF_SIZE 1 +#define PCI_RC_VNET_DMA_DESC_CTRL_PKT_MISALIGN_OFFSET 3 +#define PCI_RC_VNET_DMA_DESC_CTRL_PKT_MISALIGN_SIZE 5 +#define PCI_RC_VNET_DMA_DESC_CTRL_PKT_RESERVE_OFFSET 8 +#define PCI_RC_VNET_DMA_DESC_CTRL_PKT_RESERVE_SIZE 8 +#define PCI_RC_VNET_DMA_DESC_CTRL_PKT_PKT_LEN_OFFSET 16 +#define PCI_RC_VNET_DMA_DESC_CTRL_PKT_PKT_LEN_SIZE 16 +#define PCI_RC_VNET_DMA_DESC_CTRL_PKT_CHECKSUM_OFFSET 32 +#define PCI_RC_VNET_DMA_DESC_CTRL_PKT_CHECKSUM_SIZE 32 + +#define PCI_RC_VNET_DMA_DESC_CTRL_PKT_USED BIT(0) + +#define PCI_RC_VNET_RXBUFFER_1024 1024 +#define PCI_RC_VNET_RXBUFFER_2048 2048 +#define PCI_RC_VNET_RXBUFFER_4096 4096 +#define PCI_RC_VNET_RXBUFFER_8192 8192 +#define PCI_RC_VNET_RXBUFFER_16384 16384 + +#define PCI_RC_VNET_DMA_BUFF_NUM 4096 +#define PCI_RC_VNET_WAKE_NET_QUEUE_THRESHOLD 10 +#define PCI_RC_VNET_TX_QUEUE_STOP_THRESHOLD 64 +#define PCI_RC_VNET_TX_QUEUE_RESTART_THRESHOLD 64 + +#define PCI_RC_VNET_WATCHDOG_PERIOD (2 * HZ) +#define PCI_RC_VNET_LINK_DETECT_PERIOD_NS 500000000 +#define PCI_RC_VNET_RECEIVE_PERIOD_NS 5000 +#define PCI_RC_VNET_TX_POLL_PERIOD_NS 5000 +#define PCI_RC_VNET_NAPI_PULL_BUDGET 64 + +#define BAR_NUM 6 +#define TX_IRQ 2 +#define RX_IRQ 1 + +#define PCI_RC_VNET_FLAGS_EP_LINK BIT(0) +#define PCI_RC_VNET_FLAGS_RC_LINK BIT(1) + +#define MAX_JUMBO_FRAME_SIZE 4000 + +#define PCI_RC_VNET_DMA_DESC_CTRL_FIELD_MASK(name) \ + (((1UL << PCI_RC_VNET_DMA_DESC_CTRL_PKT_##name##_SIZE) - 1) \ + << PCI_RC_VNET_DMA_DESC_CTRL_PKT_##name##_OFFSET) + +#define PCI_RC_VNET_DMA_DESC_FIELD_GET(name, desc) \ + (((desc) >> PCI_RC_VNET_DMA_DESC_CTRL_PKT_##name##_OFFSET) \ + & ((1UL << PCI_RC_VNET_DMA_DESC_CTRL_PKT_##name##_SIZE) - 1)) + +#define PCI_RC_VNET_DMA_DESC_SET(name, value) \ + ((((u64)value) & ((1UL << PCI_RC_VNET_DMA_DESC_CTRL_PKT_##name##_SIZE) - 1)) \ + << PCI_RC_VNET_DMA_DESC_CTRL_PKT_##name##_OFFSET) + +#define SEND_MSI_IRQ 0xf +#define RC_SKB_PAD (0) + +#define RC_RX_DMA_ATTR \ + (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING) + +#define RC_GFP_FLAGS \ + (GFP_ATOMIC | __GFP_NOWARN) + +#define RC_RX_PAGE_ORDER 0 +#define RC_RX_PAGE_SIZE (PAGE_SIZE << RC_RX_PAGE_ORDER) + +#define RC_RXBUFFER_2048 2048 +#define RX_BUFFER_MULTIPLE 64 +#define RC_MAX_FRAME_BUILD_SKB \ + (SKB_WITH_OVERHEAD(RC_RXBUFFER_2048) - RC_SKB_PAD) +enum pci_barno { + BAR_0, + BAR_1, + BAR_2, + BAR_3, + BAR_4, + BAR_5, +}; + +struct param_range { + u32 min; + u32 max; + u32 count; +}; + +struct params { + struct param_range rx_ring; + struct param_range tx_ring; +}; + +struct skb_dma { + void *virt_addr; + dma_addr_t phys_addr; +}; + +struct pci_rc_vnet_dma_desc { + u64 addr; + u64 ctrl; +}; + +struct pci_rc_vnet_tx_queue { + u32 irq; + u32 tail; + u32 head; + u32 nb_desc; + u32 flags; + struct pci_rc_vnet_dma_desc desc_ring[PCI_RC_VNET_DMA_BUFF_NUM]; +}; + +struct pci_rc_vnet_rx_queue { + u32 irq; + u32 tail; + u32 head; + u32 nb_desc; + u32 flags; + struct pci_rc_vnet_dma_desc desc_ring[PCI_RC_VNET_DMA_BUFF_NUM]; +}; + +struct rc_rx_buffer { + dma_addr_t addr; + struct page *page; + __u16 page_offset; + __u16 pagecnt_bias; +}; + +/* bar space */ +struct pci_rc_vnet_queue { + struct pci_rc_vnet_tx_queue tx_queue; + struct pci_rc_vnet_rx_queue rx_queue; +} __packed; + +struct pci_rc_vnet_private { + u32 msg_enable ____cacheline_aligned; + int rx_buffer_len; + struct pci_dev *pci_dev; + void __iomem *bar[BAR_NUM]; + + dma_addr_t *tx_phys_addr_list; + struct sk_buff **tx_skbuff; + struct rc_rx_buffer *rx_buffer_info; + u32 rx_next_to_alloc; + + struct pci_rc_vnet_queue *queue; + + struct pci_rc_vnet_tx_queue *tx_queue; + struct pci_rc_vnet_rx_queue *rx_queue; + + struct net_device *netdev; + int irq_type; + int num_irqs; + + /* Lock to protect tx */ + spinlock_t tx_lock; + /* Lock to protect tx_reclaim_num */ + spinlock_t tx_reclaim_lock; + /* Lock to protect rx */ + spinlock_t rx_lock; + + struct params params; + + struct work_struct tx_timeout_task; + + int (*clean_ep2rc)(struct pci_rc_vnet_private *tp, int budget); + + struct napi_struct tx_napi; + struct napi_struct rx_napi; + + int tx_reclaim_start; + int tx_reclaim_num; + + int rx_refill_start; + struct hrtimer link_detect_timer; + struct hrtimer receive_timer; + struct hrtimer tx_poll_timer; + void __iomem *msi_irq_addr; +}; + +#endif //end __PCI_RC_VNET_H diff --git a/drivers/net/ethernet/phytium/phytmac.h b/drivers/net/ethernet/phytium/phytmac.h index c3feca3d9611a..894e4053cd107 100644 --- a/drivers/net/ethernet/phytium/phytmac.h +++ b/drivers/net/ethernet/phytium/phytmac.h @@ -17,7 +17,7 @@ #define PHYTMAC_PCI_DRV_NAME "phytmac_pci" #define PHYTMAC_PLAT_DRV_NAME "phytmac_platform" #define PHYTMAC_DRV_DESC "PHYTIUM Ethernet Driver" -#define PHYTMAC_DRIVER_VERSION "1.0.50" +#define PHYTMAC_DRIVER_VERSION "1.0.56" #define PHYTMAC_DEFAULT_MSG_ENABLE \ (NETIF_MSG_DRV | \ NETIF_MSG_PROBE | \ @@ -30,6 +30,14 @@ #define IRQ_TYPE_MSI 1 #define IRQ_TYPE_INTX 2 +#define PHY_INIT_SMC_ID 0xc2000015 +#define PHY_SLOT(i) ((i) * 4) +#define PHY_INIT(i) (0x1U << (i)) +#define PHY_MULTIPLEX_GMAC(i) (0x2U << PHY_SLOT(i)) +#define PHY_MODE_SGMII(i) (0x3U << PHY_SLOT(i)) +#define PHY_SPEED_1000(i) (0x3U << PHY_SLOT(i)) +#define PHY_SPEED_2500(i) (0x4U << PHY_SLOT(i)) + #define PHYTMAC_MAX_QUEUES 8 #define DEFAULT_DMA_BURST_LENGTH 16 #define DEFAULT_JUMBO_MAX_LENGTH 10240 @@ -182,6 +190,13 @@ struct packet_info { u32 seq; }; +struct phyinit_cfg { + u64 phy_domain; + u64 phy_multiplex; + u64 phy_mode; + u64 phy_speed; +}; + #define DEV_TYPE_PLATFORM 0 #define DEV_TYPE_PCI 1 @@ -494,6 +509,7 @@ struct phytmac { int pause; phy_interface_t phy_interface; enum phytmac_interface phytmac_v2_interface; + int ori_speed; int speed; int duplex; int autoneg; @@ -572,6 +588,7 @@ struct phytmac_hw_if { int (*del_fdir_entry)(struct phytmac *pdata, struct ethtool_rx_flow_spec *rx_flow); /* mido ops */ + void (*phy_speed_switch)(struct phytmac *pdata, int speed); int (*enable_mdio_control)(struct phytmac *pdata, int enable); int (*mdio_read)(struct phytmac *pdata, int mii_id, int regnum); int (*mdio_write)(struct phytmac *pdata, int mii_id, diff --git a/drivers/net/ethernet/phytium/phytmac_main.c b/drivers/net/ethernet/phytium/phytmac_main.c index ece14f0a7b47e..70611ef867f47 100644 --- a/drivers/net/ethernet/phytium/phytmac_main.c +++ b/drivers/net/ethernet/phytium/phytmac_main.c @@ -1131,7 +1131,7 @@ static struct sk_buff *phytmac_rx_mbuffer(struct phytmac_queue *queue) for (rx_tail = queue->rx_tail; ; rx_tail++) { desc = phytmac_get_rx_desc(queue, rx_tail); if (!hw_if->rx_complete(desc)) - return NULL; + return ERR_PTR(-EAGAIN); if (hw_if->rx_pkt_start(desc)) { if (first_frag != -1) @@ -1218,6 +1218,9 @@ static int phytmac_rx(struct phytmac_queue *queue, struct napi_struct *napi, if (pdata->xdp_prog) netdev_warn(pdata->ndev, "xdp does not support multiple buffers!!\n"); skb = phytmac_rx_mbuffer(queue); + if (PTR_ERR(skb) == -EAGAIN) { + break; + } } if (!skb) { @@ -1849,6 +1852,36 @@ static int phytmac_phylink_connect(struct phytmac *pdata) return 0; } +static void phytmac_sgmii_speed_switch(struct phytmac *pdata, int speed) +{ + struct phytmac_hw_if *hw_if = pdata->hw_if; + int ori_speed = pdata->ori_speed; + + pdata->ori_speed = speed; + + if (pdata->phy_interface != PHY_INTERFACE_MODE_SGMII) + goto out; + + if (pdata->version != VERSION_V0) + goto out; + + if (ori_speed == speed) + goto out; + + /* Reject switching between 1G/100M/10M, since these share the same + * internal PHY speed mode. + */ + if ((ori_speed == SPEED_1000 || ori_speed == SPEED_100 || ori_speed == SPEED_10) && + (speed == SPEED_1000 || speed == SPEED_100 || speed == SPEED_10)) + goto out; + + if (hw_if->phy_speed_switch) + hw_if->phy_speed_switch(pdata, speed); + +out: + return; +} + int phytmac_pcs_config(struct phylink_pcs *pcs, unsigned int mode, phy_interface_t interface, const unsigned long *advertising, @@ -1976,7 +2009,10 @@ static void phytmac_mac_link_up(struct phylink_config *config, spin_lock_irqsave(&pdata->lock, flags); - hw_if->mac_linkup(pdata, interface, speed, duplex); + if (phy) + phytmac_sgmii_speed_switch(pdata, speed); + + hw_if->mac_linkup(pdata, pdata->phy_interface, speed, duplex); if (rx_pause != pdata->pause) { hw_if->enable_pause(pdata, rx_pause); @@ -2365,7 +2401,7 @@ static netdev_features_t phytmac_features_check(struct sk_buff *skb, hdrlen = skb_transport_offset(skb); if (!IS_ALIGNED(skb_headlen(skb) - hdrlen, PHYTMAC_TX_LEN_ALIGN)) - return features & ~NETIF_F_TSO; + return features & ~(NETIF_F_TSO | NETIF_F_TSO6); nr_frags = skb_shinfo(skb)->nr_frags; /* No need to check last fragment */ @@ -2374,7 +2410,7 @@ static netdev_features_t phytmac_features_check(struct sk_buff *skb, const skb_frag_t *frag = &skb_shinfo(skb)->frags[f]; if (!IS_ALIGNED(skb_frag_size(frag), PHYTMAC_TX_LEN_ALIGN)) - return features & ~NETIF_F_TSO; + return features & ~(NETIF_F_TSO | NETIF_F_TSO6); } return features; } @@ -2664,7 +2700,7 @@ void phytmac_default_config(struct phytmac *pdata) ndev->hw_features = NETIF_F_SG; if (pdata->capacities & PHYTMAC_CAPS_LSO) - ndev->hw_features |= NETIF_F_TSO; + ndev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6; if (pdata->use_ncsi) { ndev->hw_features &= ~(NETIF_F_HW_CSUM | NETIF_F_RXCSUM); diff --git a/drivers/net/ethernet/phytium/phytmac_v1.c b/drivers/net/ethernet/phytium/phytmac_v1.c index 870f07edc58fc..d467644c4a293 100644 --- a/drivers/net/ethernet/phytium/phytmac_v1.c +++ b/drivers/net/ethernet/phytium/phytmac_v1.c @@ -16,6 +16,7 @@ #include #include #include +#include #include "phytmac.h" #include "phytmac_v1.h" @@ -177,6 +178,52 @@ static int phytmac_pcs_software_reset(struct phytmac *pdata, int reset) return 0; } +static void phytmac_phy_speed_switch(struct phytmac *pdata, int speed) +{ + struct arm_smccc_res res; + struct phyinit_cfg cfg; + int id; + + switch (pdata->ndev->base_addr) { + case MAC0_ADDR_BASE: + id = 0; + break; + case MAC1_ADDR_BASE: + id = 1; + break; + default: + return; + } + + switch (speed) { + case SPEED_1000: + case SPEED_100: + case SPEED_10: + cfg.phy_speed = PHY_SPEED_1000(id); + break; + case SPEED_2500: + cfg.phy_speed = PHY_SPEED_2500(id); + break; + default: + return; + } + + cfg.phy_domain = PHY_INIT(id); + cfg.phy_multiplex = PHY_MULTIPLEX_GMAC(id); + cfg.phy_mode = PHY_MODE_SGMII(id); + + arm_smccc_smc(PHY_INIT_SMC_ID, cfg.phy_domain, cfg.phy_multiplex, + cfg.phy_mode, cfg.phy_speed, 0, 0, 0, &res); + if (res.a0 == SMCCC_RET_NOT_SUPPORTED) { + netdev_warn(pdata->ndev, "PHY_INIT_SMC_ID not implemented, skipping ....\n"); + goto out; + } + netdev_info(pdata->ndev, "switch speed to %dMb/s successfully\n", speed); + +out: + return; +} + static int phytmac_mac_linkup(struct phytmac *pdata, phy_interface_t interface, int speed, int duplex) { @@ -216,6 +263,8 @@ static int phytmac_mac_linkup(struct phytmac *pdata, phy_interface_t interface, if (interface == PHY_INTERFACE_MODE_SGMII) { if (speed == SPEED_2500) phytmac_enable_autoneg(pdata, 0); + else + phytmac_enable_autoneg(pdata, 1); } return 0; @@ -443,14 +492,6 @@ static int phytmac_powerup_hw(struct phytmac *pdata, int on) { u32 status, data0, data1, rdata1; int ret; - acpi_handle handle; - union acpi_object args[3]; - struct acpi_object_list arg_list = { - .pointer = args, - .count = ARRAY_SIZE(args), - }; - acpi_status acpi_sts; - unsigned long long rv; if (!(pdata->capacities & PHYTMAC_CAPS_PWCTRL)) { pdata->power_state = on; @@ -458,6 +499,16 @@ static int phytmac_powerup_hw(struct phytmac *pdata, int on) } if (has_acpi_companion(pdata->dev)) { +#ifdef CONFIG_ACPI + acpi_handle handle; + union acpi_object args[3]; + struct acpi_object_list arg_list = { + .pointer = args, + .count = ARRAY_SIZE(args), + }; + acpi_status acpi_sts; + unsigned long long rv; + handle = ACPI_HANDLE(pdata->dev); netdev_info(pdata->ndev, "set gmac power %s\n", @@ -482,6 +533,7 @@ static int phytmac_powerup_hw(struct phytmac *pdata, int on) if (rv) netdev_err(pdata->ndev, "Failed to power off\n"); } +#endif } else { ret = readx_poll_timeout(PHYTMAC_READ_STAT, pdata, status, !status, 1, PHYTMAC_TIMEOUT); @@ -1418,6 +1470,7 @@ struct phytmac_hw_if phytmac_1p0_hw = { .get_stats = phytmac_get_hw_stats, .set_mac_address = phytmac_set_mac_addr, .get_mac_address = phytmac_get_mac_addr, + .phy_speed_switch = phytmac_phy_speed_switch, .mdio_idle = phytmac_mdio_idle, .mdio_read = phytmac_mdio_data_read_c22, .mdio_write = phytmac_mdio_data_write_c22, diff --git a/drivers/net/ethernet/phytium/phytmac_v1.h b/drivers/net/ethernet/phytium/phytmac_v1.h index e0ca0ab835474..603c96b55cc42 100644 --- a/drivers/net/ethernet/phytium/phytmac_v1.h +++ b/drivers/net/ethernet/phytium/phytmac_v1.h @@ -9,6 +9,9 @@ extern struct phytmac_hw_if phytmac_1p0_hw; #define PHYTMAC_FRAME_MASK 0x1fff #define PHYTMAC_JUMBO_FRAME_MASK 0x3fff +#define MAC0_ADDR_BASE 0x36ce0000 +#define MAC1_ADDR_BASE 0x36ce2000 + #define PHYTMAC_SPEED_100M 0 #define PHYTMAC_SPEED_1000M 1 #define PHYTMAC_SPEED_2500M 2 diff --git a/drivers/net/ethernet/phytium/phytmac_v2.c b/drivers/net/ethernet/phytium/phytmac_v2.c index 4dd3ddd18e714..46b1e5ce20a04 100644 --- a/drivers/net/ethernet/phytium/phytmac_v2.c +++ b/drivers/net/ethernet/phytium/phytmac_v2.c @@ -68,7 +68,7 @@ static int phytmac_v2_msg_send(struct phytmac *pdata, u16 cmd_id, msg.cmd_type, msg.cmd_subid, msg.status0, len, tx_tail); } - memcpy(pdata->msg_regs + PHYTMAC_MSG(tx_tail), &msg, sizeof(msg)); + memcpy_toio(pdata->msg_regs + PHYTMAC_MSG(tx_tail), &msg, sizeof(msg)); tx_tail = phytmac_v2_tx_ring_wrap(pdata, ++tx_tail); PHYTMAC_WRITE(pdata, PHYTMAC_TX_MSG_TAIL, tx_tail | PHYTMAC_BIT(TX_MSG_INT)); pdata->msg_ring.tx_msg_wr_tail = tx_tail; @@ -86,8 +86,9 @@ static int phytmac_v2_msg_send(struct phytmac *pdata, u16 cmd_id, } } - memcpy(&msg_rx, pdata->msg_regs + PHYTMAC_MSG(pdata->msg_ring.tx_msg_rd_tail), - PHYTMAC_MSG_HDR_LEN); + memcpy_fromio(&msg_rx, pdata->msg_regs + + PHYTMAC_MSG(pdata->msg_ring.tx_msg_rd_tail), + PHYTMAC_MSG_HDR_LEN); if (!(msg_rx.status0 & PHYTMAC_CMD_PRC_SUCCESS)) { netdev_err(pdata->ndev, "Msg process error, cmdid:%d, subid:%d, status0:%d, tail:%d", msg.cmd_type, msg.cmd_subid, msg.status0, tx_tail); @@ -142,8 +143,8 @@ static int phytmac_v2_get_mac_addr(struct phytmac *pdata, u8 *addr) phytmac_v2_msg_send(pdata, cmd_id, cmd_subid, NULL, 0, 1); index = phytmac_v2_tx_ring_wrap(pdata, pdata->msg_ring.tx_msg_rd_tail); - memcpy(¶, pdata->msg_regs + PHYTMAC_MSG(index) + PHYTMAC_MSG_HDR_LEN, - sizeof(struct phytmac_mac)); + memcpy_fromio(¶, pdata->msg_regs + PHYTMAC_MSG(index) + PHYTMAC_MSG_HDR_LEN, + sizeof(struct phytmac_mac)); addr[0] = para.addrl & 0xff; addr[1] = (para.addrl >> 8) & 0xff; @@ -194,7 +195,6 @@ static int phytmac_v2_init_hw(struct phytmac *pdata) struct phytmac_dma_info dma; struct phytmac_eth_info eth; u32 ptrconfig = 0; - u8 mdc; if (pdata->capacities & PHYTMAC_CAPS_TAILPTR) ptrconfig |= PHYTMAC_BIT(TXTAIL_EN); @@ -257,10 +257,6 @@ static int phytmac_v2_init_hw(struct phytmac *pdata) dma.hw_dma_cap |= HW_DMA_CAP_DDW128; phytmac_v2_msg_send(pdata, cmd_id, cmd_subid, (void *)&dma, sizeof(dma), 0); - cmd_subid = PHYTMAC_MSG_CMD_SET_MDC; - mdc = PHYTMAC_CLK_DIV96; - phytmac_v2_msg_send(pdata, cmd_id, cmd_subid, (void *)(&mdc), sizeof(mdc), 0); - memset(ð, 0, sizeof(eth)); cmd_subid = PHYTMAC_MSG_CMD_SET_ETH_MATCH; eth.index = 0; @@ -368,8 +364,8 @@ static int phytmac_v2_get_feature_all(struct phytmac *pdata) cmd_subid = PHYTMAC_MSG_CMD_GET_CAPS; phytmac_v2_msg_send(pdata, cmd_id, cmd_subid, NULL, 0, 1); index = phytmac_v2_tx_ring_wrap(pdata, pdata->msg_ring.tx_msg_rd_tail); - memcpy(¶, pdata->msg_regs + PHYTMAC_MSG(index) + PHYTMAC_MSG_HDR_LEN, - sizeof(struct phytmac_feature)); + memcpy_fromio(¶, pdata->msg_regs + PHYTMAC_MSG(index) + PHYTMAC_MSG_HDR_LEN, + sizeof(struct phytmac_feature)); pdata->queues_max_num = para.queue_num; if (para.dma_addr_width) @@ -409,15 +405,15 @@ static void phytmac_v2_get_regs(struct phytmac *pdata, u32 *reg_buff) msg.cnt = 0; phytmac_v2_msg_send(pdata, cmd_id, cmd_subid, (void *)(&msg), sizeof(msg), 1); index = phytmac_v2_tx_ring_wrap(pdata, pdata->msg_ring.tx_msg_rd_tail); - memcpy(reg_buff, pdata->msg_regs + PHYTMAC_MSG(index) + PHYTMAC_MSG_HDR_LEN, - PHYTMAC_MSG_PARA_LEN); + memcpy_fromio(reg_buff, pdata->msg_regs + PHYTMAC_MSG(index) + PHYTMAC_MSG_HDR_LEN, + PHYTMAC_MSG_PARA_LEN); msg.cnt = 1; phytmac_v2_msg_send(pdata, cmd_id, cmd_subid, (void *)(&msg), sizeof(msg), 1); index = phytmac_v2_tx_ring_wrap(pdata, pdata->msg_ring.tx_msg_rd_tail); - memcpy(reg_buff + PHYTMAC_MSG_PARA_LEN / sizeof(u32), pdata->msg_regs + - PHYTMAC_MSG(index) + PHYTMAC_MSG_HDR_LEN, - PHYTMAC_ETHTOOLD_REGS_LEN - PHYTMAC_MSG_PARA_LEN); + memcpy_fromio(reg_buff + PHYTMAC_MSG_PARA_LEN / sizeof(u32), pdata->msg_regs + + PHYTMAC_MSG(index) + PHYTMAC_MSG_HDR_LEN, + PHYTMAC_ETHTOOLD_REGS_LEN - PHYTMAC_MSG_PARA_LEN); } static void phytmac_v2_get_hw_stats(struct phytmac *pdata) @@ -540,14 +536,6 @@ static int phytmac_v2_powerup_hw(struct phytmac *pdata, int on) { u32 status, data0, data1, rdata1; int ret; - acpi_handle handle; - union acpi_object args[3]; - struct acpi_object_list arg_list = { - .pointer = args, - .count = ARRAY_SIZE(args), - }; - acpi_status acpi_sts; - unsigned long long rv; if (!(pdata->capacities & PHYTMAC_CAPS_PWCTRL)) { pdata->power_state = on; @@ -555,6 +543,16 @@ static int phytmac_v2_powerup_hw(struct phytmac *pdata, int on) } if (has_acpi_companion(pdata->dev)) { +#ifdef CONFIG_ACPI + acpi_handle handle; + union acpi_object args[3]; + struct acpi_object_list arg_list = { + .pointer = args, + .count = ARRAY_SIZE(args), + }; + acpi_status acpi_sts; + unsigned long long rv; + handle = ACPI_HANDLE(pdata->dev); netdev_info(pdata->ndev, "set gmac power %s\n", @@ -579,6 +577,7 @@ static int phytmac_v2_powerup_hw(struct phytmac *pdata, int on) if (rv) netdev_err(pdata->ndev, "Failed to power off\n"); } +#endif } else { ret = readx_poll_timeout(PHYTMAC_READ_STAT, pdata, status, !status, 1, PHYTMAC_TIMEOUT); @@ -705,8 +704,13 @@ static int phytmac_v2_enable_txcsum(struct phytmac *pdata, int enable) static int phytmac_v2_enable_mdio(struct phytmac *pdata, int enable) { u16 cmd_id, cmd_subid; + u8 mdc; cmd_id = PHYTMAC_MSG_CMD_SET; + cmd_subid = PHYTMAC_MSG_CMD_SET_MDC; + mdc = PHYTMAC_CLK_DIV96; + phytmac_v2_msg_send(pdata, cmd_id, cmd_subid, (void *)(&mdc), sizeof(mdc), 0); + if (enable) cmd_subid = PHYTMAC_MSG_CMD_SET_ENABLE_MDIO; else @@ -928,6 +932,8 @@ static int phytmac_v2_interface_linkup(struct phytmac *pdata, phy_interface_t in if (interface == PHY_INTERFACE_MODE_SGMII) { if (speed == SPEED_2500) pdata->autoneg = 0; + else + pdata->autoneg = 1; } memset(¶, 0, sizeof(para));