From cb81c271e9e609d9dbd19d5d26a298c723c30225 Mon Sep 17 00:00:00 2001 From: Mike Engel Date: Sat, 24 Feb 2024 18:16:35 +0100 Subject: [PATCH 07/10] drivers: add NXP RT support Upstream-Status: Inappropriate [DEY specific] Signed-off-by: Mike Engel --- drivers/base/Kconfig | 2 +- drivers/block/zram/zram_drv.c | 36 + drivers/block/zram/zram_drv.h | 3 + drivers/char/tpm/tpm_tis.c | 29 +- drivers/clk/imx/Makefile | 1 + drivers/clk/imx/clk-frac-pll.c | 130 + drivers/clk/imx/clk-fracn-gppll.c | 208 +- drivers/clk/imx/clk-imx8dxl-acm.c | 49 + drivers/clk/imx/clk-imx8mq.c | 2 +- drivers/clk/imx/clk-pll.c | 83 + drivers/clk/imx/clk-pll.h | 33 + drivers/clk/imx/clk-pll14xx.c | 120 + drivers/clk/imx/clk-pllv3.c | 109 + drivers/clk/imx/clk-scu.c | 86 + drivers/firmware/imx/scu-pd.c | 1 + drivers/gpu/drm/i915/Kconfig | 1 - drivers/gpu/drm/i915/display/intel_crtc.c | 15 +- drivers/gpu/drm/i915/gt/intel_breadcrumbs.c | 5 +- .../drm/i915/gt/intel_execlists_submission.c | 17 +- drivers/gpu/drm/i915/i915_irq.c | 6 +- drivers/gpu/drm/i915/i915_request.c | 2 - drivers/gpu/drm/i915/i915_trace.h | 6 +- drivers/gpu/drm/i915/i915_utils.h | 2 +- drivers/irqchip/Kconfig | 4 + drivers/irqchip/Makefile | 1 + drivers/irqchip/ipi-baremetal.c | 454 +++ drivers/irqchip/irq-gic-v3.c | 10 +- drivers/irqchip/irq-gic.c | 4 +- drivers/mailbox/Kconfig | 9 + drivers/mailbox/Makefile | 2 + drivers/mailbox/generic-software-mailbox.c | 322 ++ drivers/mailbox/mailbox.c | 2 +- drivers/mxc/ipu3/ipu_common.c | 10 +- drivers/mxc/ipu3/ipu_device.c | 2 +- drivers/net/dsa/Kconfig | 2 + drivers/net/dsa/Makefile | 1 + drivers/net/dsa/netc/Kconfig | 10 + drivers/net/dsa/netc/Makefile | 12 + drivers/net/dsa/netc/netc.h | 102 + drivers/net/dsa/netc/netc_config.c | 322 ++ drivers/net/dsa/netc/netc_config.h | 288 ++ drivers/net/dsa/netc/netc_devlink.c | 111 + drivers/net/dsa/netc/netc_ethtool.c | 344 ++ drivers/net/dsa/netc/netc_main.c | 928 +++++ drivers/net/dsa/netc/netc_ptp.c | 12 + drivers/net/dsa/netc/netc_spi.c | 119 + drivers/net/dsa/ocelot/felix.c | 184 +- drivers/net/dsa/ocelot/felix.h | 5 + drivers/net/dsa/ocelot/felix_tsn.c | 6 +- drivers/net/dsa/ocelot/felix_vsc9959.c | 101 + drivers/net/dsa/sja1105/sja1105.h | 5 + drivers/net/dsa/sja1105/sja1105_clocking.c | 21 +- drivers/net/dsa/sja1105/sja1105_main.c | 64 +- drivers/net/dsa/sja1105/sja1105_tas.c | 4 + drivers/net/ethernet/alacritech/slic.h | 12 +- drivers/net/ethernet/amazon/ena/ena_ethtool.c | 4 +- drivers/net/ethernet/amazon/ena/ena_netdev.c | 12 +- .../net/ethernet/aquantia/atlantic/aq_main.c | 1 + .../net/ethernet/aquantia/atlantic/aq_ring.c | 8 +- drivers/net/ethernet/asix/ax88796c_main.c | 4 +- drivers/net/ethernet/broadcom/b44.c | 8 +- drivers/net/ethernet/broadcom/bcmsysport.c | 12 +- .../ethernet/chelsio/cxgb4/cxgb4_tc_mqprio.h | 2 +- drivers/net/ethernet/cortina/gemini.c | 24 +- .../net/ethernet/emulex/benet/be_ethtool.c | 12 +- drivers/net/ethernet/emulex/benet/be_main.c | 16 +- drivers/net/ethernet/freescale/Kconfig | 20 + drivers/net/ethernet/freescale/Makefile | 2 + drivers/net/ethernet/freescale/enetc/enetc.h | 5 + .../ethernet/freescale/enetc/enetc_ethtool.c | 152 + .../net/ethernet/freescale/enetc/enetc_hw.h | 6 + .../net/ethernet/freescale/enetc/enetc_pf.c | 16 +- .../net/ethernet/freescale/enetc/enetc_ptp.c | 15 +- .../net/ethernet/freescale/enetc/enetc_tsn.c | 194 +- drivers/net/ethernet/freescale/fec.h | 100 +- drivers/net/ethernet/freescale/fec_ecat.c | 3008 +++++++++++++++++ drivers/net/ethernet/freescale/fec_ecat.h | 713 ++++ drivers/net/ethernet/freescale/fec_main.c | 1460 +++++++- drivers/net/ethernet/freescale/fec_ptp.c | 451 ++- drivers/net/ethernet/freescale/fec_uio.c | 4 +- .../net/ethernet/freescale/gianfar_ethtool.c | 29 + .../net/ethernet/freescale/sdk_dpaa/Kconfig | 11 +- .../net/ethernet/freescale/sdk_dpaa/Makefile | 3 + .../ethernet/freescale/sdk_dpaa/dpaa_eth.h | 13 +- .../freescale/sdk_dpaa/dpaa_eth_common.c | 56 + .../ethernet/freescale/sdk_dpaa/dpaa_eth_sg.c | 28 +- .../freescale/sdk_dpaa/dpaa_ethercat.c | 1217 +++++++ .../ethernet/fungible/funeth/funeth_txrx.h | 4 +- drivers/net/ethernet/google/gve/gve_ethtool.c | 16 +- drivers/net/ethernet/google/gve/gve_main.c | 12 +- drivers/net/ethernet/hisilicon/hns3/hnae3.h | 1 + .../net/ethernet/hisilicon/hns3/hns3_enet.c | 5 +- drivers/net/ethernet/huawei/hinic/hinic_rx.c | 4 +- drivers/net/ethernet/huawei/hinic/hinic_tx.c | 4 +- .../net/ethernet/intel/fm10k/fm10k_netdev.c | 8 +- drivers/net/ethernet/intel/i40e/i40e.h | 1 + .../net/ethernet/intel/i40e/i40e_ethtool.c | 8 +- drivers/net/ethernet/intel/i40e/i40e_main.c | 20 +- drivers/net/ethernet/intel/iavf/iavf.h | 1 + .../net/ethernet/intel/iavf/iavf_ethtool.c | 8 +- drivers/net/ethernet/intel/ice/ice.h | 1 + drivers/net/ethernet/intel/ice/ice_main.c | 4 +- drivers/net/ethernet/intel/igb/igb_ethtool.c | 12 +- drivers/net/ethernet/intel/igb/igb_main.c | 8 +- drivers/net/ethernet/intel/igc/igc_ethtool.c | 12 +- drivers/net/ethernet/intel/igc/igc_main.c | 8 +- .../net/ethernet/intel/ixgbe/ixgbe_ethtool.c | 8 +- drivers/net/ethernet/intel/ixgbe/ixgbe_main.c | 8 +- drivers/net/ethernet/intel/ixgbevf/ethtool.c | 12 +- .../net/ethernet/intel/ixgbevf/ixgbevf_main.c | 8 +- drivers/net/ethernet/marvell/mvneta.c | 10 +- .../net/ethernet/marvell/mvpp2/mvpp2_main.c | 8 +- drivers/net/ethernet/marvell/sky2.c | 8 +- drivers/net/ethernet/mediatek/mtk_eth_soc.c | 8 +- .../net/ethernet/mellanox/mlx5/core/en_main.c | 1 + .../net/ethernet/mellanox/mlxsw/spectrum.c | 4 +- .../ethernet/microchip/lan966x/lan966x_tc.c | 1 + .../net/ethernet/microchip/sparx5/sparx5_tc.c | 1 + drivers/net/ethernet/microsoft/mana/mana_en.c | 8 +- .../ethernet/microsoft/mana/mana_ethtool.c | 8 +- drivers/net/ethernet/mscc/ocelot.c | 52 +- drivers/net/ethernet/mscc/ocelot_flower.c | 30 +- drivers/net/ethernet/mscc/ocelot_ptp.c | 1 + drivers/net/ethernet/mscc/ocelot_vcap.c | 173 +- .../ethernet/netronome/nfp/nfp_net_common.c | 8 +- .../ethernet/netronome/nfp/nfp_net_ethtool.c | 8 +- .../net/ethernet/netronome/nfp/nfp_net_repr.c | 4 +- drivers/net/ethernet/nvidia/forcedeth.c | 8 +- .../net/ethernet/qualcomm/rmnet/rmnet_vnd.c | 4 +- drivers/net/ethernet/realtek/8139too.c | 8 +- drivers/net/ethernet/socionext/sni_ave.c | 8 +- .../net/ethernet/stmicro/stmmac/dwmac4_core.c | 38 +- drivers/net/ethernet/stmicro/stmmac/dwmac5.c | 19 +- drivers/net/ethernet/stmicro/stmmac/dwmac5.h | 6 +- .../ethernet/stmicro/stmmac/dwxgmac2_core.c | 3 +- drivers/net/ethernet/stmicro/stmmac/hwif.h | 11 +- drivers/net/ethernet/stmicro/stmmac/stmmac.h | 1 + .../ethernet/stmicro/stmmac/stmmac_ethtool.c | 103 + .../net/ethernet/stmicro/stmmac/stmmac_main.c | 40 +- .../net/ethernet/stmicro/stmmac/stmmac_tc.c | 37 +- drivers/net/ethernet/ti/am65-cpsw-nuss.c | 4 +- drivers/net/ethernet/ti/cpsw_priv.c | 1 + drivers/net/ethernet/ti/netcp_core.c | 8 +- drivers/net/ethernet/via/via-rhine.c | 8 +- .../net/ethernet/xilinx/xilinx_axienet_main.c | 8 +- drivers/net/hyperv/netvsc_drv.c | 32 +- drivers/net/ifb.c | 12 +- drivers/net/ipvlan/ipvlan_main.c | 4 +- drivers/net/loopback.c | 4 +- drivers/net/macsec.c | 12 +- drivers/net/macvlan.c | 4 +- drivers/net/mhi_net.c | 8 +- drivers/net/netdevsim/netdev.c | 4 +- drivers/net/phy/phy.c | 5 +- drivers/net/team/team.c | 4 +- drivers/net/team/team_mode_loadbalance.c | 4 +- drivers/net/veth.c | 12 +- drivers/net/virtio_net.c | 16 +- drivers/net/vrf.c | 4 +- drivers/net/vxlan/vxlan_vnifilter.c | 4 +- drivers/net/wwan/mhi_wwan_mbim.c | 8 +- drivers/net/xen-netfront.c | 8 +- drivers/nfc/Kconfig | 1 + drivers/nfc/Makefile | 1 + drivers/nfc/pn5xx/Kconfig | 12 + drivers/nfc/pn5xx/Makefile | 6 + drivers/nfc/pn5xx/README.md | 2 + drivers/nfc/pn5xx/pn5xx_i2c.c | 672 ++++ drivers/nfc/pn5xx/pn5xx_i2c.h | 38 + drivers/nfc/pn5xx/sample_devicetree.txt | 17 + drivers/rpmsg/Kconfig | 16 +- drivers/rpmsg/Makefile | 1 + drivers/rpmsg/imx_rpmsg.c | 57 +- drivers/rpmsg/imx_rpmsg_tty.c | 18 +- drivers/rpmsg/rpmsg_perf.c | 545 +++ drivers/rpmsg/virtio_rpmsg_bus.c | 5 + drivers/rtc/Kconfig | 1 + drivers/spi/spi.c | 4 +- drivers/staging/fsl_qbman/qman_driver.c | 50 + drivers/staging/fsl_qbman/qman_high.c | 38 + drivers/staging/fsl_qbman/qman_private.h | 6 + drivers/tty/Kconfig | 2 + drivers/tty/rpmsg_tty.c | 192 +- drivers/tty/serial/8250/8250.h | 41 +- drivers/tty/serial/8250/8250_aspeed_vuart.c | 2 +- drivers/tty/serial/8250/8250_bcm7271.c | 21 +- drivers/tty/serial/8250/8250_core.c | 24 +- drivers/tty/serial/8250/8250_exar.c | 4 +- drivers/tty/serial/8250/8250_fsl.c | 3 +- drivers/tty/serial/8250/8250_ingenic.c | 3 +- drivers/tty/serial/8250/8250_mtk.c | 32 +- drivers/tty/serial/8250/8250_omap.c | 20 +- drivers/tty/serial/8250/8250_port.c | 158 +- drivers/tty/serial/8250/Kconfig | 1 + drivers/tty/serial/amba-pl011.c | 17 +- drivers/tty/serial/omap-serial.c | 12 +- drivers/tty/sysrq.c | 2 + drivers/vdpa/vdpa_user/iova_domain.h | 1 - drivers/virtio/Kconfig | 8 + drivers/virtio/Makefile | 1 + drivers/virtio/virtio_mmio.c | 318 +- drivers/virtio/virtio_ring.c | 2 - drivers/virtio/virtio_trans.c | 793 +++++ 203 files changed, 15306 insertions(+), 832 deletions(-) create mode 100644 drivers/clk/imx/clk-pll.c create mode 100644 drivers/clk/imx/clk-pll.h create mode 100644 drivers/irqchip/ipi-baremetal.c create mode 100644 drivers/mailbox/generic-software-mailbox.c create mode 100644 drivers/net/dsa/netc/Kconfig create mode 100644 drivers/net/dsa/netc/Makefile create mode 100644 drivers/net/dsa/netc/netc.h create mode 100644 drivers/net/dsa/netc/netc_config.c create mode 100644 drivers/net/dsa/netc/netc_config.h create mode 100644 drivers/net/dsa/netc/netc_devlink.c create mode 100644 drivers/net/dsa/netc/netc_ethtool.c create mode 100644 drivers/net/dsa/netc/netc_main.c create mode 100644 drivers/net/dsa/netc/netc_ptp.c create mode 100644 drivers/net/dsa/netc/netc_spi.c create mode 100644 drivers/net/ethernet/freescale/fec_ecat.c create mode 100644 drivers/net/ethernet/freescale/fec_ecat.h create mode 100644 drivers/net/ethernet/freescale/sdk_dpaa/dpaa_ethercat.c create mode 100644 drivers/nfc/pn5xx/Kconfig create mode 100644 drivers/nfc/pn5xx/Makefile create mode 100644 drivers/nfc/pn5xx/README.md create mode 100644 drivers/nfc/pn5xx/pn5xx_i2c.c create mode 100644 drivers/nfc/pn5xx/pn5xx_i2c.h create mode 100644 drivers/nfc/pn5xx/sample_devicetree.txt create mode 100644 drivers/rpmsg/rpmsg_perf.c create mode 100644 drivers/virtio/virtio_trans.c diff --git a/drivers/base/Kconfig b/drivers/base/Kconfig index 6f04b831a5c0..5cb0ba920813 100644 --- a/drivers/base/Kconfig +++ b/drivers/base/Kconfig @@ -199,7 +199,7 @@ source "drivers/base/regmap/Kconfig" config DMA_SHARED_BUFFER bool - default n + default y select IRQ_WORK help This option enables the framework for buffer-sharing between diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c index 966aab902d19..ee69e4443691 100644 --- a/drivers/block/zram/zram_drv.c +++ b/drivers/block/zram/zram_drv.c @@ -57,6 +57,40 @@ static void zram_free_page(struct zram *zram, size_t index); static int zram_bvec_read(struct zram *zram, struct bio_vec *bvec, u32 index, int offset, struct bio *bio); +#ifdef CONFIG_PREEMPT_RT +static void zram_meta_init_table_locks(struct zram *zram, size_t num_pages) +{ + size_t index; + + for (index = 0; index < num_pages; index++) + spin_lock_init(&zram->table[index].lock); +} + +static int zram_slot_trylock(struct zram *zram, u32 index) +{ + int ret; + + ret = spin_trylock(&zram->table[index].lock); + if (ret) + __set_bit(ZRAM_LOCK, &zram->table[index].flags); + return ret; +} + +static void zram_slot_lock(struct zram *zram, u32 index) +{ + spin_lock(&zram->table[index].lock); + __set_bit(ZRAM_LOCK, &zram->table[index].flags); +} + +static void zram_slot_unlock(struct zram *zram, u32 index) +{ + __clear_bit(ZRAM_LOCK, &zram->table[index].flags); + spin_unlock(&zram->table[index].lock); +} + +#else + +static void zram_meta_init_table_locks(struct zram *zram, size_t num_pages) { } static int zram_slot_trylock(struct zram *zram, u32 index) { @@ -72,6 +106,7 @@ static void zram_slot_unlock(struct zram *zram, u32 index) { bit_spin_unlock(ZRAM_LOCK, &zram->table[index].flags); } +#endif static inline bool init_done(struct zram *zram) { @@ -1187,6 +1222,7 @@ static bool zram_meta_alloc(struct zram *zram, u64 disksize) if (!huge_class_size) huge_class_size = zs_huge_class_size(zram->mem_pool); + zram_meta_init_table_locks(zram, num_pages); return true; } diff --git a/drivers/block/zram/zram_drv.h b/drivers/block/zram/zram_drv.h index a2bda53020fd..ae7950b26db5 100644 --- a/drivers/block/zram/zram_drv.h +++ b/drivers/block/zram/zram_drv.h @@ -62,6 +62,9 @@ struct zram_table_entry { unsigned long element; }; unsigned long flags; +#ifdef CONFIG_PREEMPT_RT + spinlock_t lock; +#endif #ifdef CONFIG_ZRAM_MEMORY_TRACKING ktime_t ac_time; #endif diff --git a/drivers/char/tpm/tpm_tis.c b/drivers/char/tpm/tpm_tis.c index 0d084d6652c4..5d620322bdc2 100644 --- a/drivers/char/tpm/tpm_tis.c +++ b/drivers/char/tpm/tpm_tis.c @@ -50,6 +50,31 @@ static inline struct tpm_tis_tcg_phy *to_tpm_tis_tcg_phy(struct tpm_tis_data *da return container_of(data, struct tpm_tis_tcg_phy, priv); } +#ifdef CONFIG_PREEMPT_RT +/* + * Flushes previous write operations to chip so that a subsequent + * ioread*()s won't stall a cpu. + */ +static inline void tpm_tis_flush(void __iomem *iobase) +{ + ioread8(iobase + TPM_ACCESS(0)); +} +#else +#define tpm_tis_flush(iobase) do { } while (0) +#endif + +static inline void tpm_tis_iowrite8(u8 b, void __iomem *iobase, u32 addr) +{ + iowrite8(b, iobase + addr); + tpm_tis_flush(iobase); +} + +static inline void tpm_tis_iowrite32(u32 b, void __iomem *iobase, u32 addr) +{ + iowrite32(b, iobase + addr); + tpm_tis_flush(iobase); +} + static int interrupts = -1; module_param(interrupts, int, 0444); MODULE_PARM_DESC(interrupts, "Enable interrupts"); @@ -202,12 +227,12 @@ static int tpm_tcg_write_bytes(struct tpm_tis_data *data, u32 addr, u16 len, switch (io_mode) { case TPM_TIS_PHYS_8: while (len--) - iowrite8(*value++, phy->iobase + addr); + tpm_tis_iowrite8(*value++, phy->iobase, addr); break; case TPM_TIS_PHYS_16: return -EINVAL; case TPM_TIS_PHYS_32: - iowrite32(le32_to_cpu(*((__le32 *)value)), phy->iobase + addr); + tpm_tis_iowrite32(le32_to_cpu(*((__le32 *)value)), phy->iobase, addr); break; } diff --git a/drivers/clk/imx/Makefile b/drivers/clk/imx/Makefile index bc6579e0cb0b..53e75b26648f 100644 --- a/drivers/clk/imx/Makefile +++ b/drivers/clk/imx/Makefile @@ -16,6 +16,7 @@ mxc-clk-objs += clk-gate-93.o mxc-clk-objs += clk-gate-exclusive.o mxc-clk-objs += clk-pfd.o mxc-clk-objs += clk-pfdv2.o +mxc-clk-objs += clk-pll.o mxc-clk-objs += clk-pllv1.o mxc-clk-objs += clk-pllv2.o mxc-clk-objs += clk-pllv3.o diff --git a/drivers/clk/imx/clk-frac-pll.c b/drivers/clk/imx/clk-frac-pll.c index c703056fae85..2085d8e91ca8 100644 --- a/drivers/clk/imx/clk-frac-pll.c +++ b/drivers/clk/imx/clk-frac-pll.c @@ -17,6 +17,7 @@ #include #include "clk.h" +#include "clk-pll.h" #define PLL_CFG0 0x0 #define PLL_CFG1 0x4 @@ -35,7 +36,11 @@ #define PLL_FRAC_ACK_TIMEOUT 500000 struct clk_frac_pll { + u32 orig_divff; + u32 orig_divfi; + u32 orig_pllcfg0; struct clk_hw hw; + struct clk_imx_pll imx_pll; void __iomem *base; }; @@ -202,6 +207,126 @@ static const struct clk_ops clk_frac_pll_ops = { .set_rate = clk_pll_set_rate, }; +/* This function fetches the original PLL parameters to use + * them later for ppb adjustment + */ +static void imx_frac_pll_init(struct clk_imx_pll *pll) +{ + struct clk_frac_pll *frac_pll; + u32 val; + + frac_pll = (struct clk_frac_pll *) container_of(pll, + struct clk_frac_pll, imx_pll); + + val = readl_relaxed(frac_pll->base + PLL_CFG1); + + frac_pll->orig_divff = (val >> 7) & PLL_FRAC_DIV_MASK; + frac_pll->orig_divfi = val & PLL_INT_DIV_MASK; + + frac_pll->orig_pllcfg0 = readl_relaxed(frac_pll->base + PLL_CFG0); +} + +/** + * imx_frac_pll_adjust - Adjust the Audio pll by ppb. + * + * This function adjust the audio pll by ppb (part per billion) and returns + * the exact number of ppb adjusted. + * The adjustment is done by only modifying the Fractional Divide part + * of the audio PLL. + * Since the pllout = parent_rate * 8 / 2 * (1 + DIVFI + DIVFF / 2^24) + * and the adjusted value is + * pllout_new = pllout * (1 + ppb/1e9) which equals: + * parent_rate * 8 / 2 * (1 + DIVFI + DIVFF_new / 2^24) + * The new divff is calculated as the following: + * DIVFF_new = ((1 + DIVFI) * ppb * 2^24 + DIVFF * 1e9 + DIVFF * ppb) / (1e9) + */ + +static int imx_frac_pll_adjust(struct clk_imx_pll *pll, int *ppb) +{ + u64 temp64; + u32 val; + s64 applied_ppb; + struct clk_frac_pll *frac_pll; + int rc = IMX_CLK_PLL_SUCCESS; + + int req_ppb = *ppb; + + frac_pll = (struct clk_frac_pll *) container_of(pll, + struct clk_frac_pll, imx_pll); + + /*Calcultate the new PLL Numerator*/ + temp64 = ((u64) frac_pll->orig_divfi + 1) * PLL_FRAC_DENOM * req_ppb + + (u64) frac_pll->orig_divff * 1000000000 + + (u64) frac_pll->orig_divff * req_ppb; + + do_div(temp64, 1000000000); + + if (temp64 >= PLL_FRAC_DENOM) { + rc = -IMX_CLK_PLL_PREC_ERR; + goto exit; + } + + /* clear the NEW_DIV_VAL */ + val = frac_pll->orig_pllcfg0; + val &= ~PLL_NEWDIV_VAL; + writel_relaxed(val, frac_pll->base + PLL_CFG0); + + /* Write the PLL control settings with the new DIVFF + * NOTE: This sets the reserved bit (bit 31) to zero + */ + + val = 0; + val |= (((u32)temp64 << 7) | frac_pll->orig_divfi); + writel_relaxed(val, frac_pll->base + PLL_CFG1); + + /* Set the NEW_DIV_VAL to reload the DIVFI and DIVFF */ + val = frac_pll->orig_pllcfg0; + val |= PLL_NEWDIV_VAL; + writel_relaxed(val, frac_pll->base + PLL_CFG0); + + /*Calculate and return the actual applied ppb*/ + applied_ppb = div64_s64((s64) (temp64 - frac_pll->orig_divff) * 1000000000, + frac_pll->orig_divff + ((s64) frac_pll->orig_divfi + 1) * PLL_FRAC_DENOM); + + *ppb = (int) applied_ppb; + + exit: + return rc; +} + +static unsigned long imx_frac_pll_get_rate(struct clk_imx_pll *pll, + unsigned long parent_rate) +{ + struct clk_frac_pll *frac_pll; + + frac_pll = (struct clk_frac_pll *) container_of(pll, + struct clk_frac_pll, imx_pll); + + return clk_pll_recalc_rate(&frac_pll->hw, parent_rate); +} + +static int imx_frac_pll_set_rate(struct clk_imx_pll *pll, unsigned long rate, + unsigned long parent_rate) +{ + struct clk_frac_pll *frac_pll; + int rc = IMX_CLK_PLL_SUCCESS; + + frac_pll = (struct clk_frac_pll *) container_of(pll, + struct clk_frac_pll, imx_pll); + + if (clk_pll_set_rate(&frac_pll->hw, rate, parent_rate) < 0) + rc = -IMX_CLK_PLL_INVALID_PARAM; + + return rc; +} + +static const struct clk_imx_pll_ops imx_clk_frac_pll_ops = { + .set_rate = imx_frac_pll_set_rate, + .get_rate = imx_frac_pll_get_rate, + .adjust = imx_frac_pll_adjust, + .init = imx_frac_pll_init, +}; + struct clk_hw *imx_clk_hw_frac_pll(const char *name, const char *parent_name, void __iomem *base) @@ -232,6 +357,11 @@ struct clk_hw *imx_clk_hw_frac_pll(const char *name, return ERR_PTR(ret); } + pll->imx_pll.ops = &imx_clk_frac_pll_ops; + + if (imx_pll_register(&pll->imx_pll, name) < 0) + pr_warn("Failed to register %s into imx pll\n", name); + return hw; } EXPORT_SYMBOL_GPL(imx_clk_hw_frac_pll); diff --git a/drivers/clk/imx/clk-fracn-gppll.c b/drivers/clk/imx/clk-fracn-gppll.c index d611e81cf15b..c1bd42a4e479 100644 --- a/drivers/clk/imx/clk-fracn-gppll.c +++ b/drivers/clk/imx/clk-fracn-gppll.c @@ -13,6 +13,7 @@ #include #include "clk.h" +#include "clk-pll.h" #define PLL_CTRL 0x0 #define HW_CTRL_SEL BIT(16) @@ -53,10 +54,21 @@ .odiv = (_odiv), \ } + +/* MFN : Numerator of the fractional part of divider (30-bits, signed) */ +#define MFN_MAX_VALUE ((s32)(GENMASK(29, 0) >> 1)) +#define MFN_MIN_VALUE ((s32)(-MFN_MAX_VALUE - 1)) + struct clk_fracn_gppll { struct clk_hw hw; void __iomem *base; const struct imx_fracn_gppll_rate_table *rate_table; + u32 orig_mfd; + u32 orig_mfi; + u32 orig_odiv; + u32 orig_rdiv; + s32 orig_mfn; + struct clk_imx_pll imx_pll; int rate_count; }; @@ -79,7 +91,7 @@ static const struct imx_fracn_gppll_rate_table fracn_tbl[] = { PLL_FRACN_GP(484000000U, 121, 0, 1, 0, 6), PLL_FRACN_GP(445333333U, 167, 0, 1, 0, 9), PLL_FRACN_GP(400000000U, 200, 0, 1, 0, 12), - PLL_FRACN_GP(393216000U, 163, 84, 100, 0, 10), + PLL_FRACN_GP(393216000U, 163, 1680000, 2000000, 0, 10), PLL_FRACN_GP(300000000U, 150, 0, 1, 0, 12) }; @@ -123,18 +135,19 @@ static long clk_fracn_gppll_round_rate(struct clk_hw *hw, unsigned long rate, return rate_table[pll->rate_count - 1].rate; } -static unsigned long clk_fracn_gppll_recalc_rate(struct clk_hw *hw, unsigned long parent_rate) +/* This function calculates the actual rate based on the configured PLL registers */ +static unsigned long __clk_fracn_gppll_recalc_rate(struct clk_hw *hw, unsigned long parent_rate) { struct clk_fracn_gppll *pll = to_clk_fracn_gppll(hw); - const struct imx_fracn_gppll_rate_table *rate_table = pll->rate_table; u32 pll_numerator, pll_denominator, pll_div; - u32 mfi, mfn, mfd, rdiv, odiv; + u32 mfi, mfd, rdiv, odiv; + s32 mfn; u64 fvco = parent_rate; - long rate = 0; - int i; pll_numerator = readl_relaxed(pll->base + PLL_NUMERATOR); - mfn = FIELD_GET(PLL_MFN_MASK, pll_numerator); + + /* Numerator is 30-bits signed value */ + mfn = sign_extend32(FIELD_GET(PLL_MFN_MASK, pll_numerator), 29); pll_denominator = readl_relaxed(pll->base + PLL_DENOMINATOR); mfd = FIELD_GET(PLL_MFD_MASK, pll_denominator); @@ -145,22 +158,6 @@ static unsigned long clk_fracn_gppll_recalc_rate(struct clk_hw *hw, unsigned lon rdiv = FIELD_GET(PLL_RDIV_MASK, pll_div); odiv = FIELD_GET(PLL_ODIV_MASK, pll_div); - /* - * Sometimes, the recalculated rate has deviation due to - * the frac part. So find the accurate pll rate from the table - * first, if no match rate in the table, use the rate calculated - * from the equation below. - */ - for (i = 0; i < pll->rate_count; i++) { - if (rate_table[i].mfn == mfn && rate_table[i].mfi == mfi && - rate_table[i].mfd == mfd && rate_table[i].rdiv == rdiv && - rate_table[i].odiv == odiv) - rate = rate_table[i].rate; - } - - if (rate) - return (unsigned long)rate; - if (!rdiv) rdiv = rdiv + 1; @@ -176,12 +173,49 @@ static unsigned long clk_fracn_gppll_recalc_rate(struct clk_hw *hw, unsigned lon } /* Fvco = Fref * (MFI + MFN / MFD) */ - fvco = fvco * mfi * mfd + fvco * mfn; + fvco *= (mfi * mfd + mfn); do_div(fvco, mfd * rdiv * odiv); return (unsigned long)fvco; } +static unsigned long clk_fracn_gppll_recalc_rate(struct clk_hw *hw, unsigned long parent_rate) +{ + struct clk_fracn_gppll *pll = to_clk_fracn_gppll(hw); + const struct imx_fracn_gppll_rate_table *rate_table = pll->rate_table; + u32 pll_numerator, pll_denominator, pll_div; + u32 mfi, mfn, mfd, rdiv, odiv; + long rate = 0; + int i; + + pll_numerator = readl_relaxed(pll->base + PLL_NUMERATOR); + mfn = FIELD_GET(PLL_MFN_MASK, pll_numerator); + + pll_denominator = readl_relaxed(pll->base + PLL_DENOMINATOR); + mfd = FIELD_GET(PLL_MFD_MASK, pll_denominator); + + pll_div = readl_relaxed(pll->base + PLL_DIV); + mfi = FIELD_GET(PLL_MFI_MASK, pll_div); + + rdiv = FIELD_GET(PLL_RDIV_MASK, pll_div); + odiv = FIELD_GET(PLL_ODIV_MASK, pll_div); + + /* + * Sometimes, the recalculated rate has deviation due to + * the frac part. So find the accurate pll rate from the table + * first, if no match rate in the table, use the rate calculated + * from the equation below. + */ + for (i = 0; i < pll->rate_count; i++) { + if (rate_table[i].mfn == mfn && rate_table[i].mfi == mfi && + rate_table[i].mfd == mfd && rate_table[i].rdiv == rdiv && + rate_table[i].odiv == odiv) + rate = rate_table[i].rate; + } + + return rate ? (unsigned long) rate : __clk_fracn_gppll_recalc_rate(hw, parent_rate); +} + static int clk_fracn_gppll_wait_lock(struct clk_fracn_gppll *pll) { u32 val; @@ -199,6 +233,11 @@ static int clk_fracn_gppll_set_rate(struct clk_hw *hw, unsigned long drate, int ret; rate = imx_get_pll_settings(pll, drate); + if (!rate) { + pr_err("%s: Invalid rate : %lu for pll clk %s\n", __func__, + drate, clk_hw_get_name(hw)); + return -EINVAL; + } /* Hardware control select disable. PLL is control by register */ tmp = readl_relaxed(pll->base + PLL_CTRL); @@ -306,6 +345,122 @@ static const struct clk_ops clk_fracn_gppll_ops = { .set_rate = clk_fracn_gppll_set_rate, }; +/* This function fetches the original PLL parameters to use + * them later for ppb adjustment + */ +static void imx_fracn_gppll_init(struct clk_imx_pll *pll) +{ + struct clk_fracn_gppll *fracn_pll; + u32 pll_numerator, pll_denominator, pll_div, mfn; + + fracn_pll = (struct clk_fracn_gppll *) container_of(pll, + struct clk_fracn_gppll, imx_pll); + + pll_numerator = readl_relaxed(fracn_pll->base + PLL_NUMERATOR); + mfn = FIELD_GET(PLL_MFN_MASK, pll_numerator); + + /* Numerator is 30-bits signed value */ + fracn_pll->orig_mfn = sign_extend32(mfn, 29); + + pll_denominator = readl_relaxed(fracn_pll->base + PLL_DENOMINATOR); + fracn_pll->orig_mfd = FIELD_GET(PLL_MFD_MASK, pll_denominator); + + pll_div = readl_relaxed(fracn_pll->base + PLL_DIV); + fracn_pll->orig_mfi = FIELD_GET(PLL_MFI_MASK, pll_div); + + fracn_pll->orig_rdiv = FIELD_GET(PLL_RDIV_MASK, pll_div); + fracn_pll->orig_odiv = FIELD_GET(PLL_ODIV_MASK, pll_div); +} + +/** + * imx_fracn_gppll_adjust - Adjust the Audio pll by ppb. + * + * This function adjusts the audio pll by ppb (part per billion) and returns + * the exact number of ppb adjusted. + * The adjustment is done by only modifying the numerator of the fractional part + * of the audio PLL. + * Since the pllout = (ref * (mfi + mfn/mfd)) / (rdiv * odiv) + * and the adjusted value is + * pllout_new = pllout * (1 + ppb/1e9) which equals: + * (ref * (mfi + mfn_new/mfd)) / (rdiv * odiv) + * The new numerator (mfn_new) is calculated as following: + * mfn_new = (1e9 * mfn + (mfi * mfd + mfn) * ppb) / (1e9) + */ + +static int imx_fracn_gppll_adjust(struct clk_imx_pll *pll, int *ppb) +{ + s64 temp64; + s64 applied_ppb; + struct clk_fracn_gppll *fracn_pll; + int rc = IMX_CLK_PLL_SUCCESS; + + int req_ppb = *ppb; + + fracn_pll = (struct clk_fracn_gppll *) container_of(pll, + struct clk_fracn_gppll, imx_pll); + + /* Calculate the new numerator value */ + temp64 = ((s64) fracn_pll->orig_mfn * 1000000000) + + ((s64) fracn_pll->orig_mfi * fracn_pll->orig_mfd + (s64) fracn_pll->orig_mfn) * req_ppb; + + temp64 = div_s64(temp64, 1000000000); + + /* Sanity check on the new numerator value: + * - the value is inside the 30-bits signed values range + * - mfn/mfd should be in the range [-2, 2] + */ + if (temp64 > MFN_MAX_VALUE || temp64 < MFN_MIN_VALUE || (abs(temp64) > (2 * fracn_pll->orig_mfd))) { + rc = -IMX_CLK_PLL_PREC_ERR; + goto exit; + } + + /* Write the PLL control settings with the new numerator */ + + writel_relaxed(FIELD_PREP(PLL_MFN_MASK, temp64), fracn_pll->base + PLL_NUMERATOR); + + /* Calculate and return the actual applied ppb */ + applied_ppb = div64_s64((s64) (temp64 - fracn_pll->orig_mfn) * 1000000000, + fracn_pll->orig_mfn + fracn_pll->orig_mfd * (s64) fracn_pll->orig_mfi); + + *ppb = (int) applied_ppb; + + exit: + return rc; +} + +static unsigned long imx_fracn_gppll_get_rate(struct clk_imx_pll *pll, + unsigned long parent_rate) +{ + struct clk_fracn_gppll *fracn_pll; + + fracn_pll = (struct clk_fracn_gppll *) container_of(pll, + struct clk_fracn_gppll, imx_pll); + + return __clk_fracn_gppll_recalc_rate(&fracn_pll->hw, parent_rate); +} + +static int imx_fracn_gppll_set_rate(struct clk_imx_pll *pll, unsigned long rate, + unsigned long parent_rate) +{ + struct clk_fracn_gppll *fracn_pll; + int rc = IMX_CLK_PLL_SUCCESS; + + fracn_pll = (struct clk_fracn_gppll *) container_of(pll, + struct clk_fracn_gppll, imx_pll); + + if (clk_fracn_gppll_set_rate(&fracn_pll->hw, rate, parent_rate) < 0) + rc = -IMX_CLK_PLL_INVALID_PARAM; + + return rc; +} + +static const struct clk_imx_pll_ops imx_clk_fracn_gppll_ops = { + .set_rate = imx_fracn_gppll_set_rate, + .get_rate = imx_fracn_gppll_get_rate, + .adjust = imx_fracn_gppll_adjust, + .init = imx_fracn_gppll_init, +}; + struct clk_hw *imx_clk_fracn_gppll(const char *name, const char *parent_name, void __iomem *base, const struct imx_fracn_gppll_clk *pll_clk) { @@ -338,6 +493,11 @@ struct clk_hw *imx_clk_fracn_gppll(const char *name, const char *parent_name, vo return ERR_PTR(ret); } + pll->imx_pll.ops = &imx_clk_fracn_gppll_ops; + + if (imx_pll_register(&pll->imx_pll, name) < 0) + pr_warn("%s: failed to register %s into imx pll\n", __func__, name); + return hw; } EXPORT_SYMBOL_GPL(imx_clk_fracn_gppll); diff --git a/drivers/clk/imx/clk-imx8dxl-acm.c b/drivers/clk/imx/clk-imx8dxl-acm.c index b11254522bc5..bef8d33f543b 100644 --- a/drivers/clk/imx/clk-imx8dxl-acm.c +++ b/drivers/clk/imx/clk-imx8dxl-acm.c @@ -55,6 +55,14 @@ static const char *mclk_out_sels[] = { "dummy", }; +static const char *gpt_mux_clk_sels[] = { + "aud_pll_div_clk0_lpcg_clk", + "aud_pll_div_clk1_lpcg_clk", + "acm_aud_clk0_sel", + "acm_aud_clk1_sel", + "dummy", +}; + static const char *sai_mclk_sels[] = { "aud_pll_div_clk0_lpcg_clk", "aud_pll_div_clk1_lpcg_clk", @@ -76,6 +84,41 @@ static const char *mqs_mclk_sels[] = { "acm_aud_clk1_sel", }; +/* The number of cells for the GPT capture device tree attribute */ +#define OF_GPT_CAPTURE_CELLS_NB 2 + +static void imx8dxl_acm_gpt_input_mux(struct device_node *np, struct imx8dxl_acm_priv *priv) +{ + u32 len, reg_offset, event_sel_control, num_capture_select; + int i, offset; + + if (!of_get_property(np, "gpt-capture-select", &len)) + return; + + num_capture_select = len / (sizeof(u32) * OF_GPT_CAPTURE_CELLS_NB); + + for (i = 0; i < num_capture_select; i++) { + offset = i * OF_GPT_CAPTURE_CELLS_NB; + if (of_property_read_u32_index(np, + "gpt-capture-select", + offset, ®_offset)) { + pr_err("failed to read gpt register offset cell at offset %d\n", + offset); + return; + } + + if (of_property_read_u32_index(np, + "gpt-capture-select", + offset + 1, &event_sel_control)) { + pr_err("failed to read gpt event select control cell at offset %d\n", + offset + 1); + return; + } + + writel_relaxed(event_sel_control, priv->reg + reg_offset); + } +} + static int imx8dxl_acm_clk_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -137,6 +180,8 @@ static int imx8dxl_acm_clk_probe(struct platform_device *pdev) clks[IMX_ADMA_ACM_MCLKOUT0_SEL] = imx_dev_clk_mux(dev, "acm_mclkout0_sel", base+0x020000, 0, 3, mclk_out_sels, ARRAY_SIZE(mclk_out_sels)); clks[IMX_ADMA_ACM_MCLKOUT1_SEL] = imx_dev_clk_mux(dev, "acm_mclkout1_sel", base+0x030000, 0, 3, mclk_out_sels, ARRAY_SIZE(mclk_out_sels)); + clks[IMX_ADMA_ACM_GPT0_MUX_CLK_SEL] = imx_dev_clk_mux(dev, "acm_gpt0_mux_clk_sel", base+0x080000, 0, 3, gpt_mux_clk_sels, ARRAY_SIZE(gpt_mux_clk_sels)); + clks[IMX_ADMA_ACM_SAI0_MCLK_SEL] = imx_dev_clk_mux(dev, "acm_sai0_mclk_sel", base+0x0E0000, 0, 2, sai_mclk_sels, ARRAY_SIZE(sai_mclk_sels)); clks[IMX_ADMA_ACM_SAI1_MCLK_SEL] = imx_dev_clk_mux(dev, "acm_sai1_mclk_sel", base+0x0F0000, 0, 2, sai_mclk_sels, ARRAY_SIZE(sai_mclk_sels)); clks[IMX_ADMA_ACM_SAI2_MCLK_SEL] = imx_dev_clk_mux(dev, "acm_sai2_mclk_sel", base+0x100000, 0, 2, sai_mclk_sels, ARRAY_SIZE(sai_mclk_sels)); @@ -153,6 +198,8 @@ static int imx8dxl_acm_clk_probe(struct platform_device *pdev) ret = of_clk_add_provider(np, of_clk_src_onecell_get, clk_data); + imx8dxl_acm_gpt_input_mux(np, priv); + pm_runtime_put_sync(&pdev->dev); return ret; @@ -182,6 +229,7 @@ static int __maybe_unused imx8dxl_acm_runtime_suspend(struct device *dev) priv->regs[1] = readl_relaxed(priv->reg + 0x010000); priv->regs[2] = readl_relaxed(priv->reg + 0x020000); priv->regs[3] = readl_relaxed(priv->reg + 0x030000); + priv->regs[8] = readl_relaxed(priv->reg + 0x080000); priv->regs[14] = readl_relaxed(priv->reg + 0x0E0000); priv->regs[15] = readl_relaxed(priv->reg + 0x0F0000); priv->regs[16] = readl_relaxed(priv->reg + 0x100000); @@ -200,6 +248,7 @@ static int __maybe_unused imx8dxl_acm_runtime_resume(struct device *dev) writel_relaxed(priv->regs[1], priv->reg + 0x010000); writel_relaxed(priv->regs[2], priv->reg + 0x020000); writel_relaxed(priv->regs[3], priv->reg + 0x030000); + writel_relaxed(priv->regs[8], priv->reg + 0x080000); writel_relaxed(priv->regs[14], priv->reg + 0x0E0000); writel_relaxed(priv->regs[15], priv->reg + 0x0F0000); writel_relaxed(priv->regs[16], priv->reg + 0x100000); diff --git a/drivers/clk/imx/clk-imx8mq.c b/drivers/clk/imx/clk-imx8mq.c index 8e49b49ddc3e..1e9de317ed54 100644 --- a/drivers/clk/imx/clk-imx8mq.c +++ b/drivers/clk/imx/clk-imx8mq.c @@ -216,7 +216,7 @@ static const char * const imx8mq_pwm4_sels[] = {"osc_25m", "sys2_pll_100m", "sys "sys3_pll_out", "clk_ext2", "sys1_pll_80m", "video_pll1_out", }; static const char * const imx8mq_gpt1_sels[] = {"osc_25m", "sys2_pll_100m", "sys1_pll_400m", "sys1_pll_40m", - "sys1_pll_80m", "audio_pll1_out", "clk_ext1", }; + "video_pll1_out", "sys1_pll_80m", "audio_pll1_out", "clk_ext1", }; static const char * const imx8mq_wdog_sels[] = {"osc_25m", "sys1_pll_133m", "sys1_pll_160m", "vpu_pll_out", "sys2_pll_125m", "sys3_pll_out", "sys1_pll_80m", "sys2_pll_166m", }; diff --git a/drivers/clk/imx/clk-pll.c b/drivers/clk/imx/clk-pll.c new file mode 100644 index 000000000000..b77375a11add --- /dev/null +++ b/drivers/clk/imx/clk-pll.c @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2018 NXP + * + */ + +#include +#include +#include +#include +#include +#include "clk-pll.h" + + +static struct clk_imx_pll *clk_imx_pll_array[MAX_IMX_PLL_NUM] = { NULL }; + +struct clk_imx_pll *clk_imx_pll_get_by_name(const char *name) +{ + int i; + struct clk_imx_pll *pll = NULL; + + for (i = 0; i < MAX_IMX_PLL_NUM; i++) { + if (clk_imx_pll_array[i] && clk_imx_pll_array[i]->ops + && !strncmp(clk_imx_pll_array[i]->pll_name, name, MAX_PLL_NAME_SIZE)) { + + pll = clk_imx_pll_array[i]; + + /* Init all PLL original parameters*/ + if (pll && pll->ops && pll->ops->init) + pll->ops->init(pll); + + break; + } + } + + return pll; +} +EXPORT_SYMBOL(clk_imx_pll_get_by_name); + +int clk_imx_pll_adjust(struct clk_imx_pll *pll, int *ppb) +{ + if (pll && pll->ops && pll->ops->adjust) + return pll->ops->adjust(pll, ppb); + + return -IMX_CLK_PLL_INVALID_PARAM; +} +EXPORT_SYMBOL(clk_imx_pll_adjust); + +unsigned long clk_imx_pll_get_rate(struct clk_imx_pll *pll, + unsigned long parent_rate) +{ + if (pll && pll->ops && pll->ops->get_rate) + return pll->ops->get_rate(pll, parent_rate); + + return 0; +} +EXPORT_SYMBOL(clk_imx_pll_get_rate); + +int clk_imx_pll_set_rate(struct clk_imx_pll *pll, unsigned long rate, + unsigned long parent_rate) +{ + if (pll && pll->ops && pll->ops->set_rate) + return pll->ops->set_rate(pll, rate, parent_rate); + + return -IMX_CLK_PLL_INVALID_PARAM; +} +EXPORT_SYMBOL(clk_imx_pll_set_rate); + +int imx_pll_register(struct clk_imx_pll *pll, const char *name) +{ + int i; + + strncpy(pll->pll_name, name, MAX_PLL_NAME_SIZE); + + for (i = 0; i < MAX_IMX_PLL_NUM; i++) { + if (!clk_imx_pll_array[i]) { + clk_imx_pll_array[i] = pll; + return 0; + } + } + + return -EINVAL; +} diff --git a/drivers/clk/imx/clk-pll.h b/drivers/clk/imx/clk-pll.h new file mode 100644 index 000000000000..f12167842059 --- /dev/null +++ b/drivers/clk/imx/clk-pll.h @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2018 NXP + * + */ + +#ifndef __IMX_CLK_PLL_H +#define __IMX_CLK_PLL_H + +#include + +#define MAX_IMX_PLL_NUM 16 +#define MAX_PLL_NAME_SIZE 64 + +struct clk_imx_pll; + +struct clk_imx_pll_ops { + int (*set_rate)(struct clk_imx_pll *pll, unsigned long rate, + unsigned long parent_rate); + unsigned long (*get_rate)(struct clk_imx_pll *pll, + unsigned long parent_rate); + int (*adjust)(struct clk_imx_pll *pll, int *ppb); + void (*init)(struct clk_imx_pll *pll); +}; + +struct clk_imx_pll { + char pll_name[MAX_PLL_NAME_SIZE]; + const struct clk_imx_pll_ops *ops; +}; + +int imx_pll_register(struct clk_imx_pll *pll, const char *name); + +#endif /*__IMX_CLK_PLL_H*/ diff --git a/drivers/clk/imx/clk-pll14xx.c b/drivers/clk/imx/clk-pll14xx.c index c3a29e0d345e..50b848f23fd4 100644 --- a/drivers/clk/imx/clk-pll14xx.c +++ b/drivers/clk/imx/clk-pll14xx.c @@ -16,6 +16,7 @@ #include #include "clk.h" +#include "clk-pll.h" #define GNRL_CTL 0x0 #define DIV_CTL0 0x4 @@ -37,8 +38,13 @@ struct clk_pll14xx { struct clk_hw hw; void __iomem *base; + u32 orig_mdiv; + u32 orig_pdiv; + u32 orig_sdiv; + short int orig_kdiv; enum imx_pll14xx_type type; const struct imx_pll14xx_rate_table *rate_table; + struct clk_imx_pll imx_pll; int rate_count; }; @@ -519,6 +525,113 @@ static const struct clk_ops clk_pll1443x_ops = { .set_rate = clk_pll1443x_set_rate, }; +/* This function fetches the original PLL parameters to use + * them later for ppb adjustment + */ +static void imx_pll1443x_init(struct clk_imx_pll *pll) +{ + struct clk_pll14xx *pll14xx; + u32 pll_div_ctl0, pll_div_ctl1; + + pll14xx = (struct clk_pll14xx *) container_of(pll, + struct clk_pll14xx, imx_pll); + + pll_div_ctl0 = readl_relaxed(pll14xx->base + DIV_CTL0); + pll_div_ctl1 = readl_relaxed(pll14xx->base + DIV_CTL1); + + pll14xx->orig_kdiv = FIELD_GET(KDIV_MASK, pll_div_ctl1); + pll14xx->orig_mdiv = FIELD_GET(MDIV_MASK, pll_div_ctl0); + pll14xx->orig_pdiv = FIELD_GET(PDIV_MASK, pll_div_ctl0); + pll14xx->orig_sdiv = FIELD_GET(SDIV_MASK, pll_div_ctl0); +} + +/** + * imx_pll1443x_adjust - Adjust the Audio pll by ppb. + * + * This function adjust the audio pll by ppb (part per billion) and returns + * the exact number of ppb adjusted. + * The adjustment is done by only modifying the delta-sigma modulator(DSM) part + * of the audio PLL. + * Since the pllout = (parent_rate * (m + k/65536)) / (p * 2^s) + * and the adjusted value is + * pllout_new = pllout * (1 + ppb/1e9) which equals: + * (parent_rate * (m + k_new/65536)) / (p * 2^s) + * The new DSM (k_new) is calculated as the following: + * k_new = (1e9 * k + (m * 65536 + k) * ppb) / (1e9) + */ + +static int imx_pll1443x_adjust(struct clk_imx_pll *pll, int *ppb) +{ + s64 temp64; + s64 applied_ppb; + struct clk_pll14xx *pll14xx; + int rc = IMX_CLK_PLL_SUCCESS; + + int req_ppb = *ppb; + + pll14xx = (struct clk_pll14xx *) container_of(pll, + struct clk_pll14xx, imx_pll); + + /* Calcultate the new DSM value */ + temp64 = ((s64) pll14xx->orig_kdiv * 1000000000) + + ((s64) pll14xx->orig_mdiv * 65536 + (s64) pll14xx->orig_kdiv) * req_ppb; + + temp64 = div_s64(temp64, 1000000000); + + if (temp64 > KDIV_MAX || temp64 < KDIV_MIN) { + rc = -IMX_CLK_PLL_PREC_ERR; + goto exit; + } + + /* Write the PLL control settings with the new DSM + */ + + writel_relaxed(FIELD_PREP(KDIV_MASK, temp64), pll14xx->base + DIV_CTL1); + + /* Calculate and return the actual applied ppb */ + applied_ppb = div64_s64((s64) (temp64 - pll14xx->orig_kdiv) * 1000000000, + pll14xx->orig_kdiv + 65536 * (s64) pll14xx->orig_mdiv); + + *ppb = (int) applied_ppb; + + exit: + return rc; +} + +static unsigned long imx_pll1443x_get_rate(struct clk_imx_pll *pll, + unsigned long parent_rate) +{ + struct clk_pll14xx *pll14xx; + + pll14xx = (struct clk_pll14xx *) container_of(pll, + struct clk_pll14xx, imx_pll); + + return clk_pll14xx_recalc_rate(&pll14xx->hw, parent_rate); +} + +static int imx_pll1443x_set_rate(struct clk_imx_pll *pll, unsigned long rate, + unsigned long parent_rate) +{ + struct clk_pll14xx *pll14xx; + int rc = IMX_CLK_PLL_SUCCESS; + + pll14xx = (struct clk_pll14xx *) container_of(pll, + struct clk_pll14xx, imx_pll); + + if (clk_pll1443x_set_rate(&pll14xx->hw, rate, parent_rate) < 0) + rc = -IMX_CLK_PLL_INVALID_PARAM; + + return rc; +} + +static const struct clk_imx_pll_ops imx_clk_pll1443x_ops = { + .set_rate = imx_pll1443x_set_rate, + .get_rate = imx_pll1443x_get_rate, + .adjust = imx_pll1443x_adjust, + .init = imx_pll1443x_init, +}; + + struct clk_hw *imx_dev_clk_hw_pll14xx(struct device *dev, const char *name, const char *parent_name, void __iomem *base, const struct imx_pll14xx_clk *pll_clk) @@ -573,6 +686,13 @@ struct clk_hw *imx_dev_clk_hw_pll14xx(struct device *dev, const char *name, return ERR_PTR(ret); } + if (pll_clk->type == PLL_1443X) { + pll->imx_pll.ops = &imx_clk_pll1443x_ops; + + if (imx_pll_register(&pll->imx_pll, name) < 0) + pr_warn("Failed to register %s into imx pll\n", name); + } + return hw; } EXPORT_SYMBOL_GPL(imx_dev_clk_hw_pll14xx); diff --git a/drivers/clk/imx/clk-pllv3.c b/drivers/clk/imx/clk-pllv3.c index aeabb9b15675..dbd5f081d1e7 100644 --- a/drivers/clk/imx/clk-pllv3.c +++ b/drivers/clk/imx/clk-pllv3.c @@ -14,6 +14,7 @@ #include #include #include "clk.h" +#include "clk-pll.h" #define PLL_NUM_OFFSET 0x10 #define PLL_DENOM_OFFSET 0x20 @@ -47,6 +48,10 @@ */ struct clk_pllv3 { struct clk_hw hw; + struct clk_imx_pll imx_pll; + u32 orig_num; + u32 orig_denom; + u32 orig_div; void __iomem *base; u32 power_bit; bool powerup_set; @@ -440,6 +445,106 @@ static const struct clk_ops clk_pllv3_vf610_ops = { .set_rate = clk_pllv3_vf610_set_rate, }; +/** + * imx_pllv3_av_adjust - Adjust the Audio pll by ppb. + * + * This function adjust the audio pll by ppb (part per billion) and returns + * the exact number of ppb adjusted. + * The adjustment is done by only modifying the Audio PLL Numerator. + * Since the pllout = parent * (div + num/denom) and the adjusted Value is + * pllout_new = PLL_out * (1 + ppb/1e9) + * Also pllout_new = parent * (div + new_num/denom) + * The new numerator is calculated as the following: + * new_num = (div * ppb * denom + num * 1e9 + num * ppb) / (1e9) + */ + +static int imx_pllv3_av_adjust(struct clk_imx_pll *pll, int *ppb) +{ + struct clk_pllv3 *av_pll; + u64 temp64; + s32 applied_ppb; + int rc = IMX_CLK_PLL_SUCCESS; + + int req_ppb = *ppb; + + av_pll = (struct clk_pllv3 *) container_of(pll, struct clk_pllv3, + imx_pll); + + /*Calcultate the new PLL Numerator*/ + temp64 = (u64) av_pll->orig_denom * av_pll->orig_div * req_ppb + + (u64) av_pll->orig_num * 1000000000 + + (u64) av_pll->orig_num * req_ppb; + + do_div(temp64, 1000000000); + + if (temp64 >= av_pll->orig_denom) { + rc = -IMX_CLK_PLL_PREC_ERR; + goto exit; + } + + /*Write the new PLL num*/ + writel_relaxed((u32) temp64, av_pll->base + av_pll->num_offset); + + /*Calculate and return the actual applied ppb*/ + applied_ppb = div64_s64((s64) (temp64 - av_pll->orig_num) * 1000000000, + av_pll->orig_num + + (s64) av_pll->orig_denom * av_pll->orig_div); + + *ppb = (int) applied_ppb; + +exit: + return rc; + +} + +/* This function fetches the original PLL parameters to use + * them later for ppb adjustment + */ +static void imx_pllv3_av_init(struct clk_imx_pll *pll) +{ + struct clk_pllv3 *av_pll; + + av_pll = (struct clk_pllv3 *) container_of(pll, struct clk_pllv3, + imx_pll); + + av_pll->orig_num = readl_relaxed(av_pll->base + av_pll->num_offset); + av_pll->orig_denom = readl_relaxed(av_pll->base + av_pll->denom_offset); + av_pll->orig_div = readl_relaxed(av_pll->base) & av_pll->div_mask; +} + +static unsigned long imx_pllv3_av_get_rate(struct clk_imx_pll *pll, + unsigned long parent_rate) +{ + struct clk_pllv3 *av_pll; + + av_pll = (struct clk_pllv3 *) container_of(pll, struct clk_pllv3, + imx_pll); + + return clk_pllv3_av_recalc_rate(&av_pll->hw, parent_rate); +} + +static int imx_pllv3_av_set_rate(struct clk_imx_pll *pll, unsigned long rate, + unsigned long parent_rate) +{ + struct clk_pllv3 *av_pll; + int rc = IMX_CLK_PLL_SUCCESS; + + av_pll = (struct clk_pllv3 *) container_of(pll, struct clk_pllv3, + imx_pll); + + if (clk_pllv3_av_set_rate(&av_pll->hw, rate, parent_rate) < 0) + rc = -IMX_CLK_PLL_INVALID_PARAM; + + return rc; +} + +static const struct clk_imx_pll_ops imx_clk_pllv3_av_ops = { + .set_rate = imx_pllv3_av_set_rate, + .get_rate = imx_pllv3_av_get_rate, + .adjust = imx_pllv3_av_adjust, + .init = imx_pllv3_av_init, +}; + static unsigned long clk_pllv3_enet_recalc_rate(struct clk_hw *hw, unsigned long parent_rate) { @@ -495,6 +600,7 @@ struct clk_hw *imx_clk_hw_pllv3(enum imx_pllv3_type type, const char *name, fallthrough; case IMX_PLLV3_AV: ops = &clk_pllv3_av_ops; + pll->imx_pll.ops = &imx_clk_pllv3_av_ops; break; case IMX_PLLV3_ENET_IMX7: pll->power_bit = IMX7_ENET_PLL_POWER; @@ -532,5 +638,8 @@ struct clk_hw *imx_clk_hw_pllv3(enum imx_pllv3_type type, const char *name, return ERR_PTR(ret); } + if (imx_pll_register(&pll->imx_pll, name) < 0) + pr_warn("Failed to register %s into imx pll\n", name); + return hw; } diff --git a/drivers/clk/imx/clk-scu.c b/drivers/clk/imx/clk-scu.c index 6c54017529d5..bc683fb0dda9 100644 --- a/drivers/clk/imx/clk-scu.c +++ b/drivers/clk/imx/clk-scu.c @@ -18,6 +18,7 @@ #include #include "clk-scu.h" +#include "clk-pll.h" #define IMX_SIP_CPUFREQ 0xC2000001 #define IMX_SIP_SET_CPUFREQ 0x00 @@ -56,6 +57,8 @@ struct clk_scu { u8 parent_index; bool is_enabled; u32 rate; + unsigned long original_rate; + struct clk_imx_pll imx_pll; }; /* @@ -449,6 +452,82 @@ static const struct clk_ops clk_scu_pi_ops = { .set_rate = clk_scu_set_rate, }; +static void imx_scu_init(struct clk_imx_pll *pll) +{ + struct clk_scu *clk; + + clk = (struct clk_scu *) container_of(pll, struct clk_scu, imx_pll); + + clk->original_rate = clk_scu_recalc_rate(&clk->hw, 0); + + pr_info("SCU clock %p for PLL API init with original rate %lu\n", + clk, clk->original_rate); + + return; +} + +/* SCU is stating possible variation of +/-250Khz which gives around 300 ppm @ 786MHz + * Set a maximum adjustement at 250 ppm + */ +#define IMX_SCU_MAX_PPB_ADJUSTEMENT 250000 + +static int imx_scu_adjust(struct clk_imx_pll *pll, int *ppb) +{ + struct clk_scu * clk = (struct clk_scu *) container_of(pll, + struct clk_scu, imx_pll); + unsigned long new_rate; + int delta; + + if (!clk->original_rate) { + pr_warn_once("failed to get original rate for clock\n"); + return -IMX_CLK_PLL_LOCK_ERR; + } + + if (abs(*ppb) > IMX_SCU_MAX_PPB_ADJUSTEMENT) { + return -IMX_CLK_PLL_PREC_ERR; + } + + if (*ppb < 0) + delta = -1; + else + delta = 1; + + delta *= (int)div64_u64((u64)clk->original_rate * (u64)abs(*ppb), + 1000000000); + + new_rate = clk->original_rate + delta; + + if (clk_scu_set_rate(&clk->hw, new_rate, 0) < 0) + return -IMX_CLK_PLL_INVALID_PARAM; + + return IMX_CLK_PLL_SUCCESS; +} + +static int imx_scu_set_rate(struct clk_imx_pll *pll, unsigned long rate, + unsigned long parent_rate) +{ + struct clk_scu * clk = (struct clk_scu *) container_of(pll, + struct clk_scu, imx_pll); + + return clk_scu_set_rate(&clk->hw, rate, 0); +} + +static unsigned long imx_scu_get_rate(struct clk_imx_pll *pll, + unsigned long parent_rate) +{ + struct clk_scu * clk = (struct clk_scu *) container_of(pll, + struct clk_scu, imx_pll); + + return clk_scu_recalc_rate(&clk->hw, 0); +} + +static const struct clk_imx_pll_ops imx_clk_scu_ops = { + .set_rate = imx_scu_set_rate, + .get_rate = imx_scu_get_rate, + .adjust = imx_scu_adjust, + .init = imx_scu_init, +}; + struct clk_hw *__imx_clk_scu(struct device *dev, const char *name, const char * const *parents, int num_parents, u32 rsrc_id, u8 clk_type) @@ -497,6 +576,13 @@ struct clk_hw *__imx_clk_scu(struct device *dev, const char *name, if (dev) dev_set_drvdata(dev, clk); + if (rsrc_id == IMX_SC_R_AUDIO_PLL_0 && clk_type == IMX_SC_PM_CLK_PLL) { + clk->imx_pll.ops = &imx_clk_scu_ops; + + if (imx_pll_register(&clk->imx_pll, name) < 0) + pr_warn("%s: failed to register %s into imx pll\n", __func__, name); + } + return hw; } diff --git a/drivers/firmware/imx/scu-pd.c b/drivers/firmware/imx/scu-pd.c index fbef30aa49ba..ce31ff3aab1c 100755 --- a/drivers/firmware/imx/scu-pd.c +++ b/drivers/firmware/imx/scu-pd.c @@ -147,6 +147,7 @@ static const struct imx_sc_pd_range imx8qxp_scu_pd_ranges[] = { { "esai1", IMX_SC_R_ESAI_1, 1, false, 0 }, { "spdif0", IMX_SC_R_SPDIF_0, 1, false, 0 }, { "spdif1", IMX_SC_R_SPDIF_1, 1, false, 0 }, + { "gpt", IMX_SC_R_GPT_5, 4, true, 5 }, { "sai", IMX_SC_R_SAI_0, 3, true, 0 }, { "sai3", IMX_SC_R_SAI_3, 1, false, 0 }, { "sai4", IMX_SC_R_SAI_4, 1, false, 0 }, diff --git a/drivers/gpu/drm/i915/Kconfig b/drivers/gpu/drm/i915/Kconfig index 6b10868ec72f..1fbdb7b4e6e1 100644 --- a/drivers/gpu/drm/i915/Kconfig +++ b/drivers/gpu/drm/i915/Kconfig @@ -3,7 +3,6 @@ config DRM_I915 tristate "Intel 8xx/9xx/G3x/G4x/HD Graphics" depends on DRM depends on X86 && PCI - depends on !PREEMPT_RT select INTEL_GTT if X86 select INTERVAL_TREE # we need shmfs for the swappable backing store, and in particular diff --git a/drivers/gpu/drm/i915/display/intel_crtc.c b/drivers/gpu/drm/i915/display/intel_crtc.c index 6792a9056f46..43cedfef104f 100644 --- a/drivers/gpu/drm/i915/display/intel_crtc.c +++ b/drivers/gpu/drm/i915/display/intel_crtc.c @@ -521,7 +521,8 @@ void intel_pipe_update_start(struct intel_crtc_state *new_crtc_state) */ intel_psr_wait_for_idle_locked(new_crtc_state); - local_irq_disable(); + if (!IS_ENABLED(CONFIG_PREEMPT_RT)) + local_irq_disable(); crtc->debug.min_vbl = min; crtc->debug.max_vbl = max; @@ -546,11 +547,13 @@ void intel_pipe_update_start(struct intel_crtc_state *new_crtc_state) break; } - local_irq_enable(); + if (!IS_ENABLED(CONFIG_PREEMPT_RT)) + local_irq_enable(); timeout = schedule_timeout(timeout); - local_irq_disable(); + if (!IS_ENABLED(CONFIG_PREEMPT_RT)) + local_irq_disable(); } finish_wait(wq, &wait); @@ -583,7 +586,8 @@ void intel_pipe_update_start(struct intel_crtc_state *new_crtc_state) return; irq_disable: - local_irq_disable(); + if (!IS_ENABLED(CONFIG_PREEMPT_RT)) + local_irq_disable(); } #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_VBLANK_EVADE) @@ -684,7 +688,8 @@ void intel_pipe_update_end(struct intel_crtc_state *new_crtc_state) */ intel_vrr_send_push(new_crtc_state); - local_irq_enable(); + if (!IS_ENABLED(CONFIG_PREEMPT_RT)) + local_irq_enable(); if (intel_vgpu_active(dev_priv)) return; diff --git a/drivers/gpu/drm/i915/gt/intel_breadcrumbs.c b/drivers/gpu/drm/i915/gt/intel_breadcrumbs.c index ecc990ec1b95..8d04b10681f0 100644 --- a/drivers/gpu/drm/i915/gt/intel_breadcrumbs.c +++ b/drivers/gpu/drm/i915/gt/intel_breadcrumbs.c @@ -312,10 +312,9 @@ void __intel_breadcrumbs_park(struct intel_breadcrumbs *b) /* Kick the work once more to drain the signalers, and disarm the irq */ irq_work_sync(&b->irq_work); while (READ_ONCE(b->irq_armed) && !atomic_read(&b->active)) { - local_irq_disable(); - signal_irq_work(&b->irq_work); - local_irq_enable(); + irq_work_queue(&b->irq_work); cond_resched(); + irq_work_sync(&b->irq_work); } } diff --git a/drivers/gpu/drm/i915/gt/intel_execlists_submission.c b/drivers/gpu/drm/i915/gt/intel_execlists_submission.c index fc4a84628985..fc937697fe14 100644 --- a/drivers/gpu/drm/i915/gt/intel_execlists_submission.c +++ b/drivers/gpu/drm/i915/gt/intel_execlists_submission.c @@ -1302,7 +1302,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine) * and context switches) submission. */ - spin_lock(&sched_engine->lock); + spin_lock_irq(&sched_engine->lock); /* * If the queue is higher priority than the last @@ -1402,7 +1402,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine) * Even if ELSP[1] is occupied and not worthy * of timeslices, our queue might be. */ - spin_unlock(&sched_engine->lock); + spin_unlock_irq(&sched_engine->lock); return; } } @@ -1428,7 +1428,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine) if (last && !can_merge_rq(last, rq)) { spin_unlock(&ve->base.sched_engine->lock); - spin_unlock(&engine->sched_engine->lock); + spin_unlock_irq(&engine->sched_engine->lock); return; /* leave this for another sibling */ } @@ -1590,7 +1590,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine) */ sched_engine->queue_priority_hint = queue_prio(sched_engine); i915_sched_engine_reset_on_empty(sched_engine); - spin_unlock(&sched_engine->lock); + spin_unlock_irq(&sched_engine->lock); /* * We can skip poking the HW if we ended up with exactly the same set @@ -1616,13 +1616,6 @@ static void execlists_dequeue(struct intel_engine_cs *engine) } } -static void execlists_dequeue_irq(struct intel_engine_cs *engine) -{ - local_irq_disable(); /* Suspend interrupts across request submission */ - execlists_dequeue(engine); - local_irq_enable(); /* flush irq_work (e.g. breadcrumb enabling) */ -} - static void clear_ports(struct i915_request **ports, int count) { memset_p((void **)ports, NULL, count); @@ -2476,7 +2469,7 @@ static void execlists_submission_tasklet(struct tasklet_struct *t) } if (!engine->execlists.pending[0]) { - execlists_dequeue_irq(engine); + execlists_dequeue(engine); start_timeslice(engine); } diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c index f93ffa6626a5..6e9d033cf808 100644 --- a/drivers/gpu/drm/i915/i915_irq.c +++ b/drivers/gpu/drm/i915/i915_irq.c @@ -917,7 +917,8 @@ static bool i915_get_crtc_scanoutpos(struct drm_crtc *_crtc, */ spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); - /* preempt_disable_rt() should go right here in PREEMPT_RT patchset. */ + if (IS_ENABLED(CONFIG_PREEMPT_RT)) + preempt_disable(); /* Get optional system timestamp before query. */ if (stime) @@ -981,7 +982,8 @@ static bool i915_get_crtc_scanoutpos(struct drm_crtc *_crtc, if (etime) *etime = ktime_get(); - /* preempt_enable_rt() should go right here in PREEMPT_RT patchset. */ + if (IS_ENABLED(CONFIG_PREEMPT_RT)) + preempt_enable(); spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); diff --git a/drivers/gpu/drm/i915/i915_request.c b/drivers/gpu/drm/i915/i915_request.c index 7ce126a01cbf..64a032dfaa90 100644 --- a/drivers/gpu/drm/i915/i915_request.c +++ b/drivers/gpu/drm/i915/i915_request.c @@ -609,7 +609,6 @@ bool __i915_request_submit(struct i915_request *request) RQ_TRACE(request, "\n"); - GEM_BUG_ON(!irqs_disabled()); lockdep_assert_held(&engine->sched_engine->lock); /* @@ -718,7 +717,6 @@ void __i915_request_unsubmit(struct i915_request *request) */ RQ_TRACE(request, "\n"); - GEM_BUG_ON(!irqs_disabled()); lockdep_assert_held(&engine->sched_engine->lock); /* diff --git a/drivers/gpu/drm/i915/i915_trace.h b/drivers/gpu/drm/i915/i915_trace.h index 37b5c9e9d260..73f29d8008f0 100644 --- a/drivers/gpu/drm/i915/i915_trace.h +++ b/drivers/gpu/drm/i915/i915_trace.h @@ -6,6 +6,10 @@ #if !defined(_I915_TRACE_H_) || defined(TRACE_HEADER_MULTI_READ) #define _I915_TRACE_H_ +#ifdef CONFIG_PREEMPT_RT +#define NOTRACE +#endif + #include #include #include @@ -323,7 +327,7 @@ DEFINE_EVENT(i915_request, i915_request_add, TP_ARGS(rq) ); -#if defined(CONFIG_DRM_I915_LOW_LEVEL_TRACEPOINTS) +#if defined(CONFIG_DRM_I915_LOW_LEVEL_TRACEPOINTS) && !defined(NOTRACE) DEFINE_EVENT(i915_request, i915_request_guc_submit, TP_PROTO(struct i915_request *rq), TP_ARGS(rq) diff --git a/drivers/gpu/drm/i915/i915_utils.h b/drivers/gpu/drm/i915/i915_utils.h index 6c14d13364bf..de58855e6926 100644 --- a/drivers/gpu/drm/i915/i915_utils.h +++ b/drivers/gpu/drm/i915/i915_utils.h @@ -294,7 +294,7 @@ wait_remaining_ms_from_jiffies(unsigned long timestamp_jiffies, int to_wait_ms) #define wait_for(COND, MS) _wait_for((COND), (MS) * 1000, 10, 1000) /* If CONFIG_PREEMPT_COUNT is disabled, in_atomic() always reports false. */ -#if defined(CONFIG_DRM_I915_DEBUG) && defined(CONFIG_PREEMPT_COUNT) +#if defined(CONFIG_DRM_I915_DEBUG) && defined(CONFIG_PREEMPT_COUNT) && !defined(CONFIG_PREEMPT_RT) # define _WAIT_FOR_ATOMIC_CHECK(ATOMIC) WARN_ON_ONCE((ATOMIC) && !in_atomic()) #else # define _WAIT_FOR_ATOMIC_CHECK(ATOMIC) do { } while (0) diff --git a/drivers/irqchip/Kconfig b/drivers/irqchip/Kconfig index 1602a37db178..b11fdbf93b80 100644 --- a/drivers/irqchip/Kconfig +++ b/drivers/irqchip/Kconfig @@ -706,4 +706,8 @@ config SUNPLUS_SP7021_INTC chained controller, routing all interrupt source in P-Chip to the primary controller on C-Chip. +config BAREMETAL + bool + depends on ARM_GIC || ARM_GIC_V3 + endmenu diff --git a/drivers/irqchip/Makefile b/drivers/irqchip/Makefile index 87b49a10962c..1c6fe6f29519 100644 --- a/drivers/irqchip/Makefile +++ b/drivers/irqchip/Makefile @@ -121,3 +121,4 @@ obj-$(CONFIG_IRQ_IDT3243X) += irq-idt3243x.o obj-$(CONFIG_APPLE_AIC) += irq-apple-aic.o obj-$(CONFIG_MCHP_EIC) += irq-mchp-eic.o obj-$(CONFIG_SUNPLUS_SP7021_INTC) += irq-sp7021-intc.o +obj-$(CONFIG_BAREMETAL) += ipi-baremetal.o \ No newline at end of file diff --git a/drivers/irqchip/ipi-baremetal.c b/drivers/irqchip/ipi-baremetal.c new file mode 100644 index 000000000000..58f88546b6f0 --- /dev/null +++ b/drivers/irqchip/ipi-baremetal.c @@ -0,0 +1,454 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * IPI for inter-core communiction for NXP Layerscape baremetal + * + * Copyright 2018-2023 NXP + * + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +int pid; +int mycoreid; + +#undef IPI_BAREMETAL_SIGNAL + +#define DEVICE_NAME "ipi_bm" + +/* + * choose 50 as baremetal signal number. + * real-time signals are in the range of 33 to 64. + */ +#define SIG_BM 50 + +#ifndef IPI_BAREMETAL_SIGNAL +void __iomem *share_base; +#ifdef CONFIG_SOC_IMX6Q_BAREMETAL +#define GICD_BASE 0x00A01000 +#define GICD_SIZE 0x1000 +#define GICC_BASE 0x00A00100 +#define GICC_SIZE 0x1000 +#define GIC_DIST_IGROUP 0x080 +#define GIC_DIST_CTRL 0x000 +#define GIC_CPU_CTRL 0x00 +#define GICD_ENABLE 0x3 +#define GICC_ENABLE 0x7 +#endif +#if defined(CONFIG_SOC_IMX6Q_BAREMETAL) +#define CONFIG_SYS_DDR_SDRAM_BASE 0x10000000UL +#define CONFIG_SYS_DDR_SDRAM_SLAVE_SIZE (128 * 1024 * 1024) +#define CONFIG_SYS_DDR_SDRAM_MASTER_SIZE (512 * 1024 * 1024) +#elif defined(CONFIG_LX2160A_BAREMETAL) +#define CONFIG_SYS_DDR_SDRAM_BASE 0x80000000UL +#define CONFIG_SYS_DDR_SDRAM_SLAVE_SIZE (64 * 1024 * 1024) +#define CONFIG_SYS_DDR_SDRAM_MASTER_SIZE (512 * 1024 * 1024) +#elif defined(CONFIG_IMX8M_BAREMETAL) +#define CONFIG_SYS_DDR_SDRAM_SLAVE_ADDR (0x60000000) +#define CONFIG_SYS_DDR_SDRAM_SLAVE_SIZE (32 * 1024 * 1024) +#define CONFIG_SYS_DDR_SDRAM_SLAVE_RESERVE_SIZE (32 * 1024 *1024) +#elif defined(CONFIG_IMX93_BAREMETAL) +#define CONFIG_SYS_DDR_SDRAM_SLAVE_ADDR (0xb0000000) +#define CONFIG_SYS_DDR_SDRAM_SLAVE_SIZE (32 * 1024 * 1024) +#define CONFIG_SYS_DDR_SDRAM_SLAVE_RESERVE_SIZE (32 * 1024 *1024) +#else +#define CONFIG_SYS_DDR_SDRAM_BASE 0x80000000UL +#define CONFIG_SYS_DDR_SDRAM_SLAVE_SIZE (256 * 1024 * 1024) +#define CONFIG_SYS_DDR_SDRAM_MASTER_SIZE (512 * 1024 * 1024) +#endif + +#if defined(CONFIG_IMX8M_BAREMETAL) || defined(CONFIG_IMX93_BAREMETAL) +#define CONFIG_SYS_DDR_SDRAM_SHARE_BASE (CONFIG_SYS_DDR_SDRAM_SLAVE_ADDR \ + + CONFIG_SYS_DDR_SDRAM_SLAVE_SIZE*(CONFIG_MAX_CPUS-1)) +#define CONFIG_SYS_DDR_SDRAM_SHARE_RESERVE_SIZE (4 * 1024 * 1024) +#else +#define CONFIG_SYS_DDR_SDRAM_SHARE_BASE \ + (CONFIG_SYS_DDR_SDRAM_BASE + CONFIG_SYS_DDR_SDRAM_MASTER_SIZE \ + + CONFIG_SYS_DDR_SDRAM_SLAVE_SIZE * (CONFIG_MAX_CPUS - 1)) + +#define CONFIG_SYS_DDR_SDRAM_SHARE_RESERVE_SIZE (16 * 1024 * 1024) +#endif + +#if defined(CONFIG_SOC_IMX6Q_BAREMETAL) +#define CONFIG_SYS_DDR_SDRAM_SHARE_SIZE \ + ((128 * 1024 * 1024) - CONFIG_SYS_DDR_SDRAM_SHARE_RESERVE_SIZE) +#elif defined(CONFIG_LX2160A_BAREMETAL) +#define CONFIG_SYS_DDR_SDRAM_SHARE_SIZE \ + ((64 * 1024 * 1024) - CONFIG_SYS_DDR_SDRAM_SHARE_RESERVE_SIZE) +#elif defined(CONFIG_IMX8M_BAREMETAL) || defined(CONFIG_IMX93_BAREMETAL) +#define CONFIG_SYS_DDR_SDRAM_SHARE_SIZE (CONFIG_SYS_DDR_SDRAM_SLAVE_RESERVE_SIZE \ + - CONFIG_SYS_DDR_SDRAM_SHARE_RESERVE_SIZE) +#else +#define CONFIG_SYS_DDR_SDRAM_SHARE_SIZE \ + ((256 * 1024 * 1024) - CONFIG_SYS_DDR_SDRAM_SHARE_RESERVE_SIZE) +#endif +#define CONFIG_SYS_DDR_SDRAM_SHARE_RESERVE_BASE \ + (CONFIG_SYS_DDR_SDRAM_SHARE_BASE + CONFIG_SYS_DDR_SDRAM_SHARE_SIZE) + +/* number of descriptor for each ring */ +#define ICC_RING_ENTRY 128 +/* size of each block */ +#define ICC_BLOCK_UNIT_SIZE (4 * 1024) +/* 2M space for core's ring and desc struct */ +#define ICC_RING_DESC_SPACE (2 * 1024 * 1024) + +/* share memory size for each core icc */ +#define ICC_CORE_MEM_SPACE (CONFIG_SYS_DDR_SDRAM_SHARE_SIZE / CONFIG_MAX_CPUS) +/* share memory base for core x */ +#define ICC_CORE_MEM_BASE_PHY(x) \ + (CONFIG_SYS_DDR_SDRAM_SHARE_BASE + (x) * ICC_CORE_MEM_SPACE) +/* share memory base for core x */ +#define ICC_CORE_MEM_BASE(x) \ + ((unsigned long)share_base + (x) * ICC_CORE_MEM_SPACE) +/* the ring struct addr of core x ring y */ +#define ICC_CORE_RING_BASE(x, y) \ + (ICC_CORE_MEM_BASE(x) + (y) * sizeof(struct icc_ring)) +/* the desc struct addr of core x */ +#define ICC_CORE_DESC_BASE_PHY(x) \ + (ICC_CORE_MEM_BASE_PHY(x) + CONFIG_MAX_CPUS * sizeof(struct icc_ring)) +/* + * The core x block memory base addr for icc data transfer. + * The beginning 2M space of core x icc memory is for + * core x ring and desc struct. + */ +#define ICC_CORE_BLOCK_BASE_PHY(x) \ + (ICC_CORE_MEM_BASE_PHY(x) + ICC_RING_DESC_SPACE) +#define ICC_CORE_BLOCK_BASE(x) (ICC_CORE_MEM_BASE(x) + ICC_RING_DESC_SPACE) +#define ICC_CORE_BLOCK_END_PHY(x) \ + (ICC_CORE_MEM_BASE_PHY(x) + ICC_CORE_MEM_SPACE) +#define ICC_CORE_BLOCK_END(x) (ICC_CORE_MEM_BASE(x) + ICC_CORE_MEM_SPACE) +#define ICC_CORE_BLOCK_COUNT \ + ((ICC_CORE_MEM_SPACE - ICC_RING_DESC_SPACE)/ICC_BLOCK_UNIT_SIZE) + +#define ICC_PHY2VIRT(x) \ + (((void *)x - ICC_CORE_MEM_BASE_PHY(mycoreid)) \ + + ICC_CORE_MEM_BASE(mycoreid)) +#define ICC_VIRT2PHY(x) \ + (((void *)x - ICC_CORE_MEM_BASE(mycoreid)) \ + + ICC_CORE_MEM_BASE_PHY(mycoreid)) + +#define IPIDEV_IOCIRQ 1 + +#define ICC_CMD_TX_DATA 0x00 +#define ICC_CMD_DUMP_TIME 0x01 + +struct icc_desc { + unsigned long block_addr; /* block address */ + unsigned int byte_count; /* available bytes */ + unsigned int option_mode; /* option mode for this icc irq */ +}; + +struct icc_ring { + unsigned int src_coreid; /* which core created the ring */ + unsigned int dest_coreid; /* which core the ring sends SGI to */ + unsigned int interrupt; /* which interrupt (SGI) be used */ + unsigned int desc_num; /* number of descriptor */ + struct icc_desc *desc; /* pointer of the first descriptor */ + unsigned int desc_head; /* modified by producer */ + unsigned int desc_tail; /* modified by consumer */ + unsigned long busy_counts; /* statistic: ring full */ + unsigned long interrupt_counts; /* statistic: total sent number */ + unsigned int irq_status; /* status of the ring, set by producer, reset by consumer */ +}; +#endif + +int ipi_baremetal_open(struct inode *inode, struct file *filp) +{ + return 0; +} + +ssize_t ipi_baremetal_read(struct file *file, + char __user *buff, size_t count, loff_t *offp) +{ + return 0; +} + +ssize_t ipi_baremetal_write(struct file *file, + const char __user *buff, size_t count, loff_t *offp) +{ + char mybuf[10]; + int ret; + + /* read the value from user space */ + if (count > 10) + return -EINVAL; + copy_from_user(mybuf, buff, count); + ret = kstrtoint(mybuf, 0, &pid); + if (ret) + return -EINVAL; + + return 0; +} + +static int ipi_baremetal_release(struct inode *inode, struct file *file) +{ + return 0; +} + +#ifndef CONFIG_LS1021A_BAREMETAL +static long ipi_baremetal_ioctl(struct file *file, + unsigned int cmd, unsigned long arg) +{ + unsigned long val; + unsigned long __user *argp = (unsigned long __user *)arg; + int err; +#if defined(CONFIG_LX2160A_BAREMETAL) + unsigned long i, cluster, mask; +#endif + + err = copy_from_user(&val, argp, sizeof(val)); + if (err) + return -EFAULT; + + val |= ((unsigned long)1 << 40); + +#if defined(CONFIG_LX2160A_BAREMETAL) + val = *(unsigned long *)arg; + + for (i = 0; i < 16; i++) { + if ((val >> i) & 0x1) { + cluster = i / 2; + mask = i % 2; + val |= (1 << mask) | (cluster << 16); + } + } +#endif + switch (cmd) { + case IPIDEV_IOCIRQ: + write_sysreg_s(val, SYS_ICC_SGI1R_EL1); + break; + default: + return -EINVAL; + } + return 0; +} +#endif + +static int icc_ring_empty(struct icc_ring *ring) +{ + if (ring->desc_tail == ring->desc_head) + return 1; + return 0; +} + +/* how many rx blocks are valid waiting to be handled */ +static int icc_ring_valid(struct icc_ring *ring) +{ + int valid; + + if (icc_ring_empty(ring)) + return 0; + + if (ring->desc_head > ring->desc_tail) + valid = ring->desc_head - ring->desc_tail; + else + valid = ring->desc_num - ring->desc_tail + ring->desc_head; + return valid; +} + +int ipi_baremetal_handle(u32 irqnr, u32 irqsrc) +{ +#ifdef IPI_BAREMETAL_SIGNAL + struct siginfo info; + struct task_struct *t; + int si_data; + int ret; + + if (!pid) + return 0; + + si_data = (irqnr << 16) | irqsrc; + /* send the signal */ + memset(&info, 0, sizeof(struct siginfo)); + info.si_signo = SIG_BM; + info.si_code = SI_QUEUE; + info.si_int = si_data; + + rcu_read_lock(); + /* find the task_struct associated with this pid */ + t = find_task_by_vpid(pid); + if (t == NULL) { + pr_info("no such pid\n"); + rcu_read_unlock(); + return -ENODEV; + } + rcu_read_unlock(); + + /* send the signal */ + ret = send_sig_info(SIG_BM, &info, t); + if (ret < 0) { + pr_info("error sending signal\n"); + return ret; + } +#else + struct icc_ring *ring; + struct icc_desc *desc; + struct icc_desc *desc_phy; + unsigned long block_addr; + unsigned int byte_count, option_mode; + int i, valid; + int hw_irq, src_coreid; + + hw_irq = irqnr; + src_coreid = irqsrc; + + if (src_coreid == mycoreid) { + pr_err("Do not support self-icc now!\n"); + return -1; + } + + /* get the ring for this core from source core */ + ring = (struct icc_ring *)ICC_CORE_RING_BASE(src_coreid, mycoreid); + valid = icc_ring_valid(ring); + for (i = 0; i < valid; i++) { + desc_phy = ring->desc + ring->desc_tail; + desc = ICC_PHY2VIRT(desc_phy); + option_mode = desc->option_mode; + + if (option_mode & ICC_CMD_DUMP_TIME) { + pr_info("Get the SGI from CoreID: %d\n", src_coreid); + } else { + block_addr = desc->block_addr; + byte_count = desc->byte_count; + + if ((*(char *)ICC_PHY2VIRT(block_addr)) != 0x5a) + pr_info("Get the ICC from core %d; block: 0x%lx, bytes: %d, value: 0x%x\n", + src_coreid, block_addr, byte_count, + (*(char *)ICC_PHY2VIRT(block_addr))); + } + + /* add desc_tail */ + ring->desc_tail = (ring->desc_tail + 1) % ring->desc_num; + } +#endif + return 0; +} + +static const struct vm_operations_struct shd_mmap_mem_ops = { +#ifdef CONFIG_HAVE_IOREMAP_PROT + .access = generic_access_phys +#endif +}; + +static int shd_mmap_mem(struct file *file, struct vm_area_struct *vma) +{ + size_t size = vma->vm_end - vma->vm_start; + +#if defined(CONFIG_LS1021A_BAREMETAL) || defined(CONFIG_SOC_IMX6Q_BAREMETAL) + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); +#else + vma->vm_page_prot = pgprot_cached(vma->vm_page_prot); +#endif + vma->vm_ops = &shd_mmap_mem_ops; + + /* Remap-pfn-range will mark the range VM_IO */ + if (remap_pfn_range(vma, + vma->vm_start, + vma->vm_pgoff, + size, + vma->vm_page_prot)) { + return -EAGAIN; + } + return 0; +} + +const struct file_operations ipi_bm_ops = { + .owner = THIS_MODULE, + .open = ipi_baremetal_open, + .release = ipi_baremetal_release, + .read = ipi_baremetal_read, + .write = ipi_baremetal_write, + .mmap = shd_mmap_mem, +#ifndef CONFIG_LS1021A_BAREMETAL + .unlocked_ioctl = ipi_baremetal_ioctl, +#endif +}; + +static struct miscdevice ipi_bm_misc = { + .minor = MISC_DYNAMIC_MINOR, + .name = DEVICE_NAME, + .fops = &ipi_bm_ops, +}; + +#ifdef CONFIG_SOC_IMX6Q_BAREMETAL +void gic_enable_dist(void) +{ + void __iomem *gicd_base, *gicc_base; + + gicd_base = ioremap((phys_addr_t)GICD_BASE, GICD_SIZE); + if (!gicd_base) { + pr_err("failed to remap gicd base for ICC\n"); + return -ENOMEM; + } + gicc_base = ioremap((phys_addr_t)GICC_BASE, GICC_SIZE); + if (!gicc_base) { + pr_err("failed to remap gicc base for ICC\n"); + return -ENOMEM; + } + /* set the SGI interrupts for this core to group 1 */ + writel(0xffffffff, gicd_base + GIC_DIST_IGROUP); + writel(GICD_ENABLE, gicd_base + GIC_DIST_CTRL); + writel(GICC_ENABLE, gicc_base + GIC_CPU_CTRL); + iounmap(gicd_base); + iounmap(gicc_base); +} +#endif + +static int __init ipi_baremetal_init(void) +{ + int ret; + + pr_info("NXP inter-core communiction IRQ driver\n"); +#ifndef IPI_BAREMETAL_SIGNAL +#if defined(CONFIG_LS1021A_BAREMETAL) || defined(CONFIG_SOC_IMX6Q_BAREMETAL) + share_base = ioremap((phys_addr_t)CONFIG_SYS_DDR_SDRAM_SHARE_BASE, + CONFIG_SYS_DDR_SDRAM_SHARE_SIZE); +#else + share_base = ioremap_cache((phys_addr_t)CONFIG_SYS_DDR_SDRAM_SHARE_BASE, + CONFIG_SYS_DDR_SDRAM_SHARE_SIZE); +#endif + if (!share_base) { + pr_err("failed to remap share base (%lu/%u) for ICC\n", + CONFIG_SYS_DDR_SDRAM_SHARE_BASE, + CONFIG_SYS_DDR_SDRAM_SHARE_SIZE); + return -ENOMEM; + } + mycoreid = 0; +#ifdef CONFIG_SOC_IMX6Q_BAREMETAL + gic_enable_dist(); +#endif +#endif + ret = misc_register(&ipi_bm_misc); + if (ret < 0) { + pr_info("Register ipi_bm error! ret: %d\n", ret); + return ret; + } + pr_info("ipi_bm device created!\n"); + return 0; +} + +static void __exit ipi_baremetal_exit(void) +{ + pid = 0; +#ifndef IPI_BAREMETAL_SIGNAL + iounmap(share_base); +#endif + misc_deregister(&ipi_bm_misc); + pr_info("ipi_bm device deleted!\n"); +} + +module_init(ipi_baremetal_init); +module_exit(ipi_baremetal_exit); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("NXP"); +MODULE_DESCRIPTION("NXP inter-core communiction IPI driver"); diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c index eccd6bb52a0b..9d06b08b8baf 100644 --- a/drivers/irqchip/irq-gic-v3.c +++ b/drivers/irqchip/irq-gic-v3.c @@ -27,6 +27,10 @@ #include #include +#ifdef CONFIG_BAREMETAL +#include +#endif + #include #include #include @@ -1392,14 +1396,14 @@ static void __init gic_smp_init(void) "irqchip/arm/gicv3:starting", gic_starting_cpu, NULL); - /* Register all 8 non-secure SGIs */ - base_sgi = __irq_domain_alloc_irqs(gic_data.domain, -1, 8, + /* Register all 16 non-secure SGIs */ + base_sgi = __irq_domain_alloc_irqs(gic_data.domain, -1, 16, NUMA_NO_NODE, &sgi_fwspec, false, NULL); if (WARN_ON(base_sgi <= 0)) return; - set_smp_ipi_range(base_sgi, 8); + set_smp_ipi_range(base_sgi, 16); } static int gic_set_affinity(struct irq_data *d, const struct cpumask *mask_val, diff --git a/drivers/irqchip/irq-gic.c b/drivers/irqchip/irq-gic.c index 4c7bae0ec8f9..40fa366baec3 100644 --- a/drivers/irqchip/irq-gic.c +++ b/drivers/irqchip/irq-gic.c @@ -867,13 +867,13 @@ static __init void gic_smp_init(void) "irqchip/arm/gic:starting", gic_starting_cpu, NULL); - base_sgi = __irq_domain_alloc_irqs(gic_data[0].domain, -1, 8, + base_sgi = __irq_domain_alloc_irqs(gic_data[0].domain, -1, 16, NUMA_NO_NODE, &sgi_fwspec, false, NULL); if (WARN_ON(base_sgi <= 0)) return; - set_smp_ipi_range(base_sgi, 8); + set_smp_ipi_range(base_sgi, 16); } #else #define gic_smp_init() do { } while(0) diff --git a/drivers/mailbox/Kconfig b/drivers/mailbox/Kconfig index 05d6fae800e3..0557e99d8dee 100644 --- a/drivers/mailbox/Kconfig +++ b/drivers/mailbox/Kconfig @@ -294,4 +294,13 @@ config QCOM_IPCC acts as an interrupt controller for receiving interrupts from clients. Say Y here if you want to build this driver. +config GENERIC_SOFTWARE_MAILBOX + tristate "Generic software Mailbox driver" + depends on OF + select GENERIC_IRQ_INJECTION + select GIC_GENTLE_CONFIG if ARM_GIC_V3 + help + This driver leverages unused interrupt line and shared memory to + implement a mailbox, which is used to send messages between different + OSes. endif diff --git a/drivers/mailbox/Makefile b/drivers/mailbox/Makefile index fc9376117111..38d6994df5e1 100644 --- a/drivers/mailbox/Makefile +++ b/drivers/mailbox/Makefile @@ -62,3 +62,5 @@ obj-$(CONFIG_SPRD_MBOX) += sprd-mailbox.o obj-$(CONFIG_QCOM_IPCC) += qcom-ipcc.o obj-$(CONFIG_APPLE_MAILBOX) += apple-mailbox.o + +obj-$(CONFIG_GENERIC_SOFTWARE_MAILBOX) += generic-software-mailbox.o diff --git a/drivers/mailbox/generic-software-mailbox.c b/drivers/mailbox/generic-software-mailbox.c new file mode 100644 index 000000000000..1938f45de6ec --- /dev/null +++ b/drivers/mailbox/generic-software-mailbox.c @@ -0,0 +1,322 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2022-2023 NXP + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * Generic software Registers: + * + * TX_STATUS[n]: TX channel n status + * RX_STATUS[n]: RX channel n status + * 0: indicates message in T/RX_CH[n] is invalid and channel ready. + * 1: indicates message in T/RX_CH[n] is valid and channel busy. + * 2: indicates message in T/RX_CH[n] has been received by the peer. + * RXDB_STATUS[n]: RX doorbell channel n status + * 0: indicates channel ready. + * 1: indicates channel busy. + * 2: indicates channel doorbell has been received by the peer. + * TX_CH[n]: Transmit data register for channel n + * RX_CH[n]: Receive data register for channel n + * + * To send a message: + * Update the data register TX_CH[n] with the message, then set the + * TX_STATUS[n] to 1, inject a interrupt to remote side; after the + * transmission done set the TX_STATUS[n] back to 0. + * + * When received a message: + * Get the received data from RX_CH[n] and then set the RX_STATUS[n] to + * 2 and inject a interrupt to notify the remote side transmission done. + */ + +#define MBOX_TX_CHAN (4) +#define MBOX_RX_CHAN (4) +#define MBOX_RXDB_CHAN (4) +#define RX_CHAN_SHFT (MBOX_TX_CHAN) +#define RXDB_CHAN_SHFT (MBOX_TX_CHAN + MBOX_RX_CHAN) +#define MBOX_CHAN_MAX (MBOX_TX_CHAN + MBOX_RX_CHAN + MBOX_RXDB_CHAN) + +struct sw_mbox_reg { + uint32_t tx_status[MBOX_TX_CHAN]; + uint32_t rx_status[MBOX_RX_CHAN]; + uint32_t rxdb_status[MBOX_RXDB_CHAN]; + uint32_t tx_ch[MBOX_TX_CHAN]; + uint32_t rx_ch[MBOX_RX_CHAN]; + uint32_t rxdb_ch[MBOX_RX_CHAN]; + uint32_t ch_ack_flags; /*from bit0 each bit for each channel(tx_ch, rx_ch, rxdb_ch), 1: ack, 0:noack */ +}; + +enum sw_mbox_channel_status { + S_READY, + S_BUSY, + S_DONE, +}; + +enum sw_mbox_type { + SW_TYPE_TX, /* Tx */ + SW_TYPE_RX, /* Rx */ + SW_TYPE_RXDB, /* Rx doorbell */ +}; + +struct sw_mbox_con_priv { + uint32_t idx; + enum sw_mbox_type type; + struct sw_mbox *priv; +}; + +struct sw_mbox { + struct device *dev; + struct sw_mbox_reg __iomem *base; + struct sw_mbox_con_priv cp[MBOX_CHAN_MAX]; + struct mbox_chan chan[MBOX_CHAN_MAX]; + struct mbox_controller controller; + int irq; + int remote_irq; +}; + +static int sw_mbox_send_data(struct mbox_chan *chan, void *msg) +{ + struct sw_mbox_con_priv *cp = chan->con_priv; + struct sw_mbox *mbox = cp->priv; + uint32_t idx = cp->idx; + uint32_t *data = msg; + int ret; + + if (cp->type != SW_TYPE_TX) { + dev_err(mbox->dev, "Channel type error\n"); + return -EINVAL; + } + + writel(*data, &mbox->base->tx_ch[idx]); + writel(S_BUSY, &mbox->base->tx_status[idx]); + ret = irq_set_irqchip_state(mbox->remote_irq, IRQCHIP_STATE_PENDING, + true); + if (ret) { + dev_err(mbox->dev, "Fail to inject IRQ\n"); + return ret; + } + + return 0; +} + +static irqreturn_t sw_mbox_interrupt(int irq, void *dev_id) +{ + struct sw_mbox *mbox = dev_id; + irqreturn_t ret = IRQ_NONE; + uint32_t rxdb_status; + uint32_t rx_status; + uint32_t tx_status; + uint32_t rx_ch; + int i; + + for (i = 0; i < MBOX_TX_CHAN; i++) { + tx_status = readl(&mbox->base->tx_status[i]); + if (tx_status == S_DONE) { + writel(S_READY, &mbox->base->tx_status[i]); + mbox_chan_txdone(&mbox->chan[i], 0); + ret = IRQ_HANDLED; + } + } + + for (i = 0; i < MBOX_RX_CHAN; i++) { + rx_status = readl(&mbox->base->rx_status[i]); + if (rx_status == S_BUSY) { + rx_ch = readl(&mbox->base->rx_ch[i]); + mbox_chan_received_data(&mbox->chan[i + RX_CHAN_SHFT], + (void *)&rx_ch); + if (mbox->base->ch_ack_flags & (1 << (i + RX_CHAN_SHFT))) { + /* Sender need ACK */ + writel(S_DONE, &mbox->base->rx_status[i]); + irq_set_irqchip_state(mbox->remote_irq, + IRQCHIP_STATE_PENDING, true); + } else { + /* set status to be ready if sender doesn't need ACK */ + writel(S_READY, &mbox->base->rx_status[i]); + } + ret = IRQ_HANDLED; + } + } + + for (i = 0; i < MBOX_RXDB_CHAN; i++) { + rxdb_status = readl(&mbox->base->rxdb_status[i]); + if (rxdb_status == S_BUSY) { + mbox_chan_received_data(&mbox->chan[i + RXDB_CHAN_SHFT], + NULL); + if (mbox->base->ch_ack_flags & (1 << (i + RXDB_CHAN_SHFT))) { + /* Sender need ACK */ + writel(S_DONE, &mbox->base->rxdb_status[i]); + irq_set_irqchip_state(mbox->remote_irq, + IRQCHIP_STATE_PENDING, true); + } else { + /* set status to be ready if sender doesn't need ACK */ + writel(S_READY, &mbox->base->rxdb_status[i]); + } + ret = IRQ_HANDLED; + } + } + + return ret; +} + +static int sw_mbox_startup(struct mbox_chan *chan) +{ + return 0; +} + +static void sw_mbox_shutdown(struct mbox_chan *chan) +{ +} + +static const struct mbox_chan_ops sw_mbox_ops = { + .send_data = sw_mbox_send_data, + .startup = sw_mbox_startup, + .shutdown = sw_mbox_shutdown, +}; + + +static struct mbox_chan *sw_mbox_xlate(struct mbox_controller *mbox, + const struct of_phandle_args *sp) +{ + struct mbox_chan *chan; + struct sw_mbox_con_priv *cp; + struct sw_mbox *sw_mb; + uint32_t type, idx, chan_idx, ack; + + if (sp->args_count != 3) { + dev_err(mbox->dev, "Invalid argument count %d\n", + sp->args_count); + return ERR_PTR(-EINVAL); + } + + type = sp->args[0]; + idx = sp->args[1]; + ack = sp->args[2]; + + switch (type) { + case SW_TYPE_TX: + chan_idx = idx; + break; + case SW_TYPE_RX: + chan_idx = RX_CHAN_SHFT + idx; + break; + case SW_TYPE_RXDB: + chan_idx = RXDB_CHAN_SHFT + idx; + break; + default: + dev_err(mbox->dev, "Invalid chan type: %d\n", type); + return ERR_PTR(-EINVAL); + } + + if (chan_idx >= MBOX_CHAN_MAX) { + dev_err(mbox->dev, "Not supported channel number: %d. (type: %d, idx: %d)\n", + chan_idx, type, idx); + return ERR_PTR(-EINVAL); + } + + chan = &mbox->chans[chan_idx]; + cp = chan->con_priv; + sw_mb = cp->priv; + if (ack) + sw_mb->base->ch_ack_flags |= 1 << chan_idx; + else + sw_mb->base->ch_ack_flags &= ~(1 << chan_idx); + + return chan; +} + +static const struct of_device_id sw_mbox_of_match[] = { + { .compatible = "fsl,generic-software-mbox", }, + {}, +}; +MODULE_DEVICE_TABLE(of, sw_mbox_of_match); + +static int sw_mailbox_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct sw_mbox *mbox; + int err, i; + + mbox = devm_kzalloc(dev, sizeof(*mbox), GFP_KERNEL); + if (!mbox) + return -ENOMEM; + mbox->dev = dev; + + mbox->irq = platform_get_irq_byname(pdev, "irq"); + if (mbox->irq <= 0) { + dev_err(dev, "Failed to get irq\n"); + return mbox->irq; + } + mbox->remote_irq = platform_get_irq_byname(pdev, "remote_irq"); + if (mbox->remote_irq <= 0) { + dev_err(dev, "Failed to get remote irq\n"); + return mbox->remote_irq; + } + + err = devm_request_irq(dev, mbox->irq, sw_mbox_interrupt, IRQF_SHARED, + pdev->name, mbox); + if (err) + return err; + + mbox->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(mbox->base)) + return PTR_ERR(mbox->base); + + memset_io(mbox->base->tx_status, 0, 4 * MBOX_TX_CHAN); + memset_io(mbox->base->rx_status, 0, 4 * MBOX_RX_CHAN); + memset_io(mbox->base->rxdb_status, 0, 4 * MBOX_RXDB_CHAN); + + mbox->controller.dev = dev; + mbox->controller.chans = mbox->chan; + mbox->controller.num_chans = MBOX_CHAN_MAX; + mbox->controller.ops = &sw_mbox_ops; + mbox->controller.of_xlate = sw_mbox_xlate; + mbox->controller.txdone_irq = true; + + for (i = 0; i < MBOX_CHAN_MAX; i++) { + mbox->chan[i].con_priv = &mbox->cp[i]; + mbox->cp[i].priv = mbox; + mbox->cp[i].idx = i; + } + + err = devm_mbox_controller_register(dev, &mbox->controller); + if (err) { + dev_err(dev, "Failed to register mailbox %d\n", err); + return err; + } + + platform_set_drvdata(pdev, mbox); + + return 0; +} + +static struct platform_driver sw_mbox_driver = { + .driver = { + .name = "generic-software-mailbox", + .of_match_table = sw_mbox_of_match, + }, + .probe = sw_mailbox_probe, +}; + +static int __init sw_mbox_init(void) +{ + return platform_driver_register(&sw_mbox_driver); +} + +static void __exit sw_mbox_exit(void) +{ + platform_driver_unregister(&sw_mbox_driver); +} + +module_init(sw_mbox_init); +module_exit(sw_mbox_exit); + +MODULE_DESCRIPTION("Generic Software mailbox driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/mailbox/mailbox.c b/drivers/mailbox/mailbox.c index 4229b9b5da98..8e2eba4a0a96 100644 --- a/drivers/mailbox/mailbox.c +++ b/drivers/mailbox/mailbox.c @@ -158,7 +158,7 @@ static enum hrtimer_restart txdone_hrtimer(struct hrtimer *hrtimer) void mbox_chan_received_data(struct mbox_chan *chan, void *mssg) { /* No buffering the received data */ - if (chan->cl->rx_callback) + if (chan->cl && chan->cl->rx_callback) chan->cl->rx_callback(chan->cl, mssg); } EXPORT_SYMBOL_GPL(mbox_chan_received_data); diff --git a/drivers/mxc/ipu3/ipu_common.c b/drivers/mxc/ipu3/ipu_common.c index 4ba7b6df7986..145fc0f1b4f9 100644 --- a/drivers/mxc/ipu3/ipu_common.c +++ b/drivers/mxc/ipu3/ipu_common.c @@ -2923,8 +2923,9 @@ static irqreturn_t ipu_sync_irq_handler(int irq, void *desc) uint32_t line, bit, int_stat, int_ctrl; irqreturn_t result = IRQ_NONE; const int int_reg[] = { 1, 2, 3, 4, 11, 12, 13, 14, 15, 0 }; + unsigned long flags; - spin_lock(&ipu->int_reg_spin_lock); + spin_lock_irqsave(&ipu->int_reg_spin_lock, flags); for (i = 0; int_reg[i] != 0; i++) { int_stat = ipu_cm_read(ipu, @@ -2949,7 +2950,7 @@ static irqreturn_t ipu_sync_irq_handler(int irq, void *desc) } } - spin_unlock(&ipu->int_reg_spin_lock); + spin_unlock_irqrestore(&ipu->int_reg_spin_lock, flags); return result; } @@ -2960,8 +2961,9 @@ static irqreturn_t ipu_err_irq_handler(int irq, void *desc) int i; uint32_t int_stat; const int err_reg[] = { 5, 6, 9, 10, 0 }; + unsigned long flags; - spin_lock(&ipu->int_reg_spin_lock); + spin_lock_irqsave(&ipu->int_reg_spin_lock, flags); for (i = 0; err_reg[i] != 0; i++) { int_stat = ipu_cm_read(ipu, @@ -2980,7 +2982,7 @@ static irqreturn_t ipu_err_irq_handler(int irq, void *desc) } } - spin_unlock(&ipu->int_reg_spin_lock); + spin_unlock_irqrestore(&ipu->int_reg_spin_lock, flags); return IRQ_HANDLED; } diff --git a/drivers/mxc/ipu3/ipu_device.c b/drivers/mxc/ipu3/ipu_device.c index f4417c7f5cda..2dcd6511bcb4 100644 --- a/drivers/mxc/ipu3/ipu_device.c +++ b/drivers/mxc/ipu3/ipu_device.c @@ -3249,7 +3249,7 @@ static int ipu_task_thread(void *argv) uint32_t size; unsigned long flags; unsigned int cpu; - struct cpumask cpu_mask; + struct cpumask cpu_mask = CPU_MASK_NONE; struct ipu_thread_data *data = (struct ipu_thread_data *)argv; thread_id++; diff --git a/drivers/net/dsa/Kconfig b/drivers/net/dsa/Kconfig index 07507b4820d7..b5c3cd5d0e65 100644 --- a/drivers/net/dsa/Kconfig +++ b/drivers/net/dsa/Kconfig @@ -62,6 +62,8 @@ source "drivers/net/dsa/xrs700x/Kconfig" source "drivers/net/dsa/realtek/Kconfig" +source "drivers/net/dsa/netc/Kconfig" + config NET_DSA_RZN1_A5PSW tristate "Renesas RZ/N1 A5PSW Ethernet switch support" depends on OF && ARCH_RZN1 diff --git a/drivers/net/dsa/Makefile b/drivers/net/dsa/Makefile index 16eb879e0cb4..47bec77c5839 100644 --- a/drivers/net/dsa/Makefile +++ b/drivers/net/dsa/Makefile @@ -24,3 +24,4 @@ obj-y += qca/ obj-y += realtek/ obj-y += sja1105/ obj-y += xrs700x/ +obj-y += netc/ \ No newline at end of file diff --git a/drivers/net/dsa/netc/Kconfig b/drivers/net/dsa/netc/Kconfig new file mode 100644 index 000000000000..40dbdd5e8a37 --- /dev/null +++ b/drivers/net/dsa/netc/Kconfig @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: GPL-2.0-only + +config NET_DSA_NETC + tristate "NXP NETC Ethernet switch family support" + depends on NET_DSA && SPI + select NET_DSA_TAG_NETC + help + This is the driver for the NXP ENTC Ethernet switch family. + These are managed over an SPI interface. Probing is handled + based on OF bindings and so is the linkage to PHYLINK. diff --git a/drivers/net/dsa/netc/Makefile b/drivers/net/dsa/netc/Makefile new file mode 100644 index 000000000000..72d6d8e2a52f --- /dev/null +++ b/drivers/net/dsa/netc/Makefile @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: GPL-2.0-only + +obj-$(CONFIG_NET_DSA_NETC) += netcdsa.o + +netcdsa-objs := \ + netc_spi.o \ + netc_config.o \ + netc_ethtool.o \ + netc_devlink.o \ + netc_ptp.o \ + netc_main.o + diff --git a/drivers/net/dsa/netc/netc.h b/drivers/net/dsa/netc/netc.h new file mode 100644 index 000000000000..cf9e27aff73f --- /dev/null +++ b/drivers/net/dsa/netc/netc.h @@ -0,0 +1,102 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2023 NXP + */ + +#ifndef _NETC_H +#define _NETC_H + +#include +#include +#include +#include +#include +#include "netc_config.h" + +struct netc_private; + +enum { + NETC_SPEED_AUTO, + NETC_SPEED_10MBPS, + NETC_SPEED_100MBPS, + NETC_SPEED_1000MBPS, + NETC_SPEED_2500MBPS, + NETC_SPEED_MAX, +}; + +enum netc_internal_phy_t { + NETC_NO_PHY = 0, +}; + +struct netc_info { + const char *name; + int device_id; + int num_ports; + enum dsa_tag_protocol tag_proto; + int ptp_ts_bits; + bool multiple_cascade_ports; + bool can_limit_mcast_flood; +}; + +struct netc_private { + const struct netc_info *info; + struct netc_config config; + int cpu_port; + phy_interface_t phy_mode[NETC_MAX_NUM_PORTS]; + bool fixed_link[NETC_MAX_NUM_PORTS]; + unsigned long ucast_egress_floods; + unsigned long bcast_egress_floods; + unsigned long hwts_tx_en; + + size_t max_xfer_len; + struct spi_device *spidev; + struct dsa_switch *ds; + u16 bridge_pvid[NETC_MAX_NUM_PORTS]; + u16 tag_8021q_pvid[NETC_MAX_NUM_PORTS]; + /* Serializes transmission of management frames so that + * the switch doesn't confuse them with one another. + */ + struct mutex mgmt_lock; + + struct devlink_region **regions; +}; + +int netc_vlan_filtering(struct dsa_switch *ds, int port, bool enabled, + struct netlink_ext_ack *extack); +void netc_frame_memory_partitioning(struct netc_private *priv); + +/* From netc_devlink.c */ +int netc_devlink_setup(struct dsa_switch *ds); +void netc_devlink_teardown(struct dsa_switch *ds); +int netc_devlink_info_get(struct dsa_switch *ds, + struct devlink_info_req *req, + struct netlink_ext_ack *extack); + +/* From netc_spi.c */ +int netc_xfer_cmd(const struct netc_private *priv, + enum netc_spi_rw_mode rw, enum netc_cmd cmd, + void *param, size_t param_len, + void *resp, size_t resp_len); +int netc_xfer_set_cmd(const struct netc_private *priv, + enum netc_cmd cmd, + void *param, size_t param_len); +int netc_xfer_get_cmd(const struct netc_private *priv, + enum netc_cmd cmd, uint32_t id, + void *resp, size_t resp_len); + +int netc_xfer_write_reg(const struct netc_private *priv, + uint32_t reg, uint32_t value); +int netc_xfer_read_reg(const struct netc_private *priv, + uint32_t reg, uint32_t *value); + +/* From netc_ethtool.c */ +void netc_get_ethtool_stats(struct dsa_switch *ds, int port, uint64_t *data); +void netc_get_strings(struct dsa_switch *ds, int port, + uint32_t stringset, uint8_t *data); +int netc_get_sset_count(struct dsa_switch *ds, int port, int sset); + +/* From netc_ptp.c */ +void netc_ptp_txtstamp_skb(struct dsa_switch *ds, int port, + struct sk_buff *skb); + +#endif /* _NETC_H */ diff --git a/drivers/net/dsa/netc/netc_config.c b/drivers/net/dsa/netc/netc_config.c new file mode 100644 index 000000000000..e4590c7684c9 --- /dev/null +++ b/drivers/net/dsa/netc/netc_config.c @@ -0,0 +1,322 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2023 NXP + */ + +#include +#include +#include +#include +#include "netc.h" + +int netc_get_devinfo(struct netc_private *priv, struct netc_config *config) +{ + struct netc_cmd_sysinfo info; + int rc; + + rc = netc_xfer_get_cmd(priv, NETC_CMD_SYS_INFO_GET, 0, + &info, sizeof(info)); + if (rc < 0) + return rc; + + config->device_id = info.device_id; + config->vendor_id = info.vendor_id; + config->version_major = info.version_major; + config->version_minor = info.version_minor; + config->version_revision = info.version_revision; + config->cpu_port_mode = info.cpu_port; + + return 0; +} + +int netc_port_mtu_set(struct netc_private *priv, int port, int mtu) +{ + struct netc_cmd_port_mtu mtu_cmd = {0}; + + mtu_cmd.port = (uint8_t)port; + mtu_cmd.mtu = (uint16_t)mtu; + + return netc_xfer_set_cmd(priv, NETC_CMD_PORT_MTU_SET, + &mtu_cmd, sizeof(mtu_cmd)); +} + +int netc_port_mtu_get(struct netc_private *priv, int port, int *mtu) +{ + int rc; + struct netc_cmd_port_mtu mtu_resp = {0}; + + rc = netc_xfer_get_cmd(priv, NETC_CMD_PORT_MTU_GET, port, + &mtu_resp, sizeof(mtu_resp)); + + if (rc != 0) + return rc; + + *mtu = mtu_resp.mtu; + + return 0; +} + +/* Set link speed in the MAC configuration for a specific port. */ +int netc_port_phylink_mode_set(struct netc_private *priv, + struct netc_mac_config *mac) +{ + struct device *dev = priv->ds->dev; + struct netc_cmd_port_phylink_mode phylink_mode = {0}; + int rc; + + phylink_mode.port = mac->port; + phylink_mode.duplex = mac->duplex; + phylink_mode.speed = mac->speed; + + rc = netc_xfer_set_cmd(priv, NETC_CMD_PORT_PHYLINK_MODE_SET, + &phylink_mode, sizeof(phylink_mode)); + if (rc < 0) { + dev_err(dev, "Failed to write phylink_mode: %d\n", rc); + return rc; + } + + return 0; +} + +/* Get link speed in the MAC configuration for a specific port. */ +int netc_port_phylink_status_get(struct netc_private *priv, + struct netc_mac_config *mac) +{ + struct device *dev = priv->ds->dev; + struct netc_cmd_port_phylink_status phylink_status = {0}; + int rc; + + rc = netc_xfer_get_cmd(priv, NETC_CMD_PORT_PHYLINK_STATUS_GET, + mac->port, + &phylink_status, sizeof(phylink_status)); + if (rc < 0) { + dev_err(dev, "Failed to get phylink status: %d\n", rc); + return rc; + } + + mac->link = phylink_status.link; + mac->speed = phylink_status.speed; + mac->duplex = phylink_status.duplex; + + return 0; +} + +int netc_port_pvid_set(struct netc_private *priv, int port, uint16_t pvid) +{ + int rc = 0; + struct netc_cmd_port_pvid cmd_pvid = {0}; + + cmd_pvid.port = (uint8_t)port; + cmd_pvid.pvid = pvid; + + rc = netc_xfer_set_cmd(priv, NETC_CMD_PORT_PVID_SET, + &cmd_pvid, sizeof(cmd_pvid)); + + return rc; +} + +int netc_port_link_set(struct netc_private *priv, int port, bool up) +{ + int rc = 0; + struct netc_cmd_port_link egress = {0}; + + egress.port = (uint8_t)port; + egress.link = up; + + rc = netc_xfer_set_cmd(priv, NETC_CMD_PORT_LINK_SET, + &egress, sizeof(egress)); + + return rc; +} + +int netc_port_dropuntag_set(struct netc_private *priv, int port, bool drop) +{ + int rc = 0; + struct netc_cmd_port_dropuntag dropuntag = {0}; + + dropuntag.port = (uint8_t)port; + dropuntag.drop = (uint16_t)drop; + + rc = netc_xfer_set_cmd(priv, NETC_CMD_PORT_DROPUNTAG_SET, + &dropuntag, sizeof(dropuntag)); + + return rc; +} + +int netc_port_dsa_add(struct netc_private *priv, int cpu_port, + int slave_port, const unsigned char *mac_addr) +{ + int rc = 0; + struct netc_cmd_port_dsa_add dsa_add = {0}; + + dsa_add.cpu_port = (uint8_t)cpu_port; + dsa_add.slave_port = (uint8_t)slave_port; + ether_addr_copy(dsa_add.mac_addr, mac_addr); + + rc = netc_xfer_set_cmd(priv, NETC_CMD_PORT_DSA_ADD, + &dsa_add, sizeof(dsa_add)); + + return rc; +} + +int netc_port_dsa_del(struct netc_private *priv, int slave_port) +{ + int rc = 0; + struct netc_cmd_port_dsa_del dsa_del = {0}; + + dsa_del.slave_port = (uint8_t)slave_port; + + rc = netc_xfer_set_cmd(priv, NETC_CMD_PORT_DSA_DEL, + &dsa_del, sizeof(dsa_del)); + + return rc; +} + +int netc_vlan_entry_add(struct netc_private *priv, + uint16_t vid, int port, bool untagged) +{ + struct device *dev = priv->ds->dev; + struct netc_cmd_vlan cmd_vlan = {0}; + int rc; + + cmd_vlan.vid = vid; + cmd_vlan.port = (uint8_t)port; + cmd_vlan.untagged = untagged; + + rc = netc_xfer_set_cmd(priv, NETC_CMD_VLAN_ADD, + &cmd_vlan, sizeof(cmd_vlan)); + if (rc < 0) { + dev_err(dev, "Failed to add vlan entry: %d\n", rc); + return rc; + } + + return 0; +} + +int netc_vlan_entry_del(struct netc_private *priv, uint16_t vid, int port) +{ + struct device *dev = priv->ds->dev; + struct netc_cmd_vlan cmd_vlan = {0}; + int rc; + + cmd_vlan.vid = vid; + cmd_vlan.port = (uint8_t)port; + + rc = netc_xfer_set_cmd(priv, NETC_CMD_VLAN_DEL, + &cmd_vlan, sizeof(cmd_vlan)); + if (rc < 0) { + dev_err(dev, "Failed to add vlan entry: %d\n", rc); + return rc; + } + + return 0; +} + +int netc_vlan_entry_read(struct netc_private *priv, + struct netc_vlan_entry *vlan, + uint32_t entry_id, uint32_t *next_id) +{ + struct device *dev = priv->ds->dev; + struct netc_cmd_vlan_dump vlan_dump = {0}; + int rc; + + rc = netc_xfer_get_cmd(priv, NETC_CMD_VLAN_DUMP, entry_id, + &vlan_dump, sizeof(vlan_dump)); + if (rc < 0) { + dev_err(dev, "Failed to read vlan entry 0x%08x: %d\n", + entry_id, rc); + return rc; + } + + vlan->entry_id = entry_id; + vlan->vid = vlan_dump.vid; + vlan->port_map = vlan_dump.port_map; + *next_id = vlan_dump.resume_entry_id; + + return 0; +} + +int netc_fdb_entry_add(struct netc_private *priv, + const unsigned char *mac_addr, + uint16_t vid, int port) +{ + struct device *dev = priv->ds->dev; + struct netc_cmd_fdb fdb_add = {0}; + int rc; + + ether_addr_copy(fdb_add.mac_addr, mac_addr); + fdb_add.vid = vid; + fdb_add.port = (uint8_t)port; + + rc = netc_xfer_set_cmd(priv, NETC_CMD_FDB_ADD, + &fdb_add, sizeof(fdb_add)); + if (rc < 0) { + dev_err(dev, "Failed to add fdb: %d\n", rc); + return rc; + } + + return 0; +} + +int netc_fdb_entry_del(struct netc_private *priv, + const unsigned char *mac_addr, + uint16_t vid) +{ + struct device *dev = priv->ds->dev; + struct netc_cmd_fdb_del fdb_del = {0}; + int rc; + + ether_addr_copy(fdb_del.mac_addr, mac_addr); + fdb_del.vid = vid; + + rc = netc_xfer_set_cmd(priv, NETC_CMD_FDB_DEL, + &fdb_del, sizeof(fdb_del)); + if (rc < 0) { + dev_err(dev, "Failed to delete fdb: %d\n", rc); + return rc; + } + + return 0; +} + +int netc_fdb_entry_get(struct netc_private *priv, struct netc_fdb_entry *fdb, + uint32_t entry_id, uint32_t *next_id) +{ + struct device *dev = priv->ds->dev; + struct netc_cmd_fdb_dump fdb_dump = {0}; + int rc; + + rc = netc_xfer_get_cmd(priv, NETC_CMD_FDB_DUMP, entry_id, + &fdb_dump, sizeof(fdb_dump)); + if (rc < 0) { + dev_err(dev, "Failed to get fdb entry: %d\n", rc); + return rc; + } + + *next_id = fdb_dump.resume_entry_id; + + ether_addr_copy(fdb->mac_addr, fdb_dump.mac_addr); + fdb->vid = fdb_dump.vid; + fdb->port_map = fdb_dump.port_map; + fdb->dynamic = fdb_dump.dynamic; + + return 0; +} + +int netc_config_setup(struct netc_config *config) +{ + if (config->vlan_max_count) { + config->vlan = kcalloc(config->vlan_max_count, + sizeof(*config->vlan), + GFP_KERNEL); + if (!config->vlan) + return -ENOMEM; + } + + return 0; +} + +void netc_config_free(struct netc_config *config) +{ + kfree(config->vlan); +} diff --git a/drivers/net/dsa/netc/netc_config.h b/drivers/net/dsa/netc/netc_config.h new file mode 100644 index 000000000000..1b181e7ef56b --- /dev/null +++ b/drivers/net/dsa/netc/netc_config.h @@ -0,0 +1,288 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2023 NXP + */ + +#ifndef _NETC_CONFIG_H +#define _NETC_CONFIG_H + +#include +#include + +#define NETC_RT1180_DEVICE_ID 0xe001 +#define NETC_NUM_PORTS 5 +#define NETC_MAX_NUM_PORTS NETC_NUM_PORTS +#define NETC_NUM_TC 8 + +#define NETC_ETHTOOL_STATS_NUM_MAX 120 + +#define NETC_SPI_WORD_BITS 8 +#define NETC_SPI_MSG_WORD_BYTES 4 +#define NETC_SPI_MSG_HEADER_SIZE 16 +#define NETC_SPI_MSG_PARAM_SIZE 12 +#define NETC_SPI_MSG_MAXLEN 4096 +#define NETC_SPI_MSG_RESPONSE_TIME 1000 /* us */ + +#define NETC_CMD_DIR_SHIFT 31 +#define NETC_CMD_LEN_SHIFT 16 + +enum netc_spi_rw_mode { + SPI_READ = 0, + SPI_WRITE = 1, +}; + +struct netc_cmd_hdr { + uint32_t cmd; + uint8_t param[NETC_SPI_MSG_PARAM_SIZE]; +}; + +/* Command */ +enum netc_cmd { + /* port related command */ + NETC_CMD_SYS_INFO_GET = 0x1, + NETC_CMD_PORT_DSA_ADD, + NETC_CMD_PORT_DSA_DEL, + NETC_CMD_PORT_MTU_SET, + NETC_CMD_PORT_MTU_GET, + NETC_CMD_PORT_PHYLINK_MODE_SET, + NETC_CMD_PORT_PHYLINK_STATUS_GET, + NETC_CMD_PORT_ETHTOOL_STATS_GET, + NETC_CMD_PORT_PVID_SET, + NETC_CMD_PORT_LINK_SET, + NETC_CMD_PORT_DROPUNTAG_SET, + + NETC_CMD_FDB_ADD = 0x1000, + NETC_CMD_FDB_DEL, + NETC_CMD_FDB_DUMP, + NETC_CMD_VLAN_ADD, + NETC_CMD_VLAN_DEL, + NETC_CMD_VLAN_DUMP, + NETC_CMD_FORWARD_MASK_SET, + + NETC_CMD_PTP_SYNC_SET = 0x2000, + + NETC_CMD_QBV_SET = 0x3000, + NETC_CMD_QBV_GET, + NETC_CMD_QBU_SET, + NETC_CMD_QBU_GET, + NETC_CMD_QCI_SET, + NETC_CMD_QCI_GET, + NETC_CMD_8021CB_SET, + NETC_CMD_8021CB_GET, + + NETC_CMD_REG_SET = 0x4000, + NETC_CMD_REG_GET, + NETC_CMD_MAX_NUM, +}; + +struct netc_cmd_sysinfo { + uint16_t device_id; + uint16_t vendor_id; + uint8_t version_major; + uint8_t version_minor; + uint8_t version_revision; + uint8_t cpu_port; +}; + +/* command data for NETC_CMD_PORT_DSA_ADD */ +struct netc_cmd_port_dsa_add { + uint8_t cpu_port; /* switch port 0, 1, 2 or 3 */ + uint8_t slave_port; /* switch port 0, 1, 2 or 3 */ + uint8_t mac_addr[ETH_ALEN]; /* MAC address of master interface */ +}; + +/* command data for NETC_CMD_PORT_DSA_DEL */ +struct netc_cmd_port_dsa_del { + uint8_t slave_port; /* switch port 0, 1, 2 or 3 */ + uint8_t reserved[3]; +}; + +/* command data for NETC_CMD_PORT_MTU_SET */ +struct netc_cmd_port_mtu { + uint8_t port; /* switch port 0, 1, 2 or 3 */ + uint8_t reserved; + uint16_t mtu; +}; + +/* command data for NETC_CMD_PORT_PHYLINK_MODE_SET */ +struct netc_cmd_port_phylink_mode { + uint8_t port; /* switch port 0, 1, 2 or 3 */ + bool duplex; /* 0: half duplex; 1: full duplex */ + uint16_t speed; /* 10: 10Mbps ; 100: 100Mbps ; 1000: 1000Mbps */ +}; + +/* command data for NETC_CMD_PORT_PVID_SET */ +struct netc_cmd_port_pvid { + uint8_t port; /* switch port 0, 1, 2 or 3 */ + uint8_t reserved; + uint16_t pvid; +}; + +/* command data for netc_cmd_port_link */ +struct netc_cmd_port_link { + uint8_t port; /* switch port 0, 1, 2 or 3 */ + bool link; /* 0: down; 1: up */ + uint8_t reserved[2]; +}; + +/* command data for netc_cmd_port_dropuntag */ +struct netc_cmd_port_dropuntag { + uint8_t port; /* switch port 0, 1, 2 or 3 */ + uint8_t reserved; + uint16_t drop; +}; + +/* command data for NETC_CMD_FDB_ADD */ +struct netc_cmd_fdb { + uint8_t mac_addr[ETH_ALEN]; + uint16_t vid; + uint8_t port; /* switch port 0, 1, 2 or 3 */ + uint8_t reserved[3]; +}; + +/* command data for NETC_CMD_FDB_DEL */ +struct netc_cmd_fdb_del { + uint8_t mac_addr[ETH_ALEN]; + uint16_t vid; +}; + +/* command data for NETC_CMD_VLAN_ADD */ +struct netc_cmd_vlan { + uint16_t vid; + uint8_t port; /* switch port 0, 1, 2 or 3 */ + bool untagged; +}; + +/* data returned for NETC_CMD_PORT_PHYLINK_STATUS_GET */ +struct netc_cmd_port_phylink_status { + uint8_t port; /* switch port 0, 1, 2 or 3 */ + bool link; + uint16_t speed; + bool duplex; /* 0: down; 1: up */ + uint8_t reserved[3]; +}; + +/* command param */ +struct netc_cmd_read_param { + uint32_t id; +}; + +/* command data for NETC_CMD_REG_SET */ +struct netc_cmd_reg_cmd { + uint32_t reg; + uint32_t value; +}; + +/* data returned for NETC_CMD_FDB_DUMP */ +struct netc_cmd_fdb_dump { + uint8_t mac_addr[ETH_ALEN]; + uint16_t vid; + /* bit 0: switch port 0 etc. */ + uint32_t port_map; + bool dynamic; + uint8_t reserved[3]; + /* non-zero means there are remaining entries, 0 means no more entries */ + uint32_t resume_entry_id; +}; + +/* data returned for NETC_CMD_VLAN_DUMP */ +struct netc_cmd_vlan_dump { + uint16_t vid; + bool untagged; + uint8_t reserved; + /* bit 0: switch port 0 etc. */ + uint32_t port_map; + /* non-zero means there are remaining entries, 0 means no more entries */ + uint32_t resume_entry_id; +}; + +struct netc_cmd_port_ethtool_stats { + uint64_t values[NETC_ETHTOOL_STATS_NUM_MAX]; +}; + +struct netc_mac_config { + uint8_t port; + uint16_t speed; + uint16_t vlanid; + bool link; + bool egress; + bool ingress; + bool duplex; + bool drptag; + bool drpuntag; + bool retag; +}; + +struct netc_fdb_entry { + uint8_t mac_addr[ETH_ALEN]; + uint16_t vid; + uint32_t port_map; /* bit 0: switch port 0 etc. */ + bool dynamic; +}; + +struct netc_vlan_entry { + uint16_t vid; + uint16_t port; + uint32_t port_map; + uint32_t tag_ports; + uint32_t entry_id; +}; + +struct netc_config { + uint16_t device_id; + uint16_t vendor_id; + uint8_t version_major; + uint8_t version_minor; + uint8_t version_revision; + uint8_t cpu_port_mode; + uint16_t tpid; + uint16_t tpid2; + struct netc_mac_config mac[NETC_MAX_NUM_PORTS]; + int cpu_port; + int vlan_count; + int vlan_max_count; + struct netc_vlan_entry *vlan; +}; + +struct netc_private; + +int netc_get_devinfo(struct netc_private *priv, struct netc_config *config); + +int netc_port_phylink_mode_set(struct netc_private *priv, + struct netc_mac_config *mac); +int netc_port_phylink_stats_get(struct netc_private *priv, + struct netc_mac_config *mac); +int netc_port_pvid_set(struct netc_private *priv, int port, uint16_t pvid); +int netc_port_link_set(struct netc_private *priv, int port, bool up); +int netc_port_dropuntag_set(struct netc_private *priv, int port, bool drop); + +int netc_port_mtu_set(struct netc_private *priv, int port, int mtu); +int netc_port_mtu_get(struct netc_private *priv, int port, int *mtu); + +int netc_port_pvid_set(struct netc_private *priv, int port, uint16_t pvid); + +int netc_port_dsa_add(struct netc_private *priv, int cpu_port, + int slave_port, const unsigned char *mac_addr); +int netc_port_dsa_del(struct netc_private *priv, int slave_port); + +int netc_fdb_entry_add(struct netc_private *priv, + const unsigned char *mac_addr, + uint16_t vid, int port); +int netc_fdb_entry_del(struct netc_private *priv, + const unsigned char *mac_addr, + uint16_t vid); +int netc_fdb_entry_get(struct netc_private *priv, + struct netc_fdb_entry *fdb, + uint32_t entry_id, uint32_t *next_id); + +int netc_vlan_entry_add(struct netc_private *priv, + uint16_t vid, int port, bool untagged); +int netc_vlan_entry_del(struct netc_private *priv, uint16_t vid, int port); +int netc_vlan_entry_get(struct netc_private *priv, + struct netc_vlan_entry *vlan, + uint32_t entry_id, uint32_t *next_id); + +int netc_config_setup(struct netc_config *config); +void netc_config_free(struct netc_config *config); + +#endif /* _NETC_CONFIG_H */ diff --git a/drivers/net/dsa/netc/netc_devlink.c b/drivers/net/dsa/netc/netc_devlink.c new file mode 100644 index 000000000000..bcf0de99b0cd --- /dev/null +++ b/drivers/net/dsa/netc/netc_devlink.c @@ -0,0 +1,111 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2023 NXP + */ + +#include "netc.h" + +static size_t netc_config_get_size(struct netc_private *priv) +{ + return sizeof(struct netc_config); +} + +static int +netc_region_config_snapshot(struct devlink *dl, + const struct devlink_region_ops *ops, + struct netlink_ext_ack *extack, + u8 **data) +{ + struct dsa_switch *ds = dsa_devlink_to_ds(dl); + struct netc_private *priv = ds->priv; + size_t len; + + len = netc_config_get_size(priv); + *data = kcalloc(len, sizeof(u8), GFP_KERNEL); + if (!*data) + return -ENOMEM; + + return netc_xfer_get_cmd(priv, NETC_CMD_SYS_INFO_GET, 0, *data, len); +} + +static struct devlink_region_ops netc_region_config_ops = { + .name = "config", + .snapshot = netc_region_config_snapshot, + .destructor = kfree, +}; + +enum netc_region_id { + NETC_REGION_CONFIG = 0, +}; + +struct netc_region { + const struct devlink_region_ops *ops; + size_t (*get_size)(struct netc_private *priv); +}; + +static struct netc_region netc_regions[] = { + [NETC_REGION_CONFIG] = { + .ops = &netc_region_config_ops, + .get_size = netc_config_get_size, + }, +}; + +int netc_devlink_info_get(struct dsa_switch *ds, + struct devlink_info_req *req, + struct netlink_ext_ack *extack) +{ + struct netc_private *priv = ds->priv; + int rc; + + rc = devlink_info_driver_name_put(req, "netc"); + if (rc) + return rc; + + rc = devlink_info_version_fixed_put(req, + DEVLINK_INFO_VERSION_GENERIC_ASIC_ID, + priv->info->name); + return rc; +} + +int netc_devlink_setup(struct dsa_switch *ds) +{ + int i, num_regions = ARRAY_SIZE(netc_regions); + struct netc_private *priv = ds->priv; + const struct devlink_region_ops *ops; + struct devlink_region *region; + u64 size; + + priv->regions = kcalloc(num_regions, sizeof(struct devlink_region *), + GFP_KERNEL); + if (!priv->regions) + return -ENOMEM; + + for (i = 0; i < num_regions; i++) { + size = netc_regions[i].get_size(priv); + ops = netc_regions[i].ops; + + region = dsa_devlink_region_create(ds, ops, 1, size); + if (IS_ERR(region)) { + while (--i >= 0) + dsa_devlink_region_destroy(priv->regions[i]); + + kfree(priv->regions); + return PTR_ERR(region); + } + + priv->regions[i] = region; + } + + return 0; +} + +void netc_devlink_teardown(struct dsa_switch *ds) +{ + int i, num_regions = ARRAY_SIZE(netc_regions); + struct netc_private *priv = ds->priv; + + for (i = 0; i < num_regions; i++) + dsa_devlink_region_destroy(priv->regions[i]); + + kfree(priv->regions); +} diff --git a/drivers/net/dsa/netc/netc_ethtool.c b/drivers/net/dsa/netc/netc_ethtool.c new file mode 100644 index 000000000000..7e6ab46ffee6 --- /dev/null +++ b/drivers/net/dsa/netc/netc_ethtool.c @@ -0,0 +1,344 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2023 NXP + */ + +#include "netc.h" + +enum netc_stat_index { + /* RX stats */ + NETC_STAT_RX_BYTES, + NETC_STAT_RX_VALID_BYTES, + NETC_STAT_RX_PAUSE_FRAMES, + NETC_STAT_RX_VALID_FRAMES, + NETC_STAT_RX_VLAN_FRAMES, + NETC_STAT_RX_UC_FRAMES, + NETC_STAT_RX_MC_FRAMES, + NETC_STAT_RX_BC_FRAMES, + NETC_STAT_RX_FRAMES, + NETC_STAT_RX_MIN_FRAMES, + NETC_STAT_RX_64_FRAMES, + NETC_STAT_RX_65_127_FRAMES, + NETC_STAT_RX_128_255_FRAMES, + NETC_STAT_RX_256_511_FRAMES, + NETC_STAT_RX_512_1023_FRAMES, + NETC_STAT_RX_1024_1522_FRAMES, + NETC_STAT_RX_1523_MAX_FRAMES, + NETC_STAT_RX_CONTROL_FRAMES, + + /* TX stats */ + NETC_STAT_TX_BYTES, + NETC_STAT_TX_VALID_BYTES, + NETC_STAT_TX_PAUSE_FRAMES, + NETC_STAT_TX_VALID_FRAMES, + NETC_STAT_TX_VLAN_FRAMES, + NETC_STAT_TX_UC_FRAMES, + NETC_STAT_TX_MC_FRAMES, + NETC_STAT_TX_BC_FRAMES, + NETC_STAT_TX_FRAMES, + NETC_STAT_TX_MIN_FRAMES, + NETC_STAT_TX_64_FRAMES, + NETC_STAT_TX_65_127_FRAMES, + NETC_STAT_TX_128_255_FRAMES, + NETC_STAT_TX_256_511_FRAMES, + NETC_STAT_TX_512_1023_FRAMES, + NETC_STAT_TX_1024_1522_FRAMES, + NETC_STAT_TX_1523_MAX_FRAMES, + NETC_STAT_TX_CONTROL_FRAMES, + + NETC_STAT_RX_VALID_REASSEMBLED_FRAMES, + NETC_STAT_RX_ADDITIONAL_MPACKETS, + NETC_STAT_RX_ERROR_FRAME_REASSEMBLY, + NETC_STAT_RX_ERROR_FRAME_SMD, + NETC_STAT_TX_ADDITIONAL_MPACKETS, + NETC_STAT_TX_HOLD_TRANSITIONS, + + /* Error stats */ + NETC_STAT_RX_ERROR, + NETC_STAT_RX_ERROR_UNDERSIZE, + NETC_STAT_RX_ERROR_OVERSIZE, + NETC_STAT_RX_ERROR_FCS, + NETC_STAT_RX_ERROR_FRAGMENT, + NETC_STAT_RX_ERROR_JABBER, + NETC_STAT_RX_ERROR_DISCARD, + NETC_STAT_RX_ERROR_NO_TRUNCATED, + NETC_STAT_TX_ERROR_FCS, + NETC_STAT_TX_ERROR_UNDERSIZE, + + /* Discard stats */ + NETC_STAT_RX_DISCARD_COUNT, + NETC_STAT_RX_DISCARD_REASON0, + NETC_STAT_RX_DISCARD_TABLE_ID, + NETC_STAT_RX_DISCARD_ENTRY_ID, + NETC_STAT_TX_DISCARD_COUNT, + NETC_STAT_TX_DISCARD_REASON0, + NETC_STAT_TX_DISCARD_TABLE_ID, + NETC_STAT_TX_DISCARD_ENTRY_ID, + NETC_STAT_BRIDGE_DISCARD_COUNT, + NETC_STAT_BRIDGE_DISCARD_REASON0, + NETC_STAT_BRIDGE_DISCARD_TABLE_ID, + NETC_STAT_BRIDGE_DISCARD_ENTRY_ID, + + /* Q0 stats */ + NETC_STAT_Q0_REJECTED_BYTES, + NETC_STAT_Q0_REJECTED_FRAMES, + NETC_STAT_Q0_DEQUEUE_BYTES, + NETC_STAT_Q0_DEQUEUE_FRAMES, + NETC_STAT_Q0_DROPPED_BYTES, + NETC_STAT_Q0_DROPPED_FRAMES, + NETC_STAT_Q0_FRAMES, + + /* Q1 stats */ + NETC_STAT_Q1_REJECTED_BYTES, + NETC_STAT_Q1_REJECTED_FRAMES, + NETC_STAT_Q1_DEQUEUE_BYTES, + NETC_STAT_Q1_DEQUEUE_FRAMES, + NETC_STAT_Q1_DROPPED_BYTES, + NETC_STAT_Q1_DROPPED_FRAMES, + NETC_STAT_Q1_FRAMES, + + /* Q2 stats */ + NETC_STAT_Q2_REJECTED_BYTES, + NETC_STAT_Q2_REJECTED_FRAMES, + NETC_STAT_Q2_DEQUEUE_BYTES, + NETC_STAT_Q2_DEQUEUE_FRAMES, + NETC_STAT_Q2_DROPPED_BYTES, + NETC_STAT_Q2_DROPPED_FRAMES, + NETC_STAT_Q2_FRAMES, + + /* Q3 stats */ + NETC_STAT_Q3_REJECTED_BYTES, + NETC_STAT_Q3_REJECTED_FRAMES, + NETC_STAT_Q3_DEQUEUE_BYTES, + NETC_STAT_Q3_DEQUEUE_FRAMES, + NETC_STAT_Q3_DROPPED_BYTES, + NETC_STAT_Q3_DROPPED_FRAMES, + NETC_STAT_Q3_FRAMES, + + /* Q4 stats */ + NETC_STAT_Q4_REJECTED_BYTES, + NETC_STAT_Q4_REJECTED_FRAMES, + NETC_STAT_Q4_DEQUEUE_BYTES, + NETC_STAT_Q4_DEQUEUE_FRAMES, + NETC_STAT_Q4_DROPPED_BYTES, + NETC_STAT_Q4_DROPPED_FRAMES, + NETC_STAT_Q4_FRAMES, + + /* Q5 stats */ + NETC_STAT_Q5_REJECTED_BYTES, + NETC_STAT_Q5_REJECTED_FRAMES, + NETC_STAT_Q5_DEQUEUE_BYTES, + NETC_STAT_Q5_DEQUEUE_FRAMES, + NETC_STAT_Q5_DROPPED_BYTES, + NETC_STAT_Q5_DROPPED_FRAMES, + NETC_STAT_Q5_FRAMES, + + /* Q6 stats */ + NETC_STAT_Q6_REJECTED_BYTES, + NETC_STAT_Q6_REJECTED_FRAMES, + NETC_STAT_Q6_DEQUEUE_BYTES, + NETC_STAT_Q6_DEQUEUE_FRAMES, + NETC_STAT_Q6_DROPPED_BYTES, + NETC_STAT_Q6_DROPPED_FRAMES, + NETC_STAT_Q6_FRAMES, + + /* Q7 stats */ + NETC_STAT_Q7_REJECTED_BYTES, + NETC_STAT_Q7_REJECTED_FRAMES, + NETC_STAT_Q7_DEQUEUE_BYTES, + NETC_STAT_Q7_DEQUEUE_FRAMES, + NETC_STAT_Q7_DROPPED_BYTES, + NETC_STAT_Q7_DROPPED_FRAMES, + NETC_STAT_Q7_FRAMES, + NETC_STAT_NUM, +}; + +char netc_stat_name[][ETH_GSTRING_LEN] = { + /* RX stats */ + [NETC_STAT_RX_BYTES] = "in-bytes", + [NETC_STAT_RX_VALID_BYTES] = "in-valid-bytes", + [NETC_STAT_RX_PAUSE_FRAMES] = "in-pause-frames", + [NETC_STAT_RX_VALID_FRAMES] = "in-valid-frames", + [NETC_STAT_RX_VLAN_FRAMES] = "in-vlan-frames", + [NETC_STAT_RX_UC_FRAMES] = "in-uc-frames", + [NETC_STAT_RX_MC_FRAMES] = "in-mc-frames", + [NETC_STAT_RX_BC_FRAMES] = "in-bc-frames", + [NETC_STAT_RX_FRAMES] = "in-frames", + [NETC_STAT_RX_MIN_FRAMES] = "in-min-frames", + [NETC_STAT_RX_64_FRAMES] = "in-64-frames", + [NETC_STAT_RX_65_127_FRAMES] = "in-65-127-frames", + [NETC_STAT_RX_128_255_FRAMES] = "in-128-255-frames", + [NETC_STAT_RX_256_511_FRAMES] = "in-256-511-frames", + [NETC_STAT_RX_512_1023_FRAMES] = "in-512-1023-frames", + [NETC_STAT_RX_1024_1522_FRAMES] = "in-1024-1522-frames", + [NETC_STAT_RX_1523_MAX_FRAMES] = "in-1523-max-frames", + [NETC_STAT_RX_CONTROL_FRAMES] = "in-control-frames", + + /* TX stats */ + [NETC_STAT_TX_BYTES] = "out-bytes", + [NETC_STAT_TX_VALID_BYTES] = "out-valid-bytes", + [NETC_STAT_TX_PAUSE_FRAMES] = "out-pause-frames", + [NETC_STAT_TX_VALID_FRAMES] = "out-valid-frames", + [NETC_STAT_TX_VLAN_FRAMES] = "out-vlan-frames", + [NETC_STAT_TX_UC_FRAMES] = "out-uc-frames", + [NETC_STAT_TX_MC_FRAMES] = "out-mc-frames", + [NETC_STAT_TX_BC_FRAMES] = "out-bc-frames", + [NETC_STAT_TX_FRAMES] = "out-frames", + [NETC_STAT_TX_MIN_FRAMES] = "out-min-frames", + [NETC_STAT_TX_64_FRAMES] = "out-64-frames", + [NETC_STAT_TX_65_127_FRAMES] = "out-65-127-frames", + [NETC_STAT_TX_128_255_FRAMES] = "out-128-255-frames", + [NETC_STAT_TX_256_511_FRAMES] = "out-256-511-frames", + [NETC_STAT_TX_512_1023_FRAMES] = "out-512-1023-frames", + [NETC_STAT_TX_1024_1522_FRAMES] = "out-1024-1522-frames", + [NETC_STAT_TX_1523_MAX_FRAMES] = "out-1523-max-frames", + [NETC_STAT_TX_CONTROL_FRAMES] = "out-control-frames", + + [NETC_STAT_RX_VALID_REASSEMBLED_FRAMES] = "in-valid-reassembled-frames", + [NETC_STAT_RX_ADDITIONAL_MPACKETS] = "in-additional-mPackets", + [NETC_STAT_RX_ERROR_FRAME_REASSEMBLY] = "in-error-frame-reassembly", + [NETC_STAT_RX_ERROR_FRAME_SMD] = "in-error-frame-smd", + [NETC_STAT_TX_ADDITIONAL_MPACKETS] = "out-additional-mPackets", + [NETC_STAT_TX_HOLD_TRANSITIONS] = "out-hold-transitions", + + /* Error stats */ + [NETC_STAT_RX_ERROR] = "in-error", + [NETC_STAT_RX_ERROR_UNDERSIZE] = "in-error-undersize", + [NETC_STAT_RX_ERROR_OVERSIZE] = "in-error-oversize", + [NETC_STAT_RX_ERROR_FCS] = "in-error-fcs", + [NETC_STAT_RX_ERROR_FRAGMENT] = "in-error-fragment", + [NETC_STAT_RX_ERROR_JABBER] = "in-error-jabber", + [NETC_STAT_RX_ERROR_DISCARD] = "in-error-discard", + [NETC_STAT_RX_ERROR_NO_TRUNCATED] = "in-error-dicard-no-truncated", + [NETC_STAT_TX_ERROR_FCS] = "out-error-fcs", + [NETC_STAT_TX_ERROR_UNDERSIZE] = "out-error-undersize", + + /* Discard stats */ + [NETC_STAT_RX_DISCARD_COUNT] = "in-discard-count", + [NETC_STAT_RX_DISCARD_REASON0] = "in-discard-reason0", + [NETC_STAT_RX_DISCARD_TABLE_ID] = "in-discard-table-id", + [NETC_STAT_RX_DISCARD_ENTRY_ID] = "in-discard-entry-id", + [NETC_STAT_TX_DISCARD_COUNT] = "out-discard-count", + [NETC_STAT_TX_DISCARD_REASON0] = "out-discard-reason0", + [NETC_STAT_TX_DISCARD_TABLE_ID] = "out-discard-table-id", + [NETC_STAT_TX_DISCARD_ENTRY_ID] = "out-discard-entry-id", + [NETC_STAT_BRIDGE_DISCARD_COUNT] = "bridge-discard-count", + [NETC_STAT_BRIDGE_DISCARD_REASON0] = "bridge-discard-reason0", + [NETC_STAT_BRIDGE_DISCARD_TABLE_ID] = "bridge-discard-table-id", + [NETC_STAT_BRIDGE_DISCARD_ENTRY_ID] = "bridge-discard-entry-id", + + /* Q0 stats */ + [NETC_STAT_Q0_REJECTED_BYTES] = "q0-rejected-bytes", + [NETC_STAT_Q0_REJECTED_FRAMES] = "q0-rejected-frames", + [NETC_STAT_Q0_DEQUEUE_BYTES] = "q0-dequeue-bytes", + [NETC_STAT_Q0_DEQUEUE_FRAMES] = "q0-dequeue-frames", + [NETC_STAT_Q0_DROPPED_BYTES] = "q0-dropped-bytes", + [NETC_STAT_Q0_DROPPED_FRAMES] = "q0-dropped-frames", + [NETC_STAT_Q0_FRAMES] = "q0-frames", + + /* Q1 stats */ + [NETC_STAT_Q1_REJECTED_BYTES] = "q1-rejected-bytes", + [NETC_STAT_Q1_REJECTED_FRAMES] = "q1-rejected-frames", + [NETC_STAT_Q1_DEQUEUE_BYTES] = "q1-dequeue-bytes", + [NETC_STAT_Q1_DEQUEUE_FRAMES] = "q1-dequeue-frames", + [NETC_STAT_Q1_DROPPED_BYTES] = "q1-dropped-bytes", + [NETC_STAT_Q1_DROPPED_FRAMES] = "q1-dropped-frames", + [NETC_STAT_Q1_FRAMES] = "q1-frames", + + /* Q2 stats */ + [NETC_STAT_Q2_REJECTED_BYTES] = "q2-rejected-bytes", + [NETC_STAT_Q2_REJECTED_FRAMES] = "q2-rejected-frames", + [NETC_STAT_Q2_DEQUEUE_BYTES] = "q2-dequeue-bytes", + [NETC_STAT_Q2_DEQUEUE_FRAMES] = "q2-dequeue-frames", + [NETC_STAT_Q2_DROPPED_BYTES] = "q2-dropped-bytes", + [NETC_STAT_Q2_DROPPED_FRAMES] = "q2-dropped-frames", + [NETC_STAT_Q2_FRAMES] = "q2-frames", + + /* Q3 stats */ + [NETC_STAT_Q3_REJECTED_BYTES] = "q3-rejected-bytes", + [NETC_STAT_Q3_REJECTED_FRAMES] = "q3-rejected-frames", + [NETC_STAT_Q3_DEQUEUE_BYTES] = "q3-dequeue-bytes", + [NETC_STAT_Q3_DEQUEUE_FRAMES] = "q3-dequeue-frames", + [NETC_STAT_Q3_DROPPED_BYTES] = "q3-dropped-bytes", + [NETC_STAT_Q3_DROPPED_FRAMES] = "q3-dropped-frames", + [NETC_STAT_Q3_FRAMES] = "q3-frames", + + /* Q4 stats */ + [NETC_STAT_Q4_REJECTED_BYTES] = "q4-rejected-bytes", + [NETC_STAT_Q4_REJECTED_FRAMES] = "q4-rejected-frames", + [NETC_STAT_Q4_DEQUEUE_BYTES] = "q4-dequeue-bytes", + [NETC_STAT_Q4_DEQUEUE_FRAMES] = "q4-dequeue-frames", + [NETC_STAT_Q4_DROPPED_BYTES] = "q4-dropped-bytes", + [NETC_STAT_Q4_DROPPED_FRAMES] = "q4-dropped-frames", + [NETC_STAT_Q4_FRAMES] = "q4-frames", + + /* Q5 stats */ + [NETC_STAT_Q5_REJECTED_BYTES] = "q5-rejected-bytes", + [NETC_STAT_Q5_REJECTED_FRAMES] = "q5-rejected-frames", + [NETC_STAT_Q5_DEQUEUE_BYTES] = "q5-dequeue-bytes", + [NETC_STAT_Q5_DEQUEUE_FRAMES] = "q5-dequeue-frames", + [NETC_STAT_Q5_DROPPED_BYTES] = "q5-dropped-bytes", + [NETC_STAT_Q5_DROPPED_FRAMES] = "q5-dropped-frames", + [NETC_STAT_Q5_FRAMES] = "q5-frames", + + /* Q6 stats */ + [NETC_STAT_Q6_REJECTED_BYTES] = "q6-rejected-bytes", + [NETC_STAT_Q6_REJECTED_FRAMES] = "q6-rejected-frames", + [NETC_STAT_Q6_DEQUEUE_BYTES] = "q6-dequeue-bytes", + [NETC_STAT_Q6_DEQUEUE_FRAMES] = "q6-dequeue-frames", + [NETC_STAT_Q6_DROPPED_BYTES] = "q6-dropped-bytes", + [NETC_STAT_Q6_DROPPED_FRAMES] = "q6-dropped-frames", + [NETC_STAT_Q6_FRAMES] = "q6-frames", + + /* Q7 stats */ + [NETC_STAT_Q7_REJECTED_BYTES] = "q7-rejected-bytes", + [NETC_STAT_Q7_REJECTED_FRAMES] = "q7-rejected-frames", + [NETC_STAT_Q7_DEQUEUE_BYTES] = "q7-dequeue-bytes", + [NETC_STAT_Q7_DEQUEUE_FRAMES] = "q7-dequeue-frames", + [NETC_STAT_Q7_DROPPED_BYTES] = "q7-dropped-bytes", + [NETC_STAT_Q7_DROPPED_FRAMES] = "q7-dropped-frames", + [NETC_STAT_Q7_FRAMES] = "q7-frames", +}; + +void netc_get_ethtool_stats(struct dsa_switch *ds, int port, u64 *data) +{ + struct netc_private *priv = ds->priv; + struct netc_cmd_port_ethtool_stats stats; + int rc; + enum netc_stat_index i; + + rc = netc_xfer_get_cmd(priv, NETC_CMD_PORT_ETHTOOL_STATS_GET, + port, &stats, sizeof(stats)); + + if (rc) { + dev_err(ds->dev, + "Failed to get port %d stats\n", port); + return; + } + + for (i = 0; i < NETC_STAT_NUM; i++) + data[i] = stats.values[i]; +} + +void netc_get_strings(struct dsa_switch *ds, int port, + u32 stringset, u8 *data) +{ + enum netc_stat_index i; + char *p = data; + + if (stringset != ETH_SS_STATS) + return; + + for (i = 0; i < NETC_STAT_NUM; i++) { + strscpy(p, netc_stat_name[i], ETH_GSTRING_LEN); + p += ETH_GSTRING_LEN; + } +} + +int netc_get_sset_count(struct dsa_switch *ds, int port, int sset) +{ + if (sset != ETH_SS_STATS) + return -EOPNOTSUPP; + + return NETC_STAT_NUM; +} diff --git a/drivers/net/dsa/netc/netc_main.c b/drivers/net/dsa/netc/netc_main.c new file mode 100644 index 000000000000..5ba2b8d7ac93 --- /dev/null +++ b/drivers/net/dsa/netc/netc_main.c @@ -0,0 +1,928 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2023 NXP + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "netc.h" + +int netc_is_vlan_configured(struct netc_private *priv, uint16_t vid) +{ + struct netc_vlan_entry *vlan; + int count, i; + + vlan = priv->config.vlan; + count = priv->config.vlan_count; + + for (i = 0; i < count; i++) { + if (vlan[i].vid == vid) + return i; + } + + /* Return an invalid entry index if not found */ + return -1; +} + +static bool vid_is_netc_dsa_8021q(struct dsa_switch *ds, u16 vid) +{ + int port; + struct dsa_port *dp; + unsigned int bridge_num; + u16 standalone_vid, bridge_vid; + + for (port = 0; port < ds->num_ports; port++) { + dp = dsa_to_port(ds, port); + standalone_vid = dsa_tag_8021q_standalone_vid(dp); + + if (vid == standalone_vid) + return true; + + if (dp->bridge) { + bridge_num = dsa_port_bridge_num_get(dp); + bridge_vid = dsa_tag_8021q_bridge_vid(bridge_num); + + if (vid == bridge_vid) + return true; + } + } + + return false; +} + +static int netc_drop_untagged(struct dsa_switch *ds, int port, bool drop) +{ + struct netc_private *priv = ds->priv; + struct netc_mac_config *mac; + + mac = &priv->config.mac[port]; + if (mac->drpuntag == drop) + return 0; + + mac->drpuntag = drop; + + return netc_port_dropuntag_set(priv, port, drop); +} + +static int netc_pvid_apply(struct netc_private *priv, int port, uint16_t pvid) +{ + struct netc_mac_config *mac; + + mac = &priv->config.mac[port]; + if (mac->vlanid == pvid) + return 0; + + mac->vlanid = pvid; + + return netc_port_pvid_set(priv, port, pvid); +} + +static int netc_commit_pvid(struct dsa_switch *ds, int port) +{ + struct dsa_port *dp = dsa_to_port(ds, port); + struct net_device *br = dsa_port_bridge_dev_get(dp); + struct netc_private *priv = ds->priv; + bool drop_untagged = false; + int rc; + uint16_t pvid; + + if (br && br_vlan_enabled(br)) + pvid = priv->bridge_pvid[port]; + else + pvid = priv->tag_8021q_pvid[port]; + + rc = netc_pvid_apply(priv, port, pvid); + if (rc) + return rc; + + /* + * Only force dropping of untagged packets when the port is under a + * VLAN-aware bridge. When the tag_8021q pvid is used, we are + * deliberately removing the RX VLAN from the port's VMEMB_PORT list, + * to prevent DSA tag spoofing from the link partner. Untagged packets + * are the only ones that should be received with tag_8021q, so + * definitely don't drop them. + */ + if (dsa_is_cpu_port(ds, port) || dsa_is_dsa_port(ds, port)) + drop_untagged = true; + + return netc_drop_untagged(ds, port, drop_untagged); +} + +static int netc_fdb_add(struct dsa_switch *ds, int port, + const unsigned char *addr, uint16_t vid, + struct dsa_db db) +{ + struct netc_private *priv = ds->priv; + + if (!vid) { + switch (db.type) { + case DSA_DB_PORT: + vid = dsa_tag_8021q_standalone_vid(db.dp); + break; + case DSA_DB_BRIDGE: + vid = dsa_tag_8021q_bridge_vid(db.bridge.num); + break; + default: + return -EOPNOTSUPP; + } + } + + /* Allow enough time between consecutive calls for adding FDB entry */ + usleep_range(NETC_SPI_MSG_RESPONSE_TIME, + NETC_SPI_MSG_RESPONSE_TIME * 10); + + return netc_fdb_entry_add(priv, addr, vid, port); +} + +static int netc_fdb_del(struct dsa_switch *ds, int port, + const unsigned char *addr, uint16_t vid, + struct dsa_db db) +{ + struct netc_private *priv = ds->priv; + + if (!vid) { + switch (db.type) { + case DSA_DB_PORT: + vid = dsa_tag_8021q_standalone_vid(db.dp); + break; + case DSA_DB_BRIDGE: + vid = dsa_tag_8021q_bridge_vid(db.bridge.num); + break; + default: + return -EOPNOTSUPP; + } + } + + return netc_fdb_entry_del(priv, addr, vid); +} + +static int netc_fdb_dump(struct dsa_switch *ds, int port, + dsa_fdb_dump_cb_t *cb, void *data) +{ + struct netc_private *priv = ds->priv; + struct device *dev = ds->dev; + u32 entry_id = 0, next_id = 0; + int rc; + + while (1) { + struct netc_fdb_entry fdb = {0}; + + rc = netc_fdb_entry_get(priv, &fdb, entry_id, &next_id); + /* No fdb entry at i, not an issue */ + if (rc) { + dev_err(dev, "Failed to dump FDB: %d\n", rc); + return rc; + } + + if (next_id == 0) /* This entry is empty */ + return 0; + + /* + * FDB dump callback is per port. This means we have to + * disregard a valid entry if it's not for this port, even if + * only to revisit it later. This is inefficient because the + * 1024-sized FDB table needs to be traversed 4 times through + * SPI during a 'bridge fdb show' command. + */ + if (fdb.port_map & BIT(port)) { + /* Need to hide the dsa_8021q VLANs from the user. */ + if (vid_is_netc_dsa_8021q(ds, fdb.vid)) + fdb.vid = 0; + + rc = cb(fdb.mac_addr, fdb.vid, fdb.dynamic, data); + if (rc) + return rc; + } + + entry_id = next_id; + + if (entry_id == 0 || entry_id == 0xffffffff) + break; + } + + return 0; +} + +static int netc_parse_ports_node(struct netc_private *priv, + struct device_node *ports_node) +{ + struct device *dev = &priv->spidev->dev; + struct device_node *child; + + for_each_available_child_of_node(ports_node, child) { + struct device_node *phy_node; + phy_interface_t phy_mode; + u32 index; + int err; + + /* Get switch port number from DT */ + if (of_property_read_u32(child, "reg", &index) < 0) { + dev_err(dev, "Port number not defined in device tree\n"); + of_node_put(child); + return -ENODEV; + } + + /* Get PHY mode from DT */ + err = of_get_phy_mode(child, &phy_mode); + if (err) { + dev_err(dev, "Failed to read phy-mode or phy-interface-type %d\n", + index); + of_node_put(child); + return -ENODEV; + } + + phy_node = of_parse_phandle(child, "phy-handle", 0); + if (!phy_node) { + if (!of_phy_is_fixed_link(child)) { + dev_err(dev, "phy-handle or fixed-link properties missing!\n"); + of_node_put(child); + return -ENODEV; + } + /* phy-handle is missing, but fixed-link isn't. + * So it's a fixed link. Default to PHY role. + */ + priv->fixed_link[index] = true; + } else { + of_node_put(phy_node); + } + + priv->phy_mode[index] = phy_mode; + } + + return 0; +} + +static int netc_parse_dt(struct netc_private *priv) +{ + struct device *dev = &priv->spidev->dev; + struct device_node *switch_node = dev->of_node; + struct device_node *ports_node; + int rc; + + ports_node = of_get_child_by_name(switch_node, "ports"); + if (!ports_node) + ports_node = of_get_child_by_name(switch_node, "ethernet-ports"); + if (!ports_node) { + dev_err(dev, "Incorrect bindings: absent \"ports\" node\n"); + return -ENODEV; + } + + rc = netc_parse_ports_node(priv, ports_node); + of_node_put(ports_node); + + return rc; +} + +static void netc_mac_link_down(struct dsa_switch *ds, int port, + unsigned int mode, + phy_interface_t interface) +{ + struct netc_private *priv = ds->priv; + struct netc_mac_config *mac; + + mac = &priv->config.mac[port]; + + mac->egress = false; + + netc_port_link_set(priv, port, false); +} + +static void netc_mac_link_up(struct dsa_switch *ds, int port, + unsigned int mode, + phy_interface_t interface, + struct phy_device *phydev, + int speed, int duplex, + bool tx_pause, bool rx_pause) +{ + struct netc_private *priv = ds->priv; + struct netc_mac_config *mac; + + mac = &priv->config.mac[port]; + + mac->speed = speed; + mac->egress = true; + + netc_port_phylink_mode_set(priv, mac); + netc_port_link_set(priv, port, true); +} + +static void netc_phylink_get_caps(struct dsa_switch *ds, int port, + struct phylink_config *config) +{ + struct netc_private *priv = ds->priv; + phy_interface_t phy_mode; + + /* + * This driver does not make use of the speed, duplex, pause or the + * advertisement in its mac_config, so it is safe to mark this driver + * as non-legacy. + */ + config->legacy_pre_march2020 = false; + + phy_mode = priv->phy_mode[port]; + __set_bit(phy_mode, config->supported_interfaces); + + /* + * The MAC does not support pause frames, and also doesn't + * support half-duplex traffic modes. + */ + config->mac_capabilities = MAC_10FD | MAC_100FD; + config->mac_capabilities |= MAC_1000FD; +} + +static int netc_bridge_member(struct dsa_switch *ds, int port, + struct dsa_bridge bridge, bool member) +{ + int rc; + + rc = netc_commit_pvid(ds, port); + if (rc) + return rc; + + return 0; +} + +static int netc_bridge_join(struct dsa_switch *ds, int port, + struct dsa_bridge bridge, + bool *tx_fwd_offload, + struct netlink_ext_ack *extack) +{ + int rc; + + rc = netc_bridge_member(ds, port, bridge, true); + if (rc) + return rc; + + rc = dsa_tag_8021q_bridge_join(ds, port, bridge); + if (rc) { + netc_bridge_member(ds, port, bridge, false); + return rc; + } + + *tx_fwd_offload = true; + + return 0; +} + +static void netc_bridge_leave(struct dsa_switch *ds, int port, + struct dsa_bridge bridge) +{ + dsa_tag_8021q_bridge_leave(ds, port, bridge); + netc_bridge_member(ds, port, bridge, false); +} + +static enum dsa_tag_protocol +netc_get_tag_protocol(struct dsa_switch *ds, int port, + enum dsa_tag_protocol mp) +{ + struct netc_private *priv = ds->priv; + + return priv->info->tag_proto; +} + +int netc_vlan_filtering(struct dsa_switch *ds, int port, bool enabled, + struct netlink_ext_ack *extack) +{ + struct netc_private *priv = ds->priv; + struct netc_config *config = &priv->config; + int rc; + + if (enabled) { + /* Enable VLAN filtering. */ + config->tpid = ETH_P_8021Q; + config->tpid2 = ETH_P_8021AD; + } else { + /* Disable VLAN filtering. */ + config->tpid = ETH_P_8021Q; + config->tpid2 = ETH_P_NETC; + } + + for (port = 0; port < ds->num_ports; port++) { + if (dsa_is_unused_port(ds, port)) + continue; + + rc = netc_commit_pvid(ds, port); + if (rc) + return rc; + } + + return 0; +} + +static int netc_bridge_vlan_add(struct dsa_switch *ds, int port, + const struct switchdev_obj_port_vlan *vlan, + struct netlink_ext_ack *extack) +{ + struct netc_private *priv = ds->priv; + uint16_t flags = vlan->flags; + bool untagged = false; + int rc; + + /* Be sure to deny the configuration done by tag_8021q. */ + if (vid_is_netc_dsa_8021q(ds, vlan->vid)) { + NL_SET_ERR_MSG_MOD(extack, + "VLAN ID 3072-3076 & 3088 reserved for dsa_8021q operation"); + return -EBUSY; + } + + /* Always install bridge VLANs as egress-tagged on CPU and DSA ports */ + if (dsa_is_cpu_port(ds, port) || dsa_is_dsa_port(ds, port)) + flags = 0; + + if (flags & BRIDGE_VLAN_INFO_UNTAGGED) + untagged = true; + + rc = netc_vlan_entry_add(priv, vlan->vid, port, untagged); + if (rc) + return rc; + + if (vlan->flags & BRIDGE_VLAN_INFO_PVID) + priv->bridge_pvid[port] = vlan->vid; + + /* Allow enough time between adding VLAN entry and setting PVID */ + usleep_range(NETC_SPI_MSG_RESPONSE_TIME, + NETC_SPI_MSG_RESPONSE_TIME * 10); + + return netc_commit_pvid(ds, port); +} + +static int netc_bridge_vlan_del(struct dsa_switch *ds, int port, + const struct switchdev_obj_port_vlan *vlan) +{ + struct netc_private *priv = ds->priv; + int rc; + + rc = netc_vlan_entry_del(priv, vlan->vid, port); + if (rc) + return rc; + + /* + * In case the pvid was deleted, make sure that untagged packets will + * be dropped. + */ + return netc_commit_pvid(ds, port); +} + +static int netc_8021q_vlan_add(struct dsa_switch *ds, int port, + uint16_t vid, uint16_t flags) +{ + struct netc_private *priv = ds->priv; + int rc; + + rc = netc_vlan_entry_add(priv, vid, port, false); + if (rc) + return rc; + + if (flags & BRIDGE_VLAN_INFO_PVID) + priv->tag_8021q_pvid[port] = vid; + + /* Allow enough time between adding VLAN entry and setting PVID */ + usleep_range(NETC_SPI_MSG_RESPONSE_TIME, + NETC_SPI_MSG_RESPONSE_TIME * 10); + + return netc_commit_pvid(ds, port); +} + +static int netc_8021q_vlan_del(struct dsa_switch *ds, int port, uint16_t vid) +{ + struct netc_private *priv = ds->priv; + + return netc_vlan_entry_del(priv, vid, port); +} + +static int netc_prechangeupper(struct dsa_switch *ds, int port, + struct netdev_notifier_changeupper_info *info) +{ + struct netlink_ext_ack *extack = info->info.extack; + struct net_device *upper = info->upper_dev; + struct dsa_switch_tree *dst = ds->dst; + struct dsa_port *dp; + + if (is_vlan_dev(upper)) { + NL_SET_ERR_MSG_MOD(extack, "8021q uppers are not supported"); + return -EBUSY; + } + + if (netif_is_bridge_master(upper)) { + list_for_each_entry(dp, &dst->ports, list) { + struct net_device *br = dsa_port_bridge_dev_get(dp); + + if (br && br != upper && br_vlan_enabled(br)) { + NL_SET_ERR_MSG_MOD(extack, + "Only one VLAN-aware bridge is supported"); + return -EBUSY; + } + } + } + + return 0; +} + +#define work_to_xmit_work(w) \ + container_of((w), struct netc_deferred_xmit_work, work) + +static void netc_port_deferred_xmit(struct kthread_work *work) +{ + struct netc_deferred_xmit_work *xmit_work = work_to_xmit_work(work); + struct sk_buff *clone, *skb = xmit_work->skb; + struct dsa_switch *ds = xmit_work->dp->ds; + int port = xmit_work->dp->index; + + clone = NETC_SKB_CB(skb)->clone; + + /* Transfer skb to the host port. */ + dsa_enqueue_skb(skb, dsa_to_port(ds, port)->slave); + + /* The clone, if there, was made by dsa_skb_tx_timestamp */ + if (clone) + netc_ptp_txtstamp_skb(ds, port, clone); + + kfree(xmit_work); +} + +static int netc_connect_tag_protocol(struct dsa_switch *ds, + enum dsa_tag_protocol proto) +{ + struct netc_private *priv = ds->priv; + struct netc_tagger_data *tagger_data; + + if (proto != priv->info->tag_proto) + return -EPROTONOSUPPORT; + + tagger_data = netc_tagger_data(ds); + tagger_data->xmit_work_fn = netc_port_deferred_xmit; + + return 0; +} + +static int netc_change_mtu(struct dsa_switch *ds, int port, int new_mtu) +{ + struct netc_private *priv = ds->priv; + int maxlen = new_mtu + ETH_HLEN + ETH_FCS_LEN; + + if (dsa_is_cpu_port(ds, port) || dsa_is_dsa_port(ds, port)) + maxlen += VLAN_HLEN; + + return netc_port_mtu_set(priv, port, maxlen); +} + +static int netc_get_max_mtu(struct dsa_switch *ds, int port) +{ + return 2000 - VLAN_ETH_HLEN - ETH_FCS_LEN; +} + +static int netc_mac_init(struct netc_private *priv) +{ + struct netc_mac_config *mac; + struct dsa_switch *ds = priv->ds; + struct dsa_port *dp; + + mac = priv->config.mac; + + dsa_switch_for_each_port(dp, ds) { + mac[dp->index].port = dp->index; + mac[dp->index].speed = 1000; + mac[dp->index].vlanid = 1; + mac[dp->index].drpuntag = false; + mac[dp->index].retag = false; + + if (dsa_port_is_dsa(dp)) + dp->learning = true; + + /* Disallow untagged packets from being received on the + * CPU and DSA ports. + */ + if (dsa_port_is_cpu(dp) || dsa_port_is_dsa(dp)) + mac[dp->index].drpuntag = true; + } + + return 0; +} + +static int netc_dsa_init(struct netc_private *priv) +{ + struct dsa_switch *ds = priv->ds; + struct dsa_port *dp, *cpu_dp = NULL; + const u8 *mac; + int port; + + for (port = 0; port < ds->num_ports; port++) { + if (dsa_is_cpu_port(ds, port)) { + cpu_dp = dsa_to_port(ds, port); + break; + } + } + + if (!cpu_dp) { + dev_err(ds->dev, "Failed to find cpu port\n"); + return -ENODEV; + } + + if (!is_zero_ether_addr(cpu_dp->mac)) + mac = cpu_dp->mac; + else + mac = cpu_dp->master->dev_addr; + + pr_info("NETC DSA: cpu port:%d master:%s\n", + cpu_dp->index, cpu_dp->master->name); + + for (port = 0; port < ds->num_ports; port++) { + dp = dsa_to_port(ds, port); + + if (dsa_port_is_unused(dp)) + continue; + if (dsa_port_is_cpu(dp)) + continue; + + pr_info("NETC DSA: add switch port:%d\n", port); + + netc_port_dsa_add(priv, cpu_dp->index, port, mac); + } + + return 0; +} + +static int netc_setup(struct dsa_switch *ds) +{ + struct netc_private *priv = ds->priv; + int port; + int rc; + + rc = netc_config_setup(&priv->config); + if (rc < 0) { + dev_err(ds->dev, "Failed to setup config: %d\n", rc); + return rc; + } + + netc_mac_init(priv); + netc_dsa_init(priv); + + for (port = 0; port < ds->num_ports; port++) { + priv->tag_8021q_pvid[port] = NETC_DEFAULT_VLAN; + priv->bridge_pvid[port] = NETC_DEFAULT_VLAN; + } + + rc = netc_devlink_setup(ds); + if (rc < 0) + goto out_config_free; + + rtnl_lock(); + rc = dsa_tag_8021q_register(ds, htons(ETH_P_8021Q)); + rtnl_unlock(); + if (rc) + goto out_devlink_teardown; + + /* + * On netc, VLAN filtering per se is always enabled in hardware. + * The only thing we can do to disable it is lie about what the 802.1Q + * EtherType is. + * So it will still try to apply VLAN filtering, but all ingress + * traffic (except frames received with EtherType of ETH_P_NETC) + * will be internally tagged with a distorted VLAN header where the + * TPID is ETH_P_NETC, and the VLAN ID is the port pvid. + */ + ds->vlan_filtering_is_global = true; + ds->untag_bridge_pvid = true; + ds->fdb_isolation = true; + /* tag_8021q has 3 bits for the VBID, and the value 0 is reserved */ + ds->max_num_bridges = 7; + + /* Advertise the 8 egress queues */ + ds->num_tx_queues = NETC_NUM_TC; + + ds->mtu_enforcement_ingress = true; + ds->assisted_learning_on_cpu_port = true; + + return 0; + +out_devlink_teardown: + netc_devlink_teardown(ds); +out_config_free: + netc_config_free(&priv->config); + + return rc; +} + +static void netc_teardown(struct dsa_switch *ds) +{ + struct netc_private *priv = ds->priv; + + rtnl_lock(); + dsa_tag_8021q_unregister(ds); + rtnl_unlock(); + + netc_devlink_teardown(ds); + netc_config_free(&priv->config); +} + +static const struct dsa_switch_ops netc_switch_ops = { + .get_tag_protocol = netc_get_tag_protocol, + .connect_tag_protocol = netc_connect_tag_protocol, + .setup = netc_setup, + .teardown = netc_teardown, + .port_change_mtu = netc_change_mtu, + .port_max_mtu = netc_get_max_mtu, + .phylink_get_caps = netc_phylink_get_caps, + .phylink_mac_link_up = netc_mac_link_up, + .phylink_mac_link_down = netc_mac_link_down, + .get_strings = netc_get_strings, + .get_ethtool_stats = netc_get_ethtool_stats, + .get_sset_count = netc_get_sset_count, + .port_fdb_dump = netc_fdb_dump, + .port_fdb_add = netc_fdb_add, + .port_fdb_del = netc_fdb_del, + .port_bridge_join = netc_bridge_join, + .port_bridge_leave = netc_bridge_leave, + .port_vlan_filtering = netc_vlan_filtering, + .port_vlan_add = netc_bridge_vlan_add, + .port_vlan_del = netc_bridge_vlan_del, + .devlink_info_get = netc_devlink_info_get, + .tag_8021q_vlan_add = netc_8021q_vlan_add, + .tag_8021q_vlan_del = netc_8021q_vlan_del, + .port_prechangeupper = netc_prechangeupper, +}; + +static const struct of_device_id netc_dt_ids[]; +static int netc_check_device_id(struct netc_private *priv) +{ + struct device *dev = &priv->spidev->dev; + struct netc_config *config = &priv->config; + int rc; + + rc = netc_get_devinfo(priv, config); + if (rc < 0) + return rc; + + if (config->device_id != priv->info->device_id) { + dev_err(dev, "Device tree specifies device ID 0x%x, but found 0x%x please fix it!\n", + priv->info->device_id, config->device_id); + return -ENODEV; + } + + return 0; +} + +static int netc_probe(struct spi_device *spi) +{ + struct device *dev = &spi->dev; + struct netc_private *priv; + struct dsa_switch *ds; + size_t max_xfer, max_msg; + int rc; + + if (!dev->of_node) { + dev_err(dev, "No DTS bindings for netc driver\n"); + return -EINVAL; + } + + priv = devm_kzalloc(dev, sizeof(struct netc_private), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + /* + * Populate our driver private structure (priv) based on + * the device tree node that was probed (spi) + */ + priv->spidev = spi; + spi_set_drvdata(spi, priv); + + /* Configure the SPI bus */ + spi->bits_per_word = NETC_SPI_WORD_BITS; + rc = spi_setup(spi); + if (rc < 0) { + dev_err(dev, "Could not init SPI\n"); + return rc; + } + + max_xfer = spi_max_transfer_size(spi); + max_msg = spi_max_message_size(spi); + + /* + * We need to send at least one 64-bit word of SPI payload per message + * in order to be able to make useful progress. + */ + if (max_msg < NETC_SPI_MSG_HEADER_SIZE + 8) { + dev_err(dev, "SPI master cannot send large enough buffers, aborting\n"); + return -EINVAL; + } + + priv->max_xfer_len = NETC_SPI_MSG_MAXLEN; + if (priv->max_xfer_len > max_xfer) + priv->max_xfer_len = max_xfer; + if (priv->max_xfer_len > max_msg - NETC_SPI_MSG_HEADER_SIZE) + priv->max_xfer_len = max_msg - NETC_SPI_MSG_HEADER_SIZE; + + priv->info = of_device_get_match_data(dev); + + /* Detect hardware device */ + rc = netc_check_device_id(priv); + if (rc < 0) { + dev_err(dev, "Device ID check failed: %d\n", rc); + return rc; + } + + dev_info(dev, "Probed switch chip:%s ID:0x%x firmware:%d.%d.%d\n", + priv->info->name, + priv->config.device_id, + priv->config.version_major, + priv->config.version_minor, + priv->config.version_revision); + + ds = devm_kzalloc(dev, sizeof(*ds), GFP_KERNEL); + if (!ds) + return -ENOMEM; + + ds->dev = dev; + ds->num_ports = priv->info->num_ports; + ds->ops = &netc_switch_ops; + ds->priv = priv; + priv->ds = ds; + + mutex_init(&priv->mgmt_lock); + + rc = netc_parse_dt(priv); + if (rc < 0) { + dev_err(ds->dev, "Failed to parse DT: %d\n", rc); + return rc; + } + + return dsa_register_switch(priv->ds); +} + +static void netc_remove(struct spi_device *spi) +{ + struct netc_private *priv = spi_get_drvdata(spi); + + if (!priv) + return; + + dsa_unregister_switch(priv->ds); +} + +static void netc_shutdown(struct spi_device *spi) +{ + struct netc_private *priv = spi_get_drvdata(spi); + + if (!priv) + return; + + dsa_switch_shutdown(priv->ds); + + spi_set_drvdata(spi, NULL); +} + +const struct netc_info netc_info = { + .device_id = NETC_RT1180_DEVICE_ID, + .tag_proto = DSA_TAG_PROTO_NETC_VALUE, + .can_limit_mcast_flood = false, + .num_ports = NETC_NUM_PORTS, + .name = "netc", +}; + +static const struct of_device_id netc_dt_ids[] = { + { .compatible = "nxp,imxrt1180-netc", .data = &netc_info}, + { /* sentinel */ }, +}; +MODULE_DEVICE_TABLE(of, netc_dt_ids); + +static const struct spi_device_id netc_spi_ids[] = { + { "imxrt1180-netc" }, + { }, +}; +MODULE_DEVICE_TABLE(spi, netc_spi_ids); + +static struct spi_driver netc_driver = { + .driver = { + .name = "netc-spi", + .owner = THIS_MODULE, + .of_match_table = of_match_ptr(netc_dt_ids), + }, + .id_table = netc_spi_ids, + .probe = netc_probe, + .remove = netc_remove, + .shutdown = netc_shutdown, +}; + +module_spi_driver(netc_driver); + +MODULE_AUTHOR("Minghuan Lian "); + +MODULE_DESCRIPTION("NETC DSA Driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/net/dsa/netc/netc_ptp.c b/drivers/net/dsa/netc/netc_ptp.c new file mode 100644 index 000000000000..de5fda25702d --- /dev/null +++ b/drivers/net/dsa/netc/netc_ptp.c @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2023 NXP + */ + +#include "netc.h" + +void netc_ptp_txtstamp_skb(struct dsa_switch *ds, int port, + struct sk_buff *skb) +{ + /* To update tx timestamp in the skb */ +} diff --git a/drivers/net/dsa/netc/netc_spi.c b/drivers/net/dsa/netc/netc_spi.c new file mode 100644 index 000000000000..056501bfdd76 --- /dev/null +++ b/drivers/net/dsa/netc/netc_spi.c @@ -0,0 +1,119 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2023 NXP + */ + +#include +#include "netc.h" + +int netc_xfer_cmd(const struct netc_private *priv, + enum netc_spi_rw_mode rw, enum netc_cmd cmd, + void *param, size_t param_len, + void *resp, size_t resp_len) +{ + struct netc_cmd_hdr hdr = {0}; + struct spi_device *spi = priv->spidev; + struct spi_transfer hdr_xfer, resp_xfer; + int rc; + + if (!IS_ALIGNED(resp_len, NETC_SPI_MSG_WORD_BYTES)) { + dev_err(&spi->dev, "netc cmd %d data size should be a multiple of 4 : %ld", + cmd, resp_len); + return -EINVAL; + } + + if (resp_len > priv->max_xfer_len) { + dev_err(&spi->dev, "netc cmd %d data size is too large\n", + cmd); + return -EINVAL; + } + + if (param_len > NETC_SPI_MSG_PARAM_SIZE) { + dev_err(&spi->dev, "netc cmd %d param size is too large\n", + cmd); + return -EINVAL; + } + + hdr.cmd = (rw << NETC_CMD_DIR_SHIFT) | + ((resp_len / NETC_SPI_MSG_WORD_BYTES) << + NETC_CMD_LEN_SHIFT) | + cmd; + if (param) + memcpy(hdr.param, param, param_len); + + hdr_xfer.tx_buf = &hdr; + hdr_xfer.len = NETC_SPI_MSG_HEADER_SIZE; + + rc = spi_sync_transfer(spi, &hdr_xfer, 1); + if (rc < 0) { + dev_err(&spi->dev, "netc cmd %d SPI transfer failed: %d\n", + cmd, rc); + return rc; + } + + usleep_range(NETC_SPI_MSG_RESPONSE_TIME, + NETC_SPI_MSG_RESPONSE_TIME * 10); + + if (!resp) + return 0; + + /* Populate the transfer's data buffer */ + if (rw == SPI_READ) + resp_xfer.rx_buf = resp; + else + resp_xfer.tx_buf = resp; + resp_xfer.len = resp_len; + + rc = spi_sync_transfer(spi, &resp_xfer, 1); + if (rc < 0) { + dev_err(&spi->dev, "netc cmd %d SPI transfer failed: %d\n", + cmd, rc); + return rc; + } + + usleep_range(NETC_SPI_MSG_RESPONSE_TIME, + NETC_SPI_MSG_RESPONSE_TIME * 10); + + return 0; +} + +int netc_xfer_set_cmd(const struct netc_private *priv, + enum netc_cmd cmd, + void *param, size_t param_len) +{ + return netc_xfer_cmd(priv, SPI_WRITE, cmd, + param, param_len, + NULL, 0); +} + +int netc_xfer_get_cmd(const struct netc_private *priv, + enum netc_cmd cmd, uint32_t id, + void *resp, size_t resp_len) +{ + struct netc_cmd_read_param param; + + param.id = id; + + return netc_xfer_cmd(priv, SPI_READ, cmd, + ¶m, sizeof(param), + resp, resp_len); +} + +int netc_xfer_write_reg(const struct netc_private *priv, + uint32_t reg, uint32_t value) +{ + struct netc_cmd_reg_cmd reg_cmd; + + reg_cmd.reg = reg; + reg_cmd.value = value; + + return netc_xfer_set_cmd(priv, NETC_CMD_REG_SET, + ®_cmd, sizeof(reg_cmd)); +} + +int netc_xfer_read_reg(const struct netc_private *priv, + uint32_t reg, uint32_t *value) +{ + return netc_xfer_get_cmd(priv, NETC_CMD_REG_GET, reg, + value, sizeof(*value)); +} diff --git a/drivers/net/dsa/ocelot/felix.c b/drivers/net/dsa/ocelot/felix.c index c8b45a131d77..f84020593ff2 100644 --- a/drivers/net/dsa/ocelot/felix.c +++ b/drivers/net/dsa/ocelot/felix.c @@ -1025,17 +1025,49 @@ static int felix_vlan_add(struct dsa_switch *ds, int port, if (err) return err; - return ocelot_vlan_add(ocelot, port, vlan->vid, - flags & BRIDGE_VLAN_INFO_PVID, - flags & BRIDGE_VLAN_INFO_UNTAGGED); + err = ocelot_vlan_add(ocelot, port, vlan->vid, + flags & BRIDGE_VLAN_INFO_PVID, + flags & BRIDGE_VLAN_INFO_UNTAGGED); + if (err) + return err; + + if (vlan->proto == ETH_P_8021AD) { + if (!ocelot->qinq_enable) { + ocelot->qinq_enable = true; + kref_init(&ocelot->qinq_refcount); + } else { + kref_get(&ocelot->qinq_refcount); + } + } + + return 0; } +static void felix_vlan_qinq_release(struct kref *ref) +{ + struct ocelot *ocelot; + + ocelot = container_of(ref, struct ocelot, qinq_refcount); + ocelot->qinq_enable = false; +} static int felix_vlan_del(struct dsa_switch *ds, int port, const struct switchdev_obj_port_vlan *vlan) { struct ocelot *ocelot = ds->priv; + int err; - return ocelot_vlan_del(ocelot, port, vlan->vid); + err = ocelot_vlan_del(ocelot, port, vlan->vid); + if (err) { + dev_err(ds->dev, "Failed to remove VLAN %d from port %d: %d\n", + vlan->vid, port, err); + return err; + } + + if (ocelot->qinq_enable && vlan->proto == ETH_P_8021AD) + kref_put(&ocelot->qinq_refcount, + felix_vlan_qinq_release); + + return 0; } static void felix_phylink_get_caps(struct dsa_switch *ds, int port, @@ -1083,9 +1115,13 @@ static void felix_phylink_mac_link_down(struct dsa_switch *ds, int port, phy_interface_t interface) { struct ocelot *ocelot = ds->priv; + struct felix *felix = ocelot_to_felix(ocelot); ocelot_phylink_mac_link_down(ocelot, port, link_an_mode, interface, FELIX_MAC_QUIRKS); + + if (felix->info->port_preempt_reset) + felix->info->port_preempt_reset(ocelot, port, 0); } static void felix_phylink_mac_link_up(struct dsa_switch *ds, int port, @@ -1247,6 +1283,44 @@ static int felix_validate_phy_mode(struct felix *felix, int port, return -EOPNOTSUPP; } +static int felix_reset_preempt(struct dsa_switch *ds, int port, bool enable) +{ + struct ocelot *ocelot = ds->priv; + struct felix *felix = ocelot_to_felix(ocelot); + + if (felix->info->port_preempt_reset) { + felix->info->port_preempt_reset(ocelot, port, enable); + + return 0; + } + + return -EOPNOTSUPP; +} + +static int felix_set_preempt(struct dsa_switch *ds, int port, + struct ethtool_fp *fpcmd) +{ + struct ocelot *ocelot = ds->priv; + struct felix *felix = ocelot_to_felix(ocelot); + + if (felix->info->port_set_preempt) + return felix->info->port_set_preempt(ocelot, port, fpcmd); + + return -EOPNOTSUPP; +} + +static int felix_get_preempt(struct dsa_switch *ds, int port, + struct ethtool_fp *fpcmd) +{ + struct ocelot *ocelot = ds->priv; + struct felix *felix = ocelot_to_felix(ocelot); + + if (felix->info->port_get_preempt) + return felix->info->port_get_preempt(ocelot, port, fpcmd); + + return -EOPNOTSUPP; +} + static int felix_parse_ports_node(struct felix *felix, struct device_node *ports_node, phy_interface_t *port_phy_modes) @@ -1549,6 +1623,97 @@ static int felix_connect_tag_protocol(struct dsa_switch *ds, } } +static int felix_qinq_port_bitmap_get(struct dsa_switch *ds, u32 *bitmap) +{ + struct ocelot *ocelot = ds->priv; + struct ocelot_port *ocelot_port; + int port; + + *bitmap = 0; + for (port = 0; port < ds->num_ports; port++) { + ocelot_port = ocelot->ports[port]; + if (ocelot_port->qinq_mode) + *bitmap |= 0x01 << port; + } + + return 0; +} + +static int felix_qinq_port_bitmap_set(struct dsa_switch *ds, u32 bitmap) +{ + struct ocelot *ocelot = ds->priv; + struct ocelot_port *ocelot_port; + int port; + + for (port = 0; port < ds->num_ports; port++) { + ocelot_port = ocelot->ports[port]; + if (bitmap & (0x01 << port)) + ocelot_port->qinq_mode = true; + else + ocelot_port->qinq_mode = false; + } + + return 0; +} + +enum felix_devlink_param_id { + FELIX_DEVLINK_PARAM_ID_BASE = DEVLINK_PARAM_GENERIC_ID_MAX, + FELIX_DEVLINK_PARAM_ID_QINQ_PORT_BITMAP, +}; + +static int felix_devlink_param_get(struct dsa_switch *ds, u32 id, + struct devlink_param_gset_ctx *ctx) +{ + int err; + + switch (id) { + case FELIX_DEVLINK_PARAM_ID_QINQ_PORT_BITMAP: + err = felix_qinq_port_bitmap_get(ds, &ctx->val.vu32); + break; + default: + err = -EOPNOTSUPP; + break; + } + + return err; +} + +static int felix_devlink_param_set(struct dsa_switch *ds, u32 id, + struct devlink_param_gset_ctx *ctx) +{ + int err; + + switch (id) { + case FELIX_DEVLINK_PARAM_ID_QINQ_PORT_BITMAP: + err = felix_qinq_port_bitmap_set(ds, ctx->val.vu32); + break; + default: + err = -EOPNOTSUPP; + break; + } + + return err; +} + +static const struct devlink_param felix_devlink_params[] = { + DSA_DEVLINK_PARAM_DRIVER(FELIX_DEVLINK_PARAM_ID_QINQ_PORT_BITMAP, + "qinq_port_bitmap", + DEVLINK_PARAM_TYPE_U32, + BIT(DEVLINK_PARAM_CMODE_RUNTIME)), +}; + +static int felix_setup_devlink_params(struct dsa_switch *ds) +{ + return dsa_devlink_params_register(ds, felix_devlink_params, + ARRAY_SIZE(felix_devlink_params)); +} + +static void felix_teardown_devlink_params(struct dsa_switch *ds) +{ + dsa_devlink_params_unregister(ds, felix_devlink_params, + ARRAY_SIZE(felix_devlink_params)); +} + /* Hardware initialization done here so that we can allocate structures with * devm without fear of dsa_register_switch returning -EPROBE_DEFER and causing * us to allocate structures twice (leak memory) and map PCI memory twice @@ -1601,6 +1766,10 @@ static int felix_setup(struct dsa_switch *ds) ds->fdb_isolation = true; ds->max_num_bridges = ds->num_ports; + err = felix_setup_devlink_params(ds); + if (err < 0) + return err; + return 0; out_deinit_ports: @@ -1628,6 +1797,8 @@ static void felix_teardown(struct dsa_switch *ds) felix->tag_proto_ops->teardown(ds); rtnl_unlock(); + felix_teardown_devlink_params(ds); + dsa_switch_for_each_available_port(dp, ds) ocelot_deinit_port(ocelot, dp->index); @@ -2080,6 +2251,9 @@ const struct dsa_switch_ops felix_switch_ops = { .get_ethtool_stats = felix_get_ethtool_stats, .get_sset_count = felix_get_sset_count, .get_ts_info = felix_get_ts_info, + .reset_preempt = felix_reset_preempt, + .set_preempt = felix_set_preempt, + .get_preempt = felix_get_preempt, .phylink_get_caps = felix_phylink_get_caps, .phylink_validate = felix_phylink_validate, .phylink_mac_select_pcs = felix_phylink_mac_select_pcs, @@ -2135,6 +2309,8 @@ const struct dsa_switch_ops felix_switch_ops = { .port_mrp_del_ring_role = felix_mrp_del_ring_role, .tag_8021q_vlan_add = felix_tag_8021q_vlan_add, .tag_8021q_vlan_del = felix_tag_8021q_vlan_del, + .devlink_param_get = felix_devlink_param_get, + .devlink_param_set = felix_devlink_param_set, .port_get_default_prio = felix_port_get_default_prio, .port_set_default_prio = felix_port_set_default_prio, .port_get_dscp_prio = felix_port_get_dscp_prio, diff --git a/drivers/net/dsa/ocelot/felix.h b/drivers/net/dsa/ocelot/felix.h index 0e57f9d518be..72bb1b687237 100644 --- a/drivers/net/dsa/ocelot/felix.h +++ b/drivers/net/dsa/ocelot/felix.h @@ -61,6 +61,11 @@ struct felix_info { void (*tas_guard_bands_update)(struct ocelot *ocelot, int port); void (*port_sched_speed_set)(struct ocelot *ocelot, int port, int speed); + int (*port_set_preempt)(struct ocelot *ocelot, int port, + struct ethtool_fp *fpcmd); + int (*port_get_preempt)(struct ocelot *ocelot, int port, + struct ethtool_fp *fpcmd); + void (*port_preempt_reset)(struct ocelot *ocelot, int port, bool enable); }; /* Methods for initializing the hardware resources specific to a tagging diff --git a/drivers/net/dsa/ocelot/felix_tsn.c b/drivers/net/dsa/ocelot/felix_tsn.c index 302e21c7fddf..fd0ea4cd7d7b 100644 --- a/drivers/net/dsa/ocelot/felix_tsn.c +++ b/drivers/net/dsa/ocelot/felix_tsn.c @@ -498,10 +498,12 @@ static int felix_cb_streamid_set(struct net_device *ndev, u32 index, bool enable if (!stream) return -EINVAL; + dst_idx = stream->dst_idx; ocelot_mact_forget(ocelot, stream->mac, stream->vid); + felix_stream_table_del(index); - felix_streamid_force_forward_clear(ocelot, stream->dst_idx); + felix_streamid_force_forward_clear(ocelot, dst_idx); return 0; } @@ -1536,7 +1538,7 @@ static int felix_pcpmap_set(struct net_device *ndev, ocelot = dp->ds->priv; port = dp->index; - index = (c->pcp & GENMASK(2, 0)) * ((c->dei & BIT(0)) + 1); + index = (c->dei & BIT(0)) * 8 + (c->pcp & GENMASK(2, 0)); ocelot_rmw_ix(ocelot, (ANA_PORT_PCP_DEI_MAP_DP_PCP_DEI_VAL & (c->dpl << 3)) | diff --git a/drivers/net/dsa/ocelot/felix_vsc9959.c b/drivers/net/dsa/ocelot/felix_vsc9959.c index 871a3f68b3ef..18d51bda4052 100644 --- a/drivers/net/dsa/ocelot/felix_vsc9959.c +++ b/drivers/net/dsa/ocelot/felix_vsc9959.c @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -2663,6 +2664,103 @@ static const struct ocelot_ops vsc9959_ops = { .update_stats = vsc9959_update_stats, }; +static void vsc9959_port_preempt_reset(struct ocelot *ocelot, int port, bool enable) +{ + struct ocelot_port *ocelot_port = ocelot->ports[port]; + + ocelot_port_rmwl(ocelot_port, 0, + DEV_MM_CONFIG_ENABLE_CONFIG_MM_RX_ENA | + DEV_MM_CONFIG_ENABLE_CONFIG_MM_TX_ENA, + DEV_MM_ENABLE_CONFIG); + + if (enable) { + if (ocelot_port->fp_enabled_admin) { + ocelot_port_rmwl(ocelot_port, + DEV_MM_CONFIG_ENABLE_CONFIG_MM_RX_ENA | + DEV_MM_CONFIG_ENABLE_CONFIG_MM_TX_ENA, + DEV_MM_CONFIG_ENABLE_CONFIG_MM_RX_ENA | + DEV_MM_CONFIG_ENABLE_CONFIG_MM_TX_ENA, + DEV_MM_ENABLE_CONFIG); + } + } +} + +static int vsc9959_port_set_preempt(struct ocelot *ocelot, int port, + struct ethtool_fp *fpcmd) +{ + struct ocelot_port *ocelot_port = ocelot->ports[port]; + struct felix *felix = ocelot_to_felix(ocelot); + int p_queues = fpcmd->preemptible_queues_mask; + int mm_fragsize, val; + + if (!fpcmd->disabled && + (fpcmd->min_frag_size < 60 || fpcmd->min_frag_size > 252)) + return -EINVAL; + + mm_fragsize = DIV_ROUND_UP((fpcmd->min_frag_size + 4), 64) - 1; + + if (!fpcmd->disabled) { + val = DEV_MM_CONFIG_ENABLE_CONFIG_MM_RX_ENA | + DEV_MM_CONFIG_ENABLE_CONFIG_MM_TX_ENA; + ocelot_port->fp_enabled_admin = 1; + } else { + val = 0; + ocelot_port->fp_enabled_admin = 0; + } + + ocelot_port_rmwl(ocelot_port, val, + DEV_MM_CONFIG_ENABLE_CONFIG_MM_RX_ENA | + DEV_MM_CONFIG_ENABLE_CONFIG_MM_TX_ENA, + DEV_MM_ENABLE_CONFIG); + + ocelot_port_rmwl(ocelot_port, + (fpcmd->fp_enabled ? + 0 : DEV_MM_CONFIG_VERIF_CONFIG_PRM_VERIFY_DIS), + DEV_MM_CONFIG_VERIF_CONFIG_PRM_VERIFY_DIS, + DEV_MM_VERIF_CONFIG); + + ocelot_rmw_rix(ocelot, + QSYS_PREEMPTION_CFG_MM_ADD_FRAG_SIZE(mm_fragsize) | + QSYS_PREEMPTION_CFG_P_QUEUES(p_queues), + QSYS_PREEMPTION_CFG_MM_ADD_FRAG_SIZE_M | + QSYS_PREEMPTION_CFG_P_QUEUES_M, + QSYS_PREEMPTION_CFG, + port); + + ocelot_port->preemptable_prios = p_queues; + + if (ocelot_port->taprio && felix->info->tas_guard_bands_update) + felix->info->tas_guard_bands_update(ocelot, port); + + return 0; +} + +static int vsc9959_port_get_preempt(struct ocelot *ocelot, int port, + struct ethtool_fp *fpcmd) +{ + struct ocelot_port *ocelot_port = ocelot->ports[port]; + u8 fragsize; + u32 val; + + fpcmd->fp_supported = 1; + fpcmd->supported_queues_mask = GENMASK(7, 0); + + val = ocelot_port_readl(ocelot_port, DEV_MM_STATUS); + val &= DEV_MM_STATISTICS_MM_STATUS_PRMPT_ACTIVE_STATUS; + fpcmd->fp_active = (val ? 1 : 0); + + val = ocelot_port_readl(ocelot_port, DEV_MM_ENABLE_CONFIG); + val &= DEV_MM_CONFIG_ENABLE_CONFIG_MM_RX_ENA; + fpcmd->fp_status = val; + + val = ocelot_read_rix(ocelot, QSYS_PREEMPTION_CFG, port); + fpcmd->preemptible_queues_mask = val & QSYS_PREEMPTION_CFG_P_QUEUES_M; + fragsize = QSYS_PREEMPTION_CFG_MM_ADD_FRAG_SIZE_X(val); + fpcmd->min_frag_size = (fragsize + 1) * 64 - 4; + + return 0; +} + static const struct felix_info felix_info_vsc9959 = { .resources = vsc9959_resources, .num_resources = ARRAY_SIZE(vsc9959_resources), @@ -2688,6 +2786,9 @@ static const struct felix_info felix_info_vsc9959 = { .port_setup_tc = vsc9959_port_setup_tc, .port_sched_speed_set = vsc9959_sched_speed_set, .tas_guard_bands_update = vsc9959_tas_guard_bands_update, + .port_set_preempt = vsc9959_port_set_preempt, + .port_get_preempt = vsc9959_port_get_preempt, + .port_preempt_reset = vsc9959_port_preempt_reset, }; static irqreturn_t felix_irq_handler(int irq, void *data) diff --git a/drivers/net/dsa/sja1105/sja1105.h b/drivers/net/dsa/sja1105/sja1105.h index 06e0ebfe652a..0e52b6ceff78 100644 --- a/drivers/net/dsa/sja1105/sja1105.h +++ b/drivers/net/dsa/sja1105/sja1105.h @@ -10,6 +10,7 @@ #include #include #include +#include #include #include "sja1105_static_config.h" @@ -253,6 +254,7 @@ struct sja1105_private { unsigned long bcast_egress_floods; unsigned long hwts_tx_en; unsigned long hwts_rx_en; + u32 hostprio; const struct sja1105_info *info; size_t max_xfer_len; struct spi_device *spidev; @@ -296,6 +298,7 @@ enum sja1105_reset_reason { SJA1105_SCHEDULING, SJA1105_BEST_EFFORT_POLICING, SJA1105_VIRTUAL_LINKS, + SJA1105_VLAN_PCP_TO_TXQ_MAPPING, }; int sja1105_static_config_reload(struct sja1105_private *priv, @@ -303,6 +306,8 @@ int sja1105_static_config_reload(struct sja1105_private *priv, int sja1105_vlan_filtering(struct dsa_switch *ds, int port, bool enabled, struct netlink_ext_ack *extack); void sja1105_frame_memory_partitioning(struct sja1105_private *priv); +int sja1105_setup_tc_mqprio(struct dsa_switch *ds, int port, + struct tc_mqprio_qopt_offload *mqprio); /* From sja1105_mdio.c */ int sja1105_mdiobus_register(struct dsa_switch *ds); diff --git a/drivers/net/dsa/sja1105/sja1105_clocking.c b/drivers/net/dsa/sja1105/sja1105_clocking.c index e3699f76f6d7..08a3e7b96254 100644 --- a/drivers/net/dsa/sja1105/sja1105_clocking.c +++ b/drivers/net/dsa/sja1105/sja1105_clocking.c @@ -153,14 +153,14 @@ static int sja1105_cgu_mii_tx_clk_config(struct sja1105_private *priv, { const struct sja1105_regs *regs = priv->info->regs; struct sja1105_cgu_mii_ctrl mii_tx_clk; - const int mac_clk_sources[] = { + static const int mac_clk_sources[] = { CLKSRC_MII0_TX_CLK, CLKSRC_MII1_TX_CLK, CLKSRC_MII2_TX_CLK, CLKSRC_MII3_TX_CLK, CLKSRC_MII4_TX_CLK, }; - const int phy_clk_sources[] = { + static const int phy_clk_sources[] = { CLKSRC_IDIV0, CLKSRC_IDIV1, CLKSRC_IDIV2, @@ -194,7 +194,7 @@ sja1105_cgu_mii_rx_clk_config(struct sja1105_private *priv, int port) const struct sja1105_regs *regs = priv->info->regs; struct sja1105_cgu_mii_ctrl mii_rx_clk; u8 packed_buf[SJA1105_SIZE_CGU_CMD] = {0}; - const int clk_sources[] = { + static const int clk_sources[] = { CLKSRC_MII0_RX_CLK, CLKSRC_MII1_RX_CLK, CLKSRC_MII2_RX_CLK, @@ -221,7 +221,7 @@ sja1105_cgu_mii_ext_tx_clk_config(struct sja1105_private *priv, int port) const struct sja1105_regs *regs = priv->info->regs; struct sja1105_cgu_mii_ctrl mii_ext_tx_clk; u8 packed_buf[SJA1105_SIZE_CGU_CMD] = {0}; - const int clk_sources[] = { + static const int clk_sources[] = { CLKSRC_IDIV0, CLKSRC_IDIV1, CLKSRC_IDIV2, @@ -248,7 +248,7 @@ sja1105_cgu_mii_ext_rx_clk_config(struct sja1105_private *priv, int port) const struct sja1105_regs *regs = priv->info->regs; struct sja1105_cgu_mii_ctrl mii_ext_rx_clk; u8 packed_buf[SJA1105_SIZE_CGU_CMD] = {0}; - const int clk_sources[] = { + static const int clk_sources[] = { CLKSRC_IDIV0, CLKSRC_IDIV1, CLKSRC_IDIV2, @@ -349,8 +349,13 @@ static int sja1105_cgu_rgmii_tx_clk_config(struct sja1105_private *priv, if (speed == priv->info->port_speed[SJA1105_SPEED_1000MBPS]) { clksrc = CLKSRC_PLL0; } else { - int clk_sources[] = {CLKSRC_IDIV0, CLKSRC_IDIV1, CLKSRC_IDIV2, - CLKSRC_IDIV3, CLKSRC_IDIV4}; + static const int clk_sources[] = { + CLKSRC_IDIV0, + CLKSRC_IDIV1, + CLKSRC_IDIV2, + CLKSRC_IDIV3, + CLKSRC_IDIV4, + }; clksrc = clk_sources[port]; } @@ -638,7 +643,7 @@ static int sja1105_cgu_rmii_ref_clk_config(struct sja1105_private *priv, const struct sja1105_regs *regs = priv->info->regs; struct sja1105_cgu_mii_ctrl ref_clk; u8 packed_buf[SJA1105_SIZE_CGU_CMD] = {0}; - const int clk_sources[] = { + static const int clk_sources[] = { CLKSRC_MII0_TX_CLK, CLKSRC_MII1_TX_CLK, CLKSRC_MII2_TX_CLK, diff --git a/drivers/net/dsa/sja1105/sja1105_main.c b/drivers/net/dsa/sja1105/sja1105_main.c index f1f1368e8146..f8b220ed97ff 100644 --- a/drivers/net/dsa/sja1105/sja1105_main.c +++ b/drivers/net/dsa/sja1105/sja1105_main.c @@ -863,7 +863,7 @@ static int sja1105_init_general_params(struct sja1105_private *priv) /* Priority queue for link-local management frames * (both ingress to and egress from CPU - PTP, STP etc) */ - .hostprio = 7, + .hostprio = priv->hostprio, .mac_fltres1 = SJA1105_LINKLOCAL_FILTER_A, .mac_flt1 = SJA1105_LINKLOCAL_FILTER_A_MASK, .incl_srcpt1 = true, @@ -1256,6 +1256,15 @@ static int sja1105_parse_dt(struct sja1105_private *priv) return -ENODEV; } + if (of_property_read_u32(switch_node, "hostprio", &priv->hostprio) < 0) { + priv->hostprio = 7; + } else if (priv->hostprio >= SJA1105_NUM_TC) { + dev_err(dev, "Out of range hostprio, must be between 0 and %d\n", (SJA1105_NUM_TC - 1)); + return -ERANGE; + } + + dev_info(dev, "Configured hostprio: using queue %u\n", priv->hostprio); + rc = sja1105_parse_ports_node(priv, ports_node); of_node_put(ports_node); @@ -2282,6 +2291,7 @@ static const char * const sja1105_reset_reasons[] = { [SJA1105_SCHEDULING] = "Time-aware scheduling", [SJA1105_BEST_EFFORT_POLICING] = "Best-effort policing", [SJA1105_VIRTUAL_LINKS] = "Virtual links", + SJA1105_VLAN_PCP_TO_TXQ_MAPPING] = "VLAN PCP to TX queue mapping", }; /* For situations where we need to change a setting at runtime that is only @@ -2433,6 +2443,56 @@ sja1105_get_tag_protocol(struct dsa_switch *ds, int port, return priv->info->tag_proto; } +int sja1105_setup_tc_mqprio(struct dsa_switch *ds, int port, + struct tc_mqprio_qopt_offload *mqprio) +{ + struct sja1105_l2_forwarding_entry *l2fwd; + struct sja1105_private *priv = ds->priv; + struct sja1105_table *table; + int pcp, tc; + + if (mqprio->qopt.num_tc > SJA1105_MAX_NUM_PCP) { + dev_err(ds->dev, + "Only a maximum of %u traffic classes are supported by hardware\n", + SJA1105_MAX_NUM_PCP); + return -ERANGE; + } + + table = &priv->static_config.tables[BLK_IDX_L2_FORWARDING]; + + l2fwd = table->entries; + + if (!mqprio->qopt.num_tc) { + /* Delete qdisc: reset to default 1:1 mapping. */ + for (pcp = 0; pcp < SJA1105_MAX_NUM_PCP; pcp++) + l2fwd[ds->num_ports + pcp].vlan_pmap[port] = pcp; + } else { + /* We restrict a single TXQ per traffic class + * The SJA1105 doesn't offer round robin among TXQs of the same priority + */ + for (tc = 0; tc < mqprio->qopt.num_tc; tc++) { + if (mqprio->qopt.count[tc] != 1) { + dev_err(ds->dev, + "Only a single TXQ per traffic class is supported\n"); + return -EOPNOTSUPP; + } + } + + /* Use MQPRIO mapping to configure Egress PCP to HW queue mapping. */ + for (pcp = 0; pcp < SJA1105_MAX_NUM_PCP; pcp++) + l2fwd[ds->num_ports + pcp].vlan_pmap[port] = mqprio->qopt.prio_tc_map[pcp]; + } + + /* Although, the Egress PCP to HW queue mapping (the latter 8 entries) + * should be configured dynamically (once max_dynp is properly set: e.g 7), + * that did not work in practice. The switch (SJA1105Q) kept using + * the old mapping until a switch reset appears and force the reload of + * the static configuration. + * So, force a switch reset on mapping offload from here. + */ + return sja1105_static_config_reload(priv, SJA1105_VLAN_PCP_TO_TXQ_MAPPING); +} + /* The TPID setting belongs to the General Parameters table, * which can only be partially reconfigured at runtime (and not the TPID). * So a switch reset is required. @@ -2834,6 +2894,8 @@ static int sja1105_port_setup_tc(struct dsa_switch *ds, int port, return sja1105_setup_tc_taprio(ds, port, type_data); case TC_SETUP_QDISC_CBS: return sja1105_setup_tc_cbs(ds, port, type_data); + case TC_SETUP_QDISC_MQPRIO: + return sja1105_setup_tc_mqprio(ds, port, type_data); default: return -EOPNOTSUPP; } diff --git a/drivers/net/dsa/sja1105/sja1105_tas.c b/drivers/net/dsa/sja1105/sja1105_tas.c index e6153848a950..20a06266134e 100644 --- a/drivers/net/dsa/sja1105/sja1105_tas.c +++ b/drivers/net/dsa/sja1105/sja1105_tas.c @@ -527,6 +527,8 @@ int sja1105_setup_tc_taprio(struct dsa_switch *ds, int port, if (rc < 0) return rc; + sja1105_setup_tc_mqprio(ds, port, &admin->mqprio); + return sja1105_static_config_reload(priv, SJA1105_SCHEDULING); } @@ -575,6 +577,8 @@ int sja1105_setup_tc_taprio(struct dsa_switch *ds, int port, if (rc < 0) return rc; + sja1105_setup_tc_mqprio(ds, port, &admin->mqprio); + return sja1105_static_config_reload(priv, SJA1105_SCHEDULING); } diff --git a/drivers/net/ethernet/alacritech/slic.h b/drivers/net/ethernet/alacritech/slic.h index 4eecbdfff3ff..82071d0e5f7f 100644 --- a/drivers/net/ethernet/alacritech/slic.h +++ b/drivers/net/ethernet/alacritech/slic.h @@ -288,13 +288,13 @@ do { \ u64_stats_update_end(&(st)->syncp); \ } while (0) -#define SLIC_GET_STATS_COUNTER(newst, st, counter) \ -{ \ - unsigned int start; \ +#define SLIC_GET_STATS_COUNTER(newst, st, counter) \ +{ \ + unsigned int start; \ do { \ - start = u64_stats_fetch_begin_irq(&(st)->syncp); \ - newst = (st)->counter; \ - } while (u64_stats_fetch_retry_irq(&(st)->syncp, start)); \ + start = u64_stats_fetch_begin(&(st)->syncp); \ + newst = (st)->counter; \ + } while (u64_stats_fetch_retry(&(st)->syncp, start)); \ } struct slic_upr { diff --git a/drivers/net/ethernet/amazon/ena/ena_ethtool.c b/drivers/net/ethernet/amazon/ena/ena_ethtool.c index 444ccef76da2..8da79eedc057 100644 --- a/drivers/net/ethernet/amazon/ena/ena_ethtool.c +++ b/drivers/net/ethernet/amazon/ena/ena_ethtool.c @@ -118,9 +118,9 @@ static void ena_safe_update_stat(u64 *src, u64 *dst, unsigned int start; do { - start = u64_stats_fetch_begin_irq(syncp); + start = u64_stats_fetch_begin(syncp); *(dst) = *src; - } while (u64_stats_fetch_retry_irq(syncp, start)); + } while (u64_stats_fetch_retry(syncp, start)); } static void ena_queue_stats(struct ena_adapter *adapter, u64 **data) diff --git a/drivers/net/ethernet/amazon/ena/ena_netdev.c b/drivers/net/ethernet/amazon/ena/ena_netdev.c index 5ce01ac72637..e8ad5ea31aff 100644 --- a/drivers/net/ethernet/amazon/ena/ena_netdev.c +++ b/drivers/net/ethernet/amazon/ena/ena_netdev.c @@ -3305,10 +3305,10 @@ static void ena_get_stats64(struct net_device *netdev, tx_ring = &adapter->tx_ring[i]; do { - start = u64_stats_fetch_begin_irq(&tx_ring->syncp); + start = u64_stats_fetch_begin(&tx_ring->syncp); packets = tx_ring->tx_stats.cnt; bytes = tx_ring->tx_stats.bytes; - } while (u64_stats_fetch_retry_irq(&tx_ring->syncp, start)); + } while (u64_stats_fetch_retry(&tx_ring->syncp, start)); stats->tx_packets += packets; stats->tx_bytes += bytes; @@ -3316,20 +3316,20 @@ static void ena_get_stats64(struct net_device *netdev, rx_ring = &adapter->rx_ring[i]; do { - start = u64_stats_fetch_begin_irq(&rx_ring->syncp); + start = u64_stats_fetch_begin(&rx_ring->syncp); packets = rx_ring->rx_stats.cnt; bytes = rx_ring->rx_stats.bytes; - } while (u64_stats_fetch_retry_irq(&rx_ring->syncp, start)); + } while (u64_stats_fetch_retry(&rx_ring->syncp, start)); stats->rx_packets += packets; stats->rx_bytes += bytes; } do { - start = u64_stats_fetch_begin_irq(&adapter->syncp); + start = u64_stats_fetch_begin(&adapter->syncp); rx_drops = adapter->dev_stats.rx_drops; tx_drops = adapter->dev_stats.tx_drops; - } while (u64_stats_fetch_retry_irq(&adapter->syncp, start)); + } while (u64_stats_fetch_retry(&adapter->syncp, start)); stats->rx_dropped = rx_drops; stats->tx_dropped = tx_drops; diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_main.c b/drivers/net/ethernet/aquantia/atlantic/aq_main.c index 77609dc0a08d..0b2a52199914 100644 --- a/drivers/net/ethernet/aquantia/atlantic/aq_main.c +++ b/drivers/net/ethernet/aquantia/atlantic/aq_main.c @@ -21,6 +21,7 @@ #include #include #include +#include #include MODULE_LICENSE("GPL v2"); diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_ring.c b/drivers/net/ethernet/aquantia/atlantic/aq_ring.c index 2dc8d215a591..7f933175cbda 100644 --- a/drivers/net/ethernet/aquantia/atlantic/aq_ring.c +++ b/drivers/net/ethernet/aquantia/atlantic/aq_ring.c @@ -948,7 +948,7 @@ unsigned int aq_ring_fill_stats_data(struct aq_ring_s *self, u64 *data) /* This data should mimic aq_ethtool_queue_rx_stat_names structure */ do { count = 0; - start = u64_stats_fetch_begin_irq(&self->stats.rx.syncp); + start = u64_stats_fetch_begin(&self->stats.rx.syncp); data[count] = self->stats.rx.packets; data[++count] = self->stats.rx.jumbo_packets; data[++count] = self->stats.rx.lro_packets; @@ -965,15 +965,15 @@ unsigned int aq_ring_fill_stats_data(struct aq_ring_s *self, u64 *data) data[++count] = self->stats.rx.xdp_tx; data[++count] = self->stats.rx.xdp_invalid; data[++count] = self->stats.rx.xdp_redirect; - } while (u64_stats_fetch_retry_irq(&self->stats.rx.syncp, start)); + } while (u64_stats_fetch_retry(&self->stats.rx.syncp, start)); } else { /* This data should mimic aq_ethtool_queue_tx_stat_names structure */ do { count = 0; - start = u64_stats_fetch_begin_irq(&self->stats.tx.syncp); + start = u64_stats_fetch_begin(&self->stats.tx.syncp); data[count] = self->stats.tx.packets; data[++count] = self->stats.tx.queue_restarts; - } while (u64_stats_fetch_retry_irq(&self->stats.tx.syncp, start)); + } while (u64_stats_fetch_retry(&self->stats.tx.syncp, start)); } return ++count; diff --git a/drivers/net/ethernet/asix/ax88796c_main.c b/drivers/net/ethernet/asix/ax88796c_main.c index 8b7cdf015a16..21376c79f671 100644 --- a/drivers/net/ethernet/asix/ax88796c_main.c +++ b/drivers/net/ethernet/asix/ax88796c_main.c @@ -662,12 +662,12 @@ static void ax88796c_get_stats64(struct net_device *ndev, s = per_cpu_ptr(ax_local->stats, cpu); do { - start = u64_stats_fetch_begin_irq(&s->syncp); + start = u64_stats_fetch_begin(&s->syncp); rx_packets = u64_stats_read(&s->rx_packets); rx_bytes = u64_stats_read(&s->rx_bytes); tx_packets = u64_stats_read(&s->tx_packets); tx_bytes = u64_stats_read(&s->tx_bytes); - } while (u64_stats_fetch_retry_irq(&s->syncp, start)); + } while (u64_stats_fetch_retry(&s->syncp, start)); stats->rx_packets += rx_packets; stats->rx_bytes += rx_bytes; diff --git a/drivers/net/ethernet/broadcom/b44.c b/drivers/net/ethernet/broadcom/b44.c index 7f876721596c..b751dc8486dc 100644 --- a/drivers/net/ethernet/broadcom/b44.c +++ b/drivers/net/ethernet/broadcom/b44.c @@ -1680,7 +1680,7 @@ static void b44_get_stats64(struct net_device *dev, unsigned int start; do { - start = u64_stats_fetch_begin_irq(&hwstat->syncp); + start = u64_stats_fetch_begin(&hwstat->syncp); /* Convert HW stats into rtnl_link_stats64 stats. */ nstat->rx_packets = hwstat->rx_pkts; @@ -1714,7 +1714,7 @@ static void b44_get_stats64(struct net_device *dev, /* Carrier lost counter seems to be broken for some devices */ nstat->tx_carrier_errors = hwstat->tx_carrier_lost; #endif - } while (u64_stats_fetch_retry_irq(&hwstat->syncp, start)); + } while (u64_stats_fetch_retry(&hwstat->syncp, start)); } @@ -2082,12 +2082,12 @@ static void b44_get_ethtool_stats(struct net_device *dev, do { data_src = &hwstat->tx_good_octets; data_dst = data; - start = u64_stats_fetch_begin_irq(&hwstat->syncp); + start = u64_stats_fetch_begin(&hwstat->syncp); for (i = 0; i < ARRAY_SIZE(b44_gstrings); i++) *data_dst++ = *data_src++; - } while (u64_stats_fetch_retry_irq(&hwstat->syncp, start)); + } while (u64_stats_fetch_retry(&hwstat->syncp, start)); } static void b44_get_wol(struct net_device *dev, struct ethtool_wolinfo *wol) diff --git a/drivers/net/ethernet/broadcom/bcmsysport.c b/drivers/net/ethernet/broadcom/bcmsysport.c index 425d6ccd5413..f8b1adc389b3 100644 --- a/drivers/net/ethernet/broadcom/bcmsysport.c +++ b/drivers/net/ethernet/broadcom/bcmsysport.c @@ -457,10 +457,10 @@ static void bcm_sysport_update_tx_stats(struct bcm_sysport_priv *priv, for (q = 0; q < priv->netdev->num_tx_queues; q++) { ring = &priv->tx_rings[q]; do { - start = u64_stats_fetch_begin_irq(&priv->syncp); + start = u64_stats_fetch_begin(&priv->syncp); bytes = ring->bytes; packets = ring->packets; - } while (u64_stats_fetch_retry_irq(&priv->syncp, start)); + } while (u64_stats_fetch_retry(&priv->syncp, start)); *tx_bytes += bytes; *tx_packets += packets; @@ -504,9 +504,9 @@ static void bcm_sysport_get_stats(struct net_device *dev, if (s->stat_sizeof == sizeof(u64) && s->type == BCM_SYSPORT_STAT_NETDEV64) { do { - start = u64_stats_fetch_begin_irq(syncp); + start = u64_stats_fetch_begin(syncp); data[i] = *(u64 *)p; - } while (u64_stats_fetch_retry_irq(syncp, start)); + } while (u64_stats_fetch_retry(syncp, start)); } else data[i] = *(u32 *)p; j++; @@ -1878,10 +1878,10 @@ static void bcm_sysport_get_stats64(struct net_device *dev, &stats->tx_packets); do { - start = u64_stats_fetch_begin_irq(&priv->syncp); + start = u64_stats_fetch_begin(&priv->syncp); stats->rx_packets = stats64->rx_packets; stats->rx_bytes = stats64->rx_bytes; - } while (u64_stats_fetch_retry_irq(&priv->syncp, start)); + } while (u64_stats_fetch_retry(&priv->syncp, start)); } static void bcm_sysport_netif_start(struct net_device *dev) diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_mqprio.h b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_mqprio.h index be96f1dc0372..d4a862a9fd7d 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_mqprio.h +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_mqprio.h @@ -4,7 +4,7 @@ #ifndef __CXGB4_TC_MQPRIO_H__ #define __CXGB4_TC_MQPRIO_H__ -#include +#include #define CXGB4_EOSW_TXQ_DEFAULT_DESC_NUM 128 diff --git a/drivers/net/ethernet/cortina/gemini.c b/drivers/net/ethernet/cortina/gemini.c index fdf10318758b..5715b9ab2712 100644 --- a/drivers/net/ethernet/cortina/gemini.c +++ b/drivers/net/ethernet/cortina/gemini.c @@ -1919,7 +1919,7 @@ static void gmac_get_stats64(struct net_device *netdev, /* Racing with RX NAPI */ do { - start = u64_stats_fetch_begin_irq(&port->rx_stats_syncp); + start = u64_stats_fetch_begin(&port->rx_stats_syncp); stats->rx_packets = port->stats.rx_packets; stats->rx_bytes = port->stats.rx_bytes; @@ -1931,11 +1931,11 @@ static void gmac_get_stats64(struct net_device *netdev, stats->rx_crc_errors = port->stats.rx_crc_errors; stats->rx_frame_errors = port->stats.rx_frame_errors; - } while (u64_stats_fetch_retry_irq(&port->rx_stats_syncp, start)); + } while (u64_stats_fetch_retry(&port->rx_stats_syncp, start)); /* Racing with MIB and TX completion interrupts */ do { - start = u64_stats_fetch_begin_irq(&port->ir_stats_syncp); + start = u64_stats_fetch_begin(&port->ir_stats_syncp); stats->tx_errors = port->stats.tx_errors; stats->tx_packets = port->stats.tx_packets; @@ -1945,15 +1945,15 @@ static void gmac_get_stats64(struct net_device *netdev, stats->rx_missed_errors = port->stats.rx_missed_errors; stats->rx_fifo_errors = port->stats.rx_fifo_errors; - } while (u64_stats_fetch_retry_irq(&port->ir_stats_syncp, start)); + } while (u64_stats_fetch_retry(&port->ir_stats_syncp, start)); /* Racing with hard_start_xmit */ do { - start = u64_stats_fetch_begin_irq(&port->tx_stats_syncp); + start = u64_stats_fetch_begin(&port->tx_stats_syncp); stats->tx_dropped = port->stats.tx_dropped; - } while (u64_stats_fetch_retry_irq(&port->tx_stats_syncp, start)); + } while (u64_stats_fetch_retry(&port->tx_stats_syncp, start)); stats->rx_dropped += stats->rx_missed_errors; } @@ -2031,18 +2031,18 @@ static void gmac_get_ethtool_stats(struct net_device *netdev, /* Racing with MIB interrupt */ do { p = values; - start = u64_stats_fetch_begin_irq(&port->ir_stats_syncp); + start = u64_stats_fetch_begin(&port->ir_stats_syncp); for (i = 0; i < RX_STATS_NUM; i++) *p++ = port->hw_stats[i]; - } while (u64_stats_fetch_retry_irq(&port->ir_stats_syncp, start)); + } while (u64_stats_fetch_retry(&port->ir_stats_syncp, start)); values = p; /* Racing with RX NAPI */ do { p = values; - start = u64_stats_fetch_begin_irq(&port->rx_stats_syncp); + start = u64_stats_fetch_begin(&port->rx_stats_syncp); for (i = 0; i < RX_STATUS_NUM; i++) *p++ = port->rx_stats[i]; @@ -2050,13 +2050,13 @@ static void gmac_get_ethtool_stats(struct net_device *netdev, *p++ = port->rx_csum_stats[i]; *p++ = port->rx_napi_exits; - } while (u64_stats_fetch_retry_irq(&port->rx_stats_syncp, start)); + } while (u64_stats_fetch_retry(&port->rx_stats_syncp, start)); values = p; /* Racing with TX start_xmit */ do { p = values; - start = u64_stats_fetch_begin_irq(&port->tx_stats_syncp); + start = u64_stats_fetch_begin(&port->tx_stats_syncp); for (i = 0; i < TX_MAX_FRAGS; i++) { *values++ = port->tx_frag_stats[i]; @@ -2065,7 +2065,7 @@ static void gmac_get_ethtool_stats(struct net_device *netdev, *values++ = port->tx_frags_linearized; *values++ = port->tx_hw_csummed; - } while (u64_stats_fetch_retry_irq(&port->tx_stats_syncp, start)); + } while (u64_stats_fetch_retry(&port->tx_stats_syncp, start)); } static int gmac_get_ksettings(struct net_device *netdev, diff --git a/drivers/net/ethernet/emulex/benet/be_ethtool.c b/drivers/net/ethernet/emulex/benet/be_ethtool.c index 77edc3d9b505..a29de29bdf23 100644 --- a/drivers/net/ethernet/emulex/benet/be_ethtool.c +++ b/drivers/net/ethernet/emulex/benet/be_ethtool.c @@ -389,10 +389,10 @@ static void be_get_ethtool_stats(struct net_device *netdev, struct be_rx_stats *stats = rx_stats(rxo); do { - start = u64_stats_fetch_begin_irq(&stats->sync); + start = u64_stats_fetch_begin(&stats->sync); data[base] = stats->rx_bytes; data[base + 1] = stats->rx_pkts; - } while (u64_stats_fetch_retry_irq(&stats->sync, start)); + } while (u64_stats_fetch_retry(&stats->sync, start)); for (i = 2; i < ETHTOOL_RXSTATS_NUM; i++) { p = (u8 *)stats + et_rx_stats[i].offset; @@ -405,19 +405,19 @@ static void be_get_ethtool_stats(struct net_device *netdev, struct be_tx_stats *stats = tx_stats(txo); do { - start = u64_stats_fetch_begin_irq(&stats->sync_compl); + start = u64_stats_fetch_begin(&stats->sync_compl); data[base] = stats->tx_compl; - } while (u64_stats_fetch_retry_irq(&stats->sync_compl, start)); + } while (u64_stats_fetch_retry(&stats->sync_compl, start)); do { - start = u64_stats_fetch_begin_irq(&stats->sync); + start = u64_stats_fetch_begin(&stats->sync); for (i = 1; i < ETHTOOL_TXSTATS_NUM; i++) { p = (u8 *)stats + et_tx_stats[i].offset; data[base + i] = (et_tx_stats[i].size == sizeof(u64)) ? *(u64 *)p : *(u32 *)p; } - } while (u64_stats_fetch_retry_irq(&stats->sync, start)); + } while (u64_stats_fetch_retry(&stats->sync, start)); base += ETHTOOL_TXSTATS_NUM; } } diff --git a/drivers/net/ethernet/emulex/benet/be_main.c b/drivers/net/ethernet/emulex/benet/be_main.c index b12152e2fca0..52d6d43e7bfe 100644 --- a/drivers/net/ethernet/emulex/benet/be_main.c +++ b/drivers/net/ethernet/emulex/benet/be_main.c @@ -665,10 +665,10 @@ static void be_get_stats64(struct net_device *netdev, const struct be_rx_stats *rx_stats = rx_stats(rxo); do { - start = u64_stats_fetch_begin_irq(&rx_stats->sync); + start = u64_stats_fetch_begin(&rx_stats->sync); pkts = rx_stats(rxo)->rx_pkts; bytes = rx_stats(rxo)->rx_bytes; - } while (u64_stats_fetch_retry_irq(&rx_stats->sync, start)); + } while (u64_stats_fetch_retry(&rx_stats->sync, start)); stats->rx_packets += pkts; stats->rx_bytes += bytes; stats->multicast += rx_stats(rxo)->rx_mcast_pkts; @@ -680,10 +680,10 @@ static void be_get_stats64(struct net_device *netdev, const struct be_tx_stats *tx_stats = tx_stats(txo); do { - start = u64_stats_fetch_begin_irq(&tx_stats->sync); + start = u64_stats_fetch_begin(&tx_stats->sync); pkts = tx_stats(txo)->tx_pkts; bytes = tx_stats(txo)->tx_bytes; - } while (u64_stats_fetch_retry_irq(&tx_stats->sync, start)); + } while (u64_stats_fetch_retry(&tx_stats->sync, start)); stats->tx_packets += pkts; stats->tx_bytes += bytes; } @@ -2156,16 +2156,16 @@ static int be_get_new_eqd(struct be_eq_obj *eqo) for_all_rx_queues_on_eq(adapter, eqo, rxo, i) { do { - start = u64_stats_fetch_begin_irq(&rxo->stats.sync); + start = u64_stats_fetch_begin(&rxo->stats.sync); rx_pkts += rxo->stats.rx_pkts; - } while (u64_stats_fetch_retry_irq(&rxo->stats.sync, start)); + } while (u64_stats_fetch_retry(&rxo->stats.sync, start)); } for_all_tx_queues_on_eq(adapter, eqo, txo, i) { do { - start = u64_stats_fetch_begin_irq(&txo->stats.sync); + start = u64_stats_fetch_begin(&txo->stats.sync); tx_pkts += txo->stats.tx_reqs; - } while (u64_stats_fetch_retry_irq(&txo->stats.sync, start)); + } while (u64_stats_fetch_retry(&txo->stats.sync, start)); } /* Skip, if wrapped around or first calculation */ diff --git a/drivers/net/ethernet/freescale/Kconfig b/drivers/net/ethernet/freescale/Kconfig index 4aaada823a67..c7707f62470c 100644 --- a/drivers/net/ethernet/freescale/Kconfig +++ b/drivers/net/ethernet/freescale/Kconfig @@ -35,6 +35,18 @@ config FEC Say Y here if you want to use the built-in 10/100 Fast ethernet controller on some Motorola ColdFire and Freescale i.MX/S32 processors. +config FEC_ECAT + tristate "FEC native dirver for FEC ethernet controller (of ColdFire and some i.MX CPUs)" + depends on (ARCH_MXC) + default ARCH_MXC if ARM + depends on PTP_1588_CLOCK_OPTIONAL + select CRC32 + select PHYLIB + imply NET_SELFTESTS + help + Say Y here if you want to use the built-in 10/100 Fast ethercat + controller on some Motorola ColdFire and Freescale i.MX/S32 processors. + config FEC_UIO tristate "FEC_UIO ethernet controller (i.MX 8M Mini CPU)" default n @@ -43,6 +55,14 @@ config FEC_UIO Say Y here if you want to use the built-in 10/100 Fast ethernet controller on Freescale i.MX 8M Mini processor. +config AVB_SUPPORT + bool "AVB interface support" + depends on FEC && ARCH_MXC + help + Say Y here if you want to enable the AVB interface in the FEC + driver. For the actual AVB functionality an external module is + still required. + config FEC_MPC52xx tristate "FEC MPC52xx driver" depends on PPC_MPC52xx && PPC_BESTCOMM diff --git a/drivers/net/ethernet/freescale/Makefile b/drivers/net/ethernet/freescale/Makefile index 13ae9401e4c2..bd3ee981ce54 100644 --- a/drivers/net/ethernet/freescale/Makefile +++ b/drivers/net/ethernet/freescale/Makefile @@ -7,6 +7,8 @@ obj-$(CONFIG_FEC) += fec.o fec-objs :=fec_main.o fec_ptp.o obj-$(CONFIG_FEC_UIO) += fec_uio.o +obj-$(CONFIG_FEC_ECAT) += fec_ecat.o + obj-$(CONFIG_FEC_MPC52xx) += fec_mpc52xx.o ifeq ($(CONFIG_FEC_MPC52xx_MDIO),y) obj-$(CONFIG_FEC_MPC52xx) += fec_mpc52xx_phy.o diff --git a/drivers/net/ethernet/freescale/enetc/enetc.h b/drivers/net/ethernet/freescale/enetc/enetc.h index 538d9f1587c8..a7877b6e9999 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc.h +++ b/drivers/net/ethernet/freescale/enetc/enetc.h @@ -396,6 +396,8 @@ struct enetc_ndev_priv { struct work_struct tx_onestep_tstamp; struct sk_buff_head tx_skbs; + + bool fp_enabled_admin; }; /* Messaging */ @@ -440,6 +442,7 @@ int enetc_xsk_wakeup(struct net_device *dev, u32 queue, u32 flags); /* ethtool */ void enetc_set_ethtool_ops(struct net_device *ndev); +int enetc_pmac_reset(struct net_device *ndev, bool enable); /* control buffer descriptor ring (CBDR) */ int enetc_setup_cbdr(struct device *dev, struct enetc_hw *hw, int bd_count, @@ -594,11 +597,13 @@ static inline int enetc_set_psfp(struct net_device *ndev, bool en) void enetc_tsn_pf_init(struct net_device *netdev, struct pci_dev *pdev); void enetc_tsn_pf_deinit(struct net_device *netdev); void enetc_pspeed_set(struct enetc_ndev_priv *priv, int speed); +void enetc_ptp_clock_update(void); #else #define enetc_tsn_pf_init(netdev, pdev) (void)0 #define enetc_tsn_pf_deinit(netdev) (void)0 #define enetc_pspeed_set(priv, speed) (void)0 +#define enetc_ptp_clock_update() (void)0 #endif diff --git a/drivers/net/ethernet/freescale/enetc/enetc_ethtool.c b/drivers/net/ethernet/freescale/enetc/enetc_ethtool.c index 5ad52e8c72f6..da33d37a83c2 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc_ethtool.c +++ b/drivers/net/ethernet/freescale/enetc/enetc_ethtool.c @@ -898,6 +898,56 @@ static int enetc_set_link_ksettings(struct net_device *dev, return phylink_ethtool_ksettings_set(priv->phylink, cmd); } +static void enetc_configure_port_pmac(struct enetc_hw *hw, bool enable) +{ + u32 temp; + + /* Set pMAC step lock */ + temp = enetc_port_rd(hw, ENETC_PFPMR); + enetc_port_wr(hw, ENETC_PFPMR, + temp | ENETC_PFPMR_PMACE | ENETC_PFPMR_MWLM); + + temp = enetc_port_rd(hw, ENETC_MMCSR); + if (enable) + temp |= ENETC_MMCSR_ME; + else + temp &= (~ENETC_MMCSR_ME); + enetc_port_wr(hw, ENETC_MMCSR, temp); +} + +int enetc_pmac_reset(struct net_device *ndev, bool enable) +{ + struct enetc_ndev_priv *priv = netdev_priv(ndev); + u32 temp; + + temp = enetc_port_rd(&priv->si->hw, ENETC_PFPMR); + if (temp & ENETC_PFPMR_PMACE) + enetc_configure_port_pmac(&priv->si->hw, enable); + + return 0; +} + +static int enetc_reset_preempt(struct net_device *ndev, bool enable) +{ + struct enetc_ndev_priv *priv = netdev_priv(ndev); + u32 temp; + + temp = enetc_rd(&priv->si->hw, ENETC_PTGCR); + if (temp & ENETC_PTGCR_TGE) + enetc_wr(&priv->si->hw, ENETC_PTGCR, + temp & (~ENETC_PTGCR_TGPE)); + + if (enable) { + if (priv->fp_enabled_admin) { + enetc_configure_port_pmac(&priv->si->hw, 1); + } + } else { + enetc_configure_port_pmac(&priv->si->hw, 0); + } + + return 0; +} + static void enetc_get_channels(struct net_device *ndev, struct ethtool_channels *chan) { @@ -907,6 +957,105 @@ static void enetc_get_channels(struct net_device *ndev, chan->combined_count = priv->bdr_int_num; } +static int enetc_set_preempt(struct net_device *ndev, + struct ethtool_fp *pt) +{ + struct enetc_ndev_priv *priv = netdev_priv(ndev); + u32 preempt, temp; + int rafs; + int i; + + if (!pt) + return -EINVAL; + + if (!pt->disabled && (pt->min_frag_size < 60 || pt->min_frag_size > 252)) + return -EINVAL; + + rafs = DIV_ROUND_UP((pt->min_frag_size + 4), 64) - 1; + + preempt = pt->preemptible_queues_mask; + + temp = enetc_rd(&priv->si->hw, ENETC_PTGCR); + if (temp & ENETC_PTGCR_TGE) + enetc_wr(&priv->si->hw, ENETC_PTGCR, + temp & (~ENETC_PTGCR_TGPE)); + + for (i = 0; i < 8; i++) { + /* 1 Enabled. Traffic is transmitted on the preemptive MAC. */ + temp = enetc_port_rd(&priv->si->hw, ENETC_PTCFPR(i)); + + if ((preempt >> i) & 0x1) + enetc_port_wr(&priv->si->hw, + ENETC_PTCFPR(i), + temp | ENETC_FPE); + else + enetc_port_wr(&priv->si->hw, + ENETC_PTCFPR(i), + temp & ~ENETC_FPE); + } + + temp = enetc_port_rd(&priv->si->hw, ENETC_MMCSR); + temp &= ~ENETC_MMCSR_RAFS_MASK; + temp |= ENETC_MMCSR_RAFS(rafs); + if (pt->fp_enabled) + temp &= ~ENETC_MMCSR_VDIS; + else + temp |= ENETC_MMCSR_VDIS; + enetc_port_wr(&priv->si->hw, ENETC_MMCSR, temp); + + if (pt->disabled) { + enetc_configure_port_pmac(&priv->si->hw, 0); + priv->fp_enabled_admin = 0; + } else { + enetc_configure_port_pmac(&priv->si->hw, 1); + priv->fp_enabled_admin = 1; + } + + if (pt->disabled) { + temp = enetc_port_rd(&priv->si->hw, ENETC_PFPMR); + enetc_port_wr(&priv->si->hw, ENETC_PFPMR, + temp & ~(ENETC_PFPMR_PMACE | ENETC_PFPMR_MWLM)); + } + + return 0; +} + +static int enetc_get_preempt(struct net_device *ndev, + struct ethtool_fp *pt) +{ + struct enetc_ndev_priv *priv = netdev_priv(ndev); + u32 temp; + int i; + + if (!pt) + return -EINVAL; + + temp = enetc_port_rd(&priv->si->hw, ENETC_MMCSR); + if (!(temp & ENETC_MMCSR_VDIS) && (ENETC_MMCSR_GET_VSTS(temp) == 3)) + pt->fp_active = true; + else if ((temp & ENETC_MMCSR_VDIS) && (temp & ENETC_MMCSR_ME)) + pt->fp_active = true; + else + pt->fp_active = false; + + if (temp & ENETC_MMCSR_ME) + pt->fp_status = true; + else + pt->fp_status = false; + + pt->preemptible_queues_mask = 0; + for (i = 0; i < 8; i++) + if (enetc_port_rd(&priv->si->hw, ENETC_PTCFPR(i)) & 0x80000000) + pt->preemptible_queues_mask |= 1 << i; + + pt->fp_supported = !!(priv->si->hw_features & ENETC_SI_F_QBU); + pt->supported_queues_mask = 0xff; + temp = enetc_port_rd(&priv->si->hw, ENETC_MMCSR); + pt->min_frag_size = (ENETC_MMCSR_GET_RAFS(temp) + 1) * 64; + + return 0; +} + static const struct ethtool_ops enetc_pf_ethtool_ops = { .supported_coalesce_params = ETHTOOL_COALESCE_USECS | ETHTOOL_COALESCE_MAX_FRAMES | @@ -938,6 +1087,9 @@ static const struct ethtool_ops enetc_pf_ethtool_ops = { .get_pauseparam = enetc_get_pauseparam, .set_pauseparam = enetc_set_pauseparam, .get_channels = enetc_get_channels, + .set_preempt = enetc_set_preempt, + .get_preempt = enetc_get_preempt, + .reset_preempt = enetc_reset_preempt, }; static const struct ethtool_ops enetc_vf_ethtool_ops = { diff --git a/drivers/net/ethernet/freescale/enetc/enetc_hw.h b/drivers/net/ethernet/freescale/enetc/enetc_hw.h index 85db68c54e82..343a47962187 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc_hw.h +++ b/drivers/net/ethernet/freescale/enetc/enetc_hw.h @@ -223,7 +223,13 @@ enum enetc_bdr_type {TX, RX}; #define ENETC_PSIVHFR0(n) (0x1e00 + (n) * 8) /* n = SI index */ #define ENETC_PSIVHFR1(n) (0x1e04 + (n) * 8) /* n = SI index */ #define ENETC_MMCSR 0x1f00 +#define ENETC_MMCSR_VSTS GENMASK(20, 18) +#define ENETC_MMCSR_GET_VSTS(x) (((x) & ENETC_MMCSR_VSTS) >> 18) +#define ENETC_MMCSR_VDIS BIT(17) #define ENETC_MMCSR_ME BIT(16) +#define ENETC_MMCSR_RAFS_MASK GENMASK(9, 8) +#define ENETC_MMCSR_RAFS(x) (((x) << 8) & ENETC_MMCSR_RAFS_MASK) +#define ENETC_MMCSR_GET_RAFS(x) (((x) & ENETC_MMCSR_RAFS_MASK) >> 8) #define ENETC_PTCMSDUR(n) (0x2020 + (n) * 4) /* n = TC index [0..7] */ #define ENETC_PM0_CMD_CFG 0x8008 diff --git a/drivers/net/ethernet/freescale/enetc/enetc_pf.c b/drivers/net/ethernet/freescale/enetc/enetc_pf.c index 3e8f2defd67f..08fe4cd64c44 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc_pf.c +++ b/drivers/net/ethernet/freescale/enetc/enetc_pf.c @@ -590,26 +590,11 @@ static void enetc_mac_enable(struct enetc_hw *hw, bool en) enetc_port_wr(hw, ENETC_PM1_CMD_CFG, val); } -static void enetc_configure_port_pmac(struct enetc_hw *hw) -{ - u32 temp; - - /* Set pMAC step lock */ - temp = enetc_port_rd(hw, ENETC_PFPMR); - enetc_port_wr(hw, ENETC_PFPMR, - temp | ENETC_PFPMR_PMACE | ENETC_PFPMR_MWLM); - - temp = enetc_port_rd(hw, ENETC_MMCSR); - enetc_port_wr(hw, ENETC_MMCSR, temp | ENETC_MMCSR_ME); -} - static void enetc_configure_port(struct enetc_pf *pf) { u8 hash_key[ENETC_RSSHASH_KEY_SIZE]; struct enetc_hw *hw = &pf->si->hw; - enetc_configure_port_pmac(hw); - enetc_configure_port_mac(hw); enetc_port_si_configure(pf->si); @@ -1111,6 +1096,7 @@ static void enetc_pl_mac_link_down(struct phylink_config *config, struct enetc_pf *pf = phylink_to_enetc_pf(config); enetc_mac_enable(&pf->si->hw, false); + enetc_pmac_reset(pf->si->ndev, 0); } static const struct phylink_mac_ops enetc_mac_phylink_ops = { diff --git a/drivers/net/ethernet/freescale/enetc/enetc_ptp.c b/drivers/net/ethernet/freescale/enetc/enetc_ptp.c index 5243fc031058..9c671494cd95 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc_ptp.c +++ b/drivers/net/ethernet/freescale/enetc/enetc_ptp.c @@ -10,6 +10,19 @@ int enetc_phc_index = -1; EXPORT_SYMBOL_GPL(enetc_phc_index); +int ptp_enetc_settime(struct ptp_clock_info *ptp, const struct timespec64 *ts) +{ + int ret; + + ret = ptp_qoriq_settime(ptp, ts); + if (ret) + return ret; + + enetc_ptp_clock_update(); + + return ret; +} + static struct ptp_clock_info enetc_ptp_caps = { .owner = THIS_MODULE, .name = "ENETC PTP clock", @@ -22,7 +35,7 @@ static struct ptp_clock_info enetc_ptp_caps = { .adjfine = ptp_qoriq_adjfine, .adjtime = ptp_qoriq_adjtime, .gettime64 = ptp_qoriq_gettime, - .settime64 = ptp_qoriq_settime, + .settime64 = ptp_enetc_settime, .enable = ptp_qoriq_enable, }; diff --git a/drivers/net/ethernet/freescale/enetc/enetc_tsn.c b/drivers/net/ethernet/freescale/enetc/enetc_tsn.c index 5ff36647be84..091e12c76cfa 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc_tsn.c +++ b/drivers/net/ethernet/freescale/enetc/enetc_tsn.c @@ -1,5 +1,5 @@ // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) -/* Copyright 2017-2019 NXP */ +/* Copyright 2017-2023 NXP */ #include "enetc.h" @@ -8,6 +8,13 @@ #include #include +struct netdev_list_entry { + struct list_head list; + struct net_device *dev; +}; + +static struct list_head netdev_list = {0}; + static int alloc_cbdr(struct enetc_si *si, struct enetc_cbd **curr_cbd) { struct enetc_cbdr *ring = &si->cbd_ring; @@ -184,17 +191,8 @@ static int enetc_qbv_set(struct net_device *ndev, gcl_config->atc = admin_basic->gate_states; gcl_config->acl_len = cpu_to_le16(gcl_len); - if (!admin_basic->base_time) { - gcl_data->btl = - cpu_to_le32(enetc_rd(hw, ENETC_SICTR0)); - gcl_data->bth = - cpu_to_le32(enetc_rd(hw, ENETC_SICTR1)); - } else { - gcl_data->btl = - cpu_to_le32(lower_32_bits(admin_basic->base_time)); - gcl_data->bth = - cpu_to_le32(upper_32_bits(admin_basic->base_time)); - } + gcl_data->btl = cpu_to_le32(lower_32_bits(admin_basic->base_time)); + gcl_data->bth = cpu_to_le32(upper_32_bits(admin_basic->base_time)); gcl_data->ct = cpu_to_le32(admin_basic->cycle_time); gcl_data->cte = cpu_to_le32(admin_basic->cycle_time_extension); @@ -256,8 +254,8 @@ static int enetc_qbv_get(struct net_device *ndev, { struct tsn_qbv_basic *admin_basic = &admin_conf->admin; struct enetc_ndev_priv *priv = netdev_priv(ndev); + u16 data_size, admin_len, oper_len, maxlen; struct enetc_hw *hw = &priv->si->hw; - u16 data_size, admin_len, maxlen; struct tgs_gcl_query *gcl_query; struct tgs_gcl_resp *gcl_data; struct enetc_cbd *cbdr; @@ -290,6 +288,7 @@ static int enetc_qbv_get(struct net_device *ndev, gce = (struct gce *)(gcl_data + 1); gcl_query->acl_len = cpu_to_le16(maxlen); + gcl_query->ocl_len = cpu_to_le16(maxlen); dma_size = cpu_to_le16(data_size); cbdr->length = dma_size; @@ -312,8 +311,12 @@ static int enetc_qbv_get(struct net_device *ndev, /* since cbdr already passed to free, below could be get wrong */ admin_len = le16_to_cpu(gcl_query->admin_list_len); + oper_len = le16_to_cpu(gcl_query->oper_list_len); - admin_basic->control_list_length = admin_len; + if (!admin_len) + admin_basic->control_list_length = oper_len; + else + admin_basic->control_list_length = admin_len; temp = ((u64)le32_to_cpu(gcl_data->abth)) << 32; admin_basic->base_time = le32_to_cpu(gcl_data->abtl) + temp; @@ -321,7 +324,7 @@ static int enetc_qbv_get(struct net_device *ndev, admin_basic->cycle_time = le32_to_cpu(gcl_data->act); admin_basic->cycle_time_extension = le32_to_cpu(gcl_data->acte); - admin_basic->control_list = kcalloc(admin_len, + admin_basic->control_list = kcalloc(admin_basic->control_list_length, sizeof(*admin_basic->control_list), GFP_KERNEL); if (!admin_basic->control_list) { @@ -330,10 +333,15 @@ static int enetc_qbv_get(struct net_device *ndev, return -ENOMEM; } - for (i = 0; i < admin_len; i++) { - struct gce *temp_gce = gce + i; + for (i = 0; i < admin_basic->control_list_length; i++) { + struct gce *temp_gce; struct tsn_qbv_entry *temp_entry; + if (!admin_len) + temp_gce = gce + maxlen + i; + else + temp_gce = gce + i; + temp_entry = admin_basic->control_list + i; temp_entry->gate_state = temp_gce->gate; @@ -468,6 +476,120 @@ static int enetc_qbv_get_status(struct net_device *ndev, return 0; } +static int enetc_qbv_gcl_reset(struct net_device *ndev, + struct tsn_qbv_conf *admin_conf) +{ + struct tsn_qbv_basic *admin_basic = &admin_conf->admin; + struct enetc_ndev_priv *priv = netdev_priv(ndev); + struct enetc_hw *hw = &priv->si->hw; + struct tgs_gcl_conf *gcl_config; + struct tgs_gcl_data *gcl_data; + struct enetc_cbd *cbdr; + struct gce *gce; + dma_addr_t dma; + u16 data_size; + int curr_cbd; + u16 gcl_len; + u32 temp; + int i; + + gcl_len = admin_basic->control_list_length; + if (gcl_len > enetc_get_max_gcl_len(hw)) + return -EINVAL; + + temp = enetc_rd(hw, ENETC_PTGCR); + if (admin_conf->gate_enabled) { + enetc_wr(hw, ENETC_PTGCR, temp & ~ENETC_PTGCR_TGE); + usleep_range(10, 20); + enetc_wr(hw, ENETC_PTGCR, temp | ENETC_PTGCR_TGE); + } else if (!admin_conf->gate_enabled) { + return 0; + } + + /* Configure the (administrative) gate control list using the + * control BD descriptor. + */ + curr_cbd = alloc_cbdr(priv->si, &cbdr); + + gcl_config = &cbdr->gcl_conf; + + data_size = struct_size(gcl_data, entry, gcl_len); + + gcl_data = kzalloc(data_size, __GFP_DMA | GFP_KERNEL); + if (!gcl_data) + return -ENOMEM; + + gce = &gcl_data->entry[0]; + + gcl_config->atc = admin_basic->gate_states; + gcl_config->acl_len = cpu_to_le16(gcl_len); + + gcl_data->btl = cpu_to_le32(lower_32_bits(admin_basic->base_time)); + gcl_data->bth = cpu_to_le32(upper_32_bits(admin_basic->base_time)); + + gcl_data->ct = cpu_to_le32(admin_basic->cycle_time); + gcl_data->cte = cpu_to_le32(admin_basic->cycle_time_extension); + + for (i = 0; i < gcl_len; i++) { + struct gce *temp_gce = gce + i; + struct tsn_qbv_entry *temp_entry; + + temp_entry = admin_basic->control_list + i; + + temp_gce->gate = temp_entry->gate_state; + temp_gce->period = cpu_to_le32(temp_entry->time_interval); + } + + cbdr->length = cpu_to_le16(data_size); + cbdr->status_flags = 0; + + dma = dma_map_single(&priv->si->pdev->dev, gcl_data, + data_size, DMA_TO_DEVICE); + if (dma_mapping_error(&priv->si->pdev->dev, dma)) { + netdev_err(priv->si->ndev, "DMA mapping failed!\n"); + kfree(gcl_data); + return -ENOMEM; + } + + cbdr->addr[0] = cpu_to_le32(lower_32_bits(dma)); + cbdr->addr[1] = cpu_to_le32(upper_32_bits(dma)); + cbdr->cmd = 0; + cbdr->cls = BDCR_CMD_PORT_GCL; + + /* Updated by ENETC on completion of the configuration + * command. A zero value indicates success. + */ + cbdr->status_flags = 0; + + xmit_cbdr(priv->si, curr_cbd); + + memset(cbdr, 0, sizeof(struct enetc_cbd)); + dma_unmap_single(&priv->si->pdev->dev, dma, data_size, DMA_TO_DEVICE); + kfree(gcl_data); + + return 0; +} + +static int enetc_est_reset(struct net_device *ndev) +{ + struct tsn_qbv_conf admin_conf = {0}; + int ret; + + ret = enetc_qbv_get(ndev, &admin_conf); + if (ret) + return ret; + + return enetc_qbv_gcl_reset(ndev, &admin_conf); +} + +void enetc_ptp_clock_update() +{ + struct netdev_list_entry *entry; + + list_for_each_entry(entry, &netdev_list, list) + enetc_est_reset(entry->dev); +} + /* CBD Class 7: Stream Identity Entry Set Descriptor - Long Format */ static int enetc_cb_streamid_set(struct net_device *ndev, u32 index, bool en, struct tsn_cb_streamid *streamid) @@ -1560,6 +1682,14 @@ static int enetc_qbu_set(struct net_device *ndev, u8 ptvector) temp & ~ENETC_FPE); } + /* Set pMAC step lock */ + temp = enetc_port_rd(&priv->si->hw, ENETC_PFPMR); + enetc_port_wr(&priv->si->hw, ENETC_PFPMR, + temp | ENETC_PFPMR_PMACE | ENETC_PFPMR_MWLM); + + temp = enetc_port_rd(&priv->si->hw, ENETC_MMCSR); + enetc_port_wr(&priv->si->hw, ENETC_MMCSR, temp | ENETC_MMCSR_ME); + return 0; } @@ -1922,6 +2052,33 @@ static struct tsn_ops enetc_tsn_ops_part = { .qci_fmi_get = enetc_qci_fmi_get, }; +static void enetc_tsn_netdev_list_add(struct net_device *ndev) +{ + struct netdev_list_entry *entry; + + entry = kzalloc(sizeof(*entry), GFP_KERNEL); + if (!entry) + return; + + entry->dev = ndev; + + if (!netdev_list.next) + INIT_LIST_HEAD(&netdev_list); + + list_add_tail(&entry->list, &netdev_list); +} + +static void enetc_tsn_netdev_list_del(struct net_device *ndev) +{ + struct netdev_list_entry *tmp, *n; + + list_for_each_entry_safe(tmp, n, &netdev_list, list) + if (tmp->dev == ndev) { + list_del(&tmp->list); + kfree(tmp); + } +} + void enetc_tsn_pf_init(struct net_device *netdev, struct pci_dev *pdev) { int port = pdev->devfn & 0x7; @@ -1932,9 +2089,12 @@ void enetc_tsn_pf_init(struct net_device *netdev, struct pci_dev *pdev) else tsn_port_register(netdev, &enetc_tsn_ops_full, (u16)pdev->bus->number); + + enetc_tsn_netdev_list_add(netdev); } void enetc_tsn_pf_deinit(struct net_device *netdev) { tsn_port_unregister(netdev); + enetc_tsn_netdev_list_del(netdev); } diff --git a/drivers/net/ethernet/freescale/fec.h b/drivers/net/ethernet/freescale/fec.h index 0465d98795d7..6775eeb3c684 100644 --- a/drivers/net/ethernet/freescale/fec.h +++ b/drivers/net/ethernet/freescale/fec.h @@ -22,6 +22,7 @@ #include #include #include +#include #if defined(CONFIG_M523x) || defined(CONFIG_M527x) || defined(CONFIG_M528x) || \ defined(CONFIG_M520x) || defined(CONFIG_M532x) || defined(CONFIG_ARM) || \ @@ -333,30 +334,30 @@ struct bufdesc_ex { (IDLE_SLOPE_2 & IDLE_SLOPE_MASK)) #define RCMR_MATCHEN (0x1 << 16) #define RCMR_CMP_CFG(v, n) (((v) & 0x7) << (n << 2)) +#ifdef CONFIG_AVB_SUPPORT +#define SR_CLASS_A_PRIORITY 3 +#define SR_CLASS_B_PRIORITY 2 +#define RCMR_CMP_1 (RCMR_CMP_CFG(SR_CLASS_A_PRIORITY, 0) | \ + RCMR_CMP_CFG(SR_CLASS_A_PRIORITY, 1) | \ + RCMR_CMP_CFG(SR_CLASS_A_PRIORITY, 2) | \ + RCMR_CMP_CFG(SR_CLASS_A_PRIORITY, 3)) +#define RCMR_CMP_2 (RCMR_CMP_CFG(SR_CLASS_B_PRIORITY, 0) | \ + RCMR_CMP_CFG(SR_CLASS_B_PRIORITY, 1) | \ + RCMR_CMP_CFG(SR_CLASS_B_PRIORITY, 2) | \ + RCMR_CMP_CFG(SR_CLASS_B_PRIORITY, 3)) +#else #define RCMR_CMP_1 (RCMR_CMP_CFG(0, 0) | RCMR_CMP_CFG(1, 1) | \ RCMR_CMP_CFG(2, 2) | RCMR_CMP_CFG(3, 3)) #define RCMR_CMP_2 (RCMR_CMP_CFG(4, 0) | RCMR_CMP_CFG(5, 1) | \ RCMR_CMP_CFG(6, 2) | RCMR_CMP_CFG(7, 3)) +#endif #define RCMR_CMP(X) (((X) == 1) ? RCMR_CMP_1 : RCMR_CMP_2) #define FEC_TX_BD_FTYPE(X) (((X) & 0xf) << 20) -/* The number of Tx and Rx buffers. These are allocated from the page - * pool. The code may assume these are power of two, so it it best - * to keep them that size. - * We don't need to allocate pages for the transmitter. We just use - * the skbuffer directly. - */ +#define FEC_RX_FLUSH(X) (1 << ((X) + 3)) -#define FEC_ENET_XDP_HEADROOM (XDP_PACKET_HEADROOM) -#define FEC_ENET_RX_PAGES 256 -#define FEC_ENET_RX_FRSIZE (PAGE_SIZE - FEC_ENET_XDP_HEADROOM \ - - SKB_DATA_ALIGN(sizeof(struct skb_shared_info))) -#define FEC_ENET_RX_FRPPG (PAGE_SIZE / FEC_ENET_RX_FRSIZE) -#define RX_RING_SIZE (FEC_ENET_RX_FRPPG * FEC_ENET_RX_PAGES) -#define FEC_ENET_TX_FRSIZE 2048 -#define FEC_ENET_TX_FRPPG (PAGE_SIZE / FEC_ENET_TX_FRSIZE) -#define TX_RING_SIZE 512 /* Must be power of two */ -#define TX_RING_MOD_MASK 511 /* for this to work */ +#define FEC_TX_SCHEME_CB 0x0 /* Credit based */ +#define FEC_TX_SCHEME_RR 0x1 /* Round-robin */ #define BD_ENET_RX_INT 0x00800000 #define BD_ENET_RX_PTP ((ushort)0x0400) @@ -401,9 +402,14 @@ struct bufdesc_ex { #define FEC_ITR_EN (0x1 << 31) #define FEC_ITR_ICFT(X) (((X) & 0xff) << 20) #define FEC_ITR_ICTT(X) ((X) & 0xffff) -#define FEC_ITR_ICFT_DEFAULT 200 /* Set 200 frame count threshold */ #define FEC_ITR_ICTT_DEFAULT 1000 /* Set 1000us timer threshold */ +#ifdef CONFIG_AVB_SUPPORT +#define FEC_ITR_ICFT_DEFAULT 50 /* Keep it coherent with FEC_TX_RING_SIZE/FEC_RX_RING_SIZE */ +#else +#define FEC_ITR_ICFT_DEFAULT 200 /* Set 200 frame count threshold */ +#endif + #define FEC_VLAN_TAG_LEN 0x04 #define FEC_ETHTYPE_LEN 0x02 @@ -556,20 +562,31 @@ struct fec_tx_buffer { struct fec_enet_priv_tx_q { struct bufdesc_prop bd; - unsigned char *tx_bounce[TX_RING_SIZE]; - struct fec_tx_buffer tx_buf[TX_RING_SIZE]; + struct bufdesc *dirty_tx; + struct fec_tx_buffer tx_buf[FEC_TX_RING_SIZE]; + + unsigned int tx_bounce_size; +#ifdef CONFIG_AVB_SUPPORT + unsigned int tx_index; + unsigned char *tx_bounce[FEC_TX_RING_SIZE + 32]; +#else + unsigned char *tx_bounce[FEC_TX_RING_SIZE]; +#endif unsigned short tx_stop_threshold; unsigned short tx_wake_threshold; - struct bufdesc *dirty_tx; char *tso_hdrs; dma_addr_t tso_hdrs_dma; + +#ifdef CONFIG_AVB_SUPPORT + unsigned long tx_idle_slope; +#endif }; struct fec_enet_priv_rx_q { struct bufdesc_prop bd; - struct fec_enet_priv_txrx_info rx_skb_info[RX_RING_SIZE]; + struct fec_enet_priv_txrx_info rx_skb_info[FEC_RX_RING_SIZE]; /* page_pool */ struct page_pool *page_pool; @@ -619,6 +636,12 @@ struct fec_enet_private { unsigned int total_tx_ring_size; unsigned int total_rx_ring_size; +#ifdef CONFIG_AVB_SUPPORT + const struct avb_ops *avb; + void *avb_data; + unsigned int avb_enabled; + __ETHTOOL_DECLARE_LINK_MODE_MASK(phy_advertising); +#endif struct platform_device *pdev; int dev_id; @@ -650,7 +673,7 @@ struct fec_enet_private { struct ptp_clock *ptp_clock; struct ptp_clock_info ptp_caps; unsigned long last_overflow_check; - spinlock_t tmreg_lock; + raw_spinlock_t tmreg_lock; struct cyclecounter cc; struct timecounter tc; int rx_hwtstamp_filter; @@ -658,6 +681,13 @@ struct fec_enet_private { u32 cycle_speed; int hwts_rx_en; int hwts_tx_en; + + /* Transmit and receive latency, depending on link speed, for + * packets timestamps in ns + */ + u32 rx_tstamp_latency; + u32 tx_tstamp_latency; + struct delayed_work time_keep; struct regulator *reg_phy; struct fec_stop_mode_gpr stop_gpr; @@ -695,9 +725,35 @@ struct fec_enet_private { /* XDP BPF Program */ struct bpf_prog *xdp_prog; + /* Configured rx/tx timestamps delays for different link speeds + * to compensate for FEC-PHY latency in ns + */ + u32 rx_delay_100; + u32 tx_delay_100; + u32 rx_delay_1000; + u32 tx_delay_1000; + +#ifdef CONFIG_AVB_SUPPORT + int rec_channel; + int rec_enable; +#endif + u64 ethtool_stats[]; }; +#ifdef CONFIG_AVB_SUPPORT +#define FEC_MAX_RATE 400 /* Mbps */ +#define FEC_MAX_RATE_HAS_AVB 1000 /* Mbps */ + +static inline int fec_max_rate(struct fec_enet_private *fep) +{ + int max_rate = (fep->quirks & FEC_QUIRK_HAS_AVB) ? FEC_MAX_RATE_HAS_AVB : FEC_MAX_RATE; + return min(max_rate, fep->speed); +} + +#define IDLE_SLOPE_DIVISOR 512 +#endif + void fec_ptp_init(struct platform_device *pdev, int irq_idx); void fec_ptp_stop(struct platform_device *pdev); void fec_ptp_start_cyclecounter(struct net_device *ndev); diff --git a/drivers/net/ethernet/freescale/fec_ecat.c b/drivers/net/ethernet/freescale/fec_ecat.c new file mode 100644 index 000000000000..86eb3341105b --- /dev/null +++ b/drivers/net/ethernet/freescale/fec_ecat.c @@ -0,0 +1,3008 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2023 NXP + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "fec_ecat.h" + + +#define DRIVER_NAME "fec_ecat" + +/* Pause frame feild and FIFO threshold */ +#define FEC_ENET_FCE (1 << 5) +#define FEC_ENET_RSEM_V 0x84 +#define FEC_ENET_RSFL_V 16 +#define FEC_ENET_RAEM_V 0x8 +#define FEC_ENET_RAFL_V 0x8 +#define FEC_ENET_OPD_V 0xFFF0 +#define FEC_MDIO_PM_TIMEOUT 100 /* ms */ + +struct fec_devinfo { + u32 quirks; +}; + +static const struct fec_devinfo fec_imx25_info = { + .quirks = FEC_QUIRK_USE_GASKET | FEC_QUIRK_MIB_CLEAR | + FEC_QUIRK_HAS_FRREG, +}; + +static const struct fec_devinfo fec_imx27_info = { + .quirks = FEC_QUIRK_MIB_CLEAR | FEC_QUIRK_HAS_FRREG, +}; + +static const struct fec_devinfo fec_imx28_info = { + .quirks = FEC_QUIRK_ENET_MAC | FEC_QUIRK_SWAP_FRAME | + FEC_QUIRK_SINGLE_MDIO | FEC_QUIRK_HAS_RACC | + FEC_QUIRK_HAS_FRREG | FEC_QUIRK_CLEAR_SETUP_MII | + FEC_QUIRK_NO_HARD_RESET, +}; + +static const struct fec_devinfo fec_imx6q_info = { + .quirks = FEC_QUIRK_ENET_MAC | FEC_QUIRK_HAS_GBIT | + FEC_QUIRK_HAS_BUFDESC_EX | FEC_QUIRK_HAS_CSUM | + FEC_QUIRK_HAS_VLAN | FEC_QUIRK_ERR006358 | + FEC_QUIRK_HAS_RACC | FEC_QUIRK_CLEAR_SETUP_MII | + FEC_QUIRK_HAS_PMQOS, +}; + +static const struct fec_devinfo fec_mvf600_info = { + .quirks = FEC_QUIRK_ENET_MAC | FEC_QUIRK_HAS_RACC, +}; + +static const struct fec_devinfo fec_imx6x_info = { + .quirks = FEC_QUIRK_ENET_MAC | FEC_QUIRK_HAS_GBIT | + FEC_QUIRK_HAS_BUFDESC_EX | FEC_QUIRK_HAS_CSUM | + FEC_QUIRK_HAS_VLAN | FEC_QUIRK_HAS_AVB | + FEC_QUIRK_ERR007885 | FEC_QUIRK_BUG_CAPTURE | + FEC_QUIRK_HAS_RACC | FEC_QUIRK_HAS_COALESCE | + FEC_QUIRK_CLEAR_SETUP_MII | FEC_QUIRK_HAS_MULTI_QUEUES, +}; + +static const struct fec_devinfo fec_imx6ul_info = { + .quirks = FEC_QUIRK_ENET_MAC | FEC_QUIRK_HAS_GBIT | + FEC_QUIRK_HAS_BUFDESC_EX | FEC_QUIRK_HAS_CSUM | + FEC_QUIRK_HAS_VLAN | FEC_QUIRK_ERR007885 | + FEC_QUIRK_BUG_CAPTURE | FEC_QUIRK_HAS_RACC | + FEC_QUIRK_HAS_COALESCE | FEC_QUIRK_CLEAR_SETUP_MII, +}; + +static const struct fec_devinfo fec_imx8mq_info = { + .quirks = FEC_QUIRK_ENET_MAC | FEC_QUIRK_HAS_GBIT | + FEC_QUIRK_HAS_BUFDESC_EX | FEC_QUIRK_HAS_CSUM | + FEC_QUIRK_HAS_VLAN | FEC_QUIRK_HAS_AVB | + FEC_QUIRK_ERR007885 | FEC_QUIRK_BUG_CAPTURE | + FEC_QUIRK_HAS_RACC | FEC_QUIRK_HAS_COALESCE | + FEC_QUIRK_CLEAR_SETUP_MII | FEC_QUIRK_HAS_MULTI_QUEUES | + FEC_QUIRK_HAS_EEE | FEC_QUIRK_WAKEUP_FROM_INT2, +}; + +static const struct fec_devinfo fec_imx8qm_info = { + .quirks = FEC_QUIRK_ENET_MAC | FEC_QUIRK_HAS_GBIT | + FEC_QUIRK_HAS_BUFDESC_EX | FEC_QUIRK_HAS_CSUM | + FEC_QUIRK_HAS_VLAN | FEC_QUIRK_HAS_AVB | + FEC_QUIRK_ERR007885 | FEC_QUIRK_BUG_CAPTURE | + FEC_QUIRK_HAS_RACC | FEC_QUIRK_HAS_COALESCE | + FEC_QUIRK_CLEAR_SETUP_MII | FEC_QUIRK_HAS_MULTI_QUEUES | + FEC_QUIRK_DELAYED_CLKS_SUPPORT, +}; + +static const struct fec_devinfo fec_s32v234_info = { + .quirks = FEC_QUIRK_ENET_MAC | FEC_QUIRK_HAS_GBIT | + FEC_QUIRK_HAS_BUFDESC_EX | FEC_QUIRK_HAS_CSUM | + FEC_QUIRK_HAS_VLAN | FEC_QUIRK_HAS_AVB | + FEC_QUIRK_ERR007885 | FEC_QUIRK_BUG_CAPTURE, +}; + +static struct platform_device_id fec_devtype[] = { + { + /* keep it for coldfire */ + .name = DRIVER_NAME, + .driver_data = 0, + }, { + .name = "imx25-fec", + .driver_data = (kernel_ulong_t)&fec_imx25_info, + }, { + .name = "imx27-fec", + .driver_data = (kernel_ulong_t)&fec_imx27_info, + }, { + .name = "imx28-fec", + .driver_data = (kernel_ulong_t)&fec_imx28_info, + }, { + .name = "imx6q-fec", + .driver_data = (kernel_ulong_t)&fec_imx6q_info, + }, { + .name = "mvf600-fec", + .driver_data = (kernel_ulong_t)&fec_mvf600_info, + }, { + .name = "imx6sx-fec", + .driver_data = (kernel_ulong_t)&fec_imx6x_info, + }, { + .name = "imx6ul-fec", + .driver_data = (kernel_ulong_t)&fec_imx6ul_info, + }, { + .name = "imx8mq-fec", + .driver_data = (kernel_ulong_t)&fec_imx8mq_info, + }, { + .name = "imx8qm-fec", + .driver_data = (kernel_ulong_t)&fec_imx8qm_info, + }, { + .name = "s32v234-fec", + .driver_data = (kernel_ulong_t)&fec_s32v234_info, + }, { + /* sentinel */ + } +}; +MODULE_DEVICE_TABLE(platform, fec_devtype); + +enum imx_fec_type { + IMX25_FEC = 1, /* runs on i.mx25/50/53 */ + IMX27_FEC, /* runs on i.mx27/35/51 */ + IMX28_FEC, + IMX6Q_FEC, + MVF600_FEC, + IMX6SX_FEC, + IMX6UL_FEC, + IMX8MQ_FEC, + IMX8QM_FEC, + S32V234_FEC, +}; + +static const struct of_device_id fec_dt_ids[] = { + { .compatible = "fsl,imx25-fec-ecat", .data = &fec_devtype[IMX25_FEC], }, + { .compatible = "fsl,imx27-fec-ecat", .data = &fec_devtype[IMX27_FEC], }, + { .compatible = "fsl,imx28-fec-ecat", .data = &fec_devtype[IMX28_FEC], }, + { .compatible = "fsl,imx6q-fec-ecat", .data = &fec_devtype[IMX6Q_FEC], }, + { .compatible = "fsl,mvf600-fec-ecat", .data = &fec_devtype[MVF600_FEC], }, + { .compatible = "fsl,imx6sx-fec-ecat", .data = &fec_devtype[IMX6SX_FEC], }, + { .compatible = "fsl,imx6ul-fec-ecat", .data = &fec_devtype[IMX6UL_FEC], }, + { .compatible = "fsl,imx8mq-fec-ecat", .data = &fec_devtype[IMX8MQ_FEC], }, + { .compatible = "fsl,imx8qm-fec-ecat", .data = &fec_devtype[IMX8QM_FEC], }, + { .compatible = "fsl,s32v234-fec-ecat", .data = &fec_devtype[S32V234_FEC], }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(of, fec_dt_ids); + +static unsigned char macaddr[ETH_ALEN]; +module_param_array(macaddr, byte, NULL, 0); +MODULE_PARM_DESC(macaddr, "FEC Ethernet MAC address"); + +#if defined(CONFIG_M5272) +/* + * Some hardware gets it MAC address out of local flash memory. + * if this is non-zero then assume it is the address to get MAC from. + */ +#if defined(CONFIG_NETtel) +#define FEC_FLASHMAC 0xf0006006 +#elif defined(CONFIG_GILBARCONAP) || defined(CONFIG_SCALES) +#define FEC_FLASHMAC 0xf0006000 +#elif defined(CONFIG_CANCam) +#define FEC_FLASHMAC 0xf0020000 +#elif defined (CONFIG_M5272C3) +#define FEC_FLASHMAC (0xffe04000 + 4) +#elif defined(CONFIG_MOD5272) +#define FEC_FLASHMAC 0xffc0406b +#else +#define FEC_FLASHMAC 0 +#endif +#endif /* CONFIG_M5272 */ + +/* The FEC stores dest/src/type/vlan, data, and checksum for receive packets. + * + * 2048 byte skbufs are allocated. However, alignment requirements + * varies between FEC variants. Worst case is 64, so round down by 64. + */ +#define PKT_MAXBUF_SIZE (round_down(2048 - 64, 64)) +#define PKT_MINBUF_SIZE 64 + +/* FEC receive acceleration */ +#define FEC_RACC_IPDIS (1 << 1) +#define FEC_RACC_PRODIS (1 << 2) +#define FEC_RACC_SHIFT16 BIT(7) +#define FEC_RACC_OPTIONS (FEC_RACC_IPDIS | FEC_RACC_PRODIS) + +/* MIB Control Register */ +#define FEC_MIB_CTRLSTAT_DISABLE BIT(31) + +/* + * The 5270/5271/5280/5282/532x RX control register also contains maximum frame + * size bits. Other FEC hardware does not, so we need to take that into + * account when setting it. + */ +#if defined(CONFIG_M523x) || defined(CONFIG_M527x) || defined(CONFIG_M528x) || \ + defined(CONFIG_M520x) || defined(CONFIG_M532x) || defined(CONFIG_ARM) || \ + defined(CONFIG_ARM64) +#define OPT_FRAME_SIZE (PKT_MAXBUF_SIZE << 16) +#else +#define OPT_FRAME_SIZE 0 +#endif + +/* FEC MII MMFR bits definition */ +#define FEC_MMFR_ST (1 << 30) +#define FEC_MMFR_ST_C45 (0) +#define FEC_MMFR_OP_READ (2 << 28) +#define FEC_MMFR_OP_READ_C45 (3 << 28) +#define FEC_MMFR_OP_WRITE (1 << 28) +#define FEC_MMFR_OP_ADDR_WRITE (0) +#define FEC_MMFR_PA(v) ((v & 0x1f) << 23) +#define FEC_MMFR_RA(v) ((v & 0x1f) << 18) +#define FEC_MMFR_TA (2 << 16) +#define FEC_MMFR_DATA(v) (v & 0xffff) +/* FEC ECR bits definition */ +#define FEC_ECR_MAGICEN (1 << 2) +#define FEC_ECR_SLEEP (1 << 3) + +#define FEC_MII_TIMEOUT 30000 /* us */ + +/* Transmitter timeout */ +#define TX_TIMEOUT (2 * HZ) + +#define FEC_PAUSE_FLAG_AUTONEG 0x1 +#define FEC_PAUSE_FLAG_ENABLE 0x2 +#define FEC_WOL_HAS_MAGIC_PACKET (0x1 << 0) +#define FEC_WOL_FLAG_ENABLE (0x1 << 1) +#define FEC_WOL_FLAG_SLEEP_ON (0x1 << 2) + +static void set_multicast_list(struct net_device *ndev); +static int mii_cnt; + +static struct bufdesc *fec_enet_get_nextdesc(struct bufdesc *bdp, + struct bufdesc_prop *bd) +{ + return (bdp >= bd->last) ? bd->base + : (struct bufdesc *)(((void *)bdp) + bd->dsize); +} + +static struct bufdesc *fec_enet_get_prevdesc(struct bufdesc *bdp, + struct bufdesc_prop *bd) +{ + return (bdp <= bd->base) ? bd->last + : (struct bufdesc *)(((void *)bdp) - bd->dsize); +} + +static int fec_enet_get_bd_index(struct bufdesc *bdp, + struct bufdesc_prop *bd) +{ + return ((const char *)bdp - (const char *)bd->base) >> bd->dsize_log2; +} + +static void swap_buffer(void *bufaddr, int len) +{ + int i; + unsigned int *buf = bufaddr; + + for (i = 0; i < len; i += 4, buf++) + swab32s(buf); +} + +static void swap_buffer2(void *dst_buf, void *src_buf, int len) +{ + int i; + unsigned int *src = src_buf; + unsigned int *dst = dst_buf; + + for (i = 0; i < len; i += 4, src++, dst++) + *dst = swab32p(src); +} + +static void fec_dump(struct net_device *ndev) +{ + struct fec_enet_private *fep = netdev_priv(ndev); + struct bufdesc *bdp; + struct fec_enet_priv_tx_q *txq; + int index = 0; + + netdev_info(ndev, "TX ring dump\n"); + pr_info("Nr SC addr len SKB\n"); + + txq = fep->tx_queue; + bdp = txq->bd.base; + + do { + pr_info("%3u %c%c 0x%04x 0x%08x %4u %p\n", + index, + bdp == txq->bd.cur ? 'S' : ' ', + bdp == txq->dirty_tx ? 'H' : ' ', + fec16_to_cpu(bdp->cbd_sc), + fec32_to_cpu(bdp->cbd_bufaddr), + fec16_to_cpu(bdp->cbd_datlen), + txq->tx_skbuff[index]); + bdp = fec_enet_get_nextdesc(bdp, &txq->bd); + index++; + } while (bdp != txq->bd.base); +} + + +static int fec_ecat_txq_submit_buff(struct fec_enet_priv_tx_q *txq, + void __user *buff, size_t len, struct net_device *ndev) +{ + struct fec_enet_private *fep = netdev_priv(ndev); + struct bufdesc *bdp, *last_bdp; + void *bufaddr; + unsigned short status; + unsigned short buflen; + unsigned int index; + struct sk_buff *skb; + + bdp = txq->bd.cur; + last_bdp = bdp; + status = fec16_to_cpu(bdp->cbd_sc); + status &= ~BD_ENET_TX_STATS; + + index = fec_enet_get_bd_index(last_bdp, &txq->bd); + skb = txq->tx_skbuff[index]; + bufaddr = skb->data; + buflen = len; + copy_from_user(skb->data, buff, len); + bdp->cbd_datlen = cpu_to_fec16(buflen); + /* Push the data cache so the CPM does not get stale memory data. */ + dma_sync_single_for_device(&fep->pdev->dev, + fec32_to_cpu(bdp->cbd_bufaddr), + buflen, + DMA_TO_DEVICE); + + status |= (BD_ENET_TX_INTR | BD_ENET_TX_LAST); + + wmb(); + + /* Send it on its way. Tell FEC it's ready, interrupt when done, + * it's the last BD of the frame, and to put the CRC on the end. + */ + status |= (BD_ENET_TX_READY | BD_ENET_TX_TC); + bdp->cbd_sc = cpu_to_fec16(status); + + /* If this was the last BD in the ring, start at the beginning again. */ + bdp = fec_enet_get_nextdesc(last_bdp, &txq->bd); + + wmb(); + txq->bd.cur = bdp; + + /* Trigger transmission start */ + if (!(fep->quirks & FEC_QUIRK_ERR007885) || + !readl(txq->bd.reg_desc_active) || + !readl(txq->bd.reg_desc_active) || + !readl(txq->bd.reg_desc_active) || + !readl(txq->bd.reg_desc_active)) + writel(0, txq->bd.reg_desc_active); + + return 0; +} + +static void fec_ecat_tx_queue(struct net_device *ndev) +{ + struct fec_enet_private *fep = netdev_priv(ndev); + struct bufdesc *bdp; + unsigned short status; + struct fec_enet_priv_tx_q *txq = fep->tx_queue; + int index = 0; + + /* get next bdp of dirty_tx */ + bdp = txq->dirty_tx; + + /* get next bdp of dirty_tx */ + bdp = fec_enet_get_nextdesc(bdp, &txq->bd); + + while (bdp != READ_ONCE(txq->bd.cur)) { + /* Order the load of bd.cur and cbd_sc */ + rmb(); + status = fec16_to_cpu(READ_ONCE(bdp->cbd_sc)); + if (status & BD_ENET_TX_READY) + break; + + /* Check for errors. */ + if (status & (BD_ENET_TX_HB | BD_ENET_TX_LC | + BD_ENET_TX_RL | BD_ENET_TX_UN | + BD_ENET_TX_CSL)) { + ndev->stats.tx_errors++; + } else { + ndev->stats.tx_packets++; + } + + wmb(); + txq->dirty_tx = bdp; + + /* Update pointer to next buffer descriptor to be transmitted */ + bdp = fec_enet_get_nextdesc(bdp, &txq->bd); + } + + /* ERR006358: Keep the transmitter going */ + if (bdp != txq->bd.cur && + readl(txq->bd.reg_desc_active) == 0) + writel(0, txq->bd.reg_desc_active); +} + +static int fec_ecat_fast_xmit(struct net_device *ndev, void __user *buff, size_t len) +{ + struct fec_enet_private *fep = netdev_priv(ndev); + int ret; + + if (!mutex_trylock(&fep->fast_ndev_lock)) { + return -EBUSY; + } + + ret = fec_ecat_txq_submit_buff(fep->tx_queue, buff, len, ndev); + fec_ecat_tx_queue(ndev); + + mutex_unlock(&fep->fast_ndev_lock); + + return NETDEV_TX_OK; +} + +/* Init RX n TX buffer descriptors + */ +static void fec_enet_bd_init(struct net_device *dev) +{ + struct fec_enet_private *fep = netdev_priv(dev); + struct fec_enet_priv_tx_q *txq = fep->tx_queue; + struct fec_enet_priv_rx_q *rxq = fep->rx_queue; + struct bufdesc *bdp; + int i; + struct sk_buff *skb; + + /* Initialize the receive buffer descriptors. */ + bdp = rxq->bd.base; + + for (i = 0; i < rxq->bd.ring_size; i++) { + /* Initialize the BD for every fragment in the page. */ + if (bdp->cbd_bufaddr) + bdp->cbd_sc = cpu_to_fec16(BD_ENET_RX_EMPTY); + else + bdp->cbd_sc = cpu_to_fec16(0); + bdp = fec_enet_get_nextdesc(bdp, &rxq->bd); + } + + /* Set the last buffer to wrap */ + bdp = fec_enet_get_prevdesc(bdp, &rxq->bd); + bdp->cbd_sc |= cpu_to_fec16(BD_SC_WRAP); + rxq->bd.cur = rxq->bd.base; + + /* ...and the same for transmit */ + bdp = txq->bd.base; + txq->bd.cur = bdp; + + for (i = 0; i < txq->bd.ring_size; i++) { + bdp->cbd_sc = cpu_to_fec16(0); + bdp = fec_enet_get_nextdesc(bdp, &txq->bd); + } + + /* Set the last buffer to wrap */ + bdp = fec_enet_get_prevdesc(bdp, &txq->bd); + bdp->cbd_sc |= cpu_to_fec16(BD_SC_WRAP); + txq->dirty_tx = bdp; +} + +/* + * This function is called to start or restart the FEC during a link + * change, transmit timeout, or to reconfigure the FEC. The network + * packet processing for this device must be stopped before this call. + */ +static void +fec_restart(struct net_device *ndev) +{ + struct fec_enet_private *fep = netdev_priv(ndev); + u32 temp_mac[2]; + u32 rcntl = OPT_FRAME_SIZE | 0x04; + u32 ecntl = FEC_ENET_ETHEREN; /* ETHEREN */ + + /* Always use disable MAC instead of MAC reset to: + * - Keep the ENET counter running + * - Avoid dead system bus for SoCs using the ENET-AXI bus + * and not the AHB bus, like the i.MX6SX + */ + writel(0, fep->hwp + FEC_ECNTRL); + + /* + * enet-mac reset will reset mac address registers too, + * so need to reconfigure it. + */ + memcpy(&temp_mac, ndev->dev_addr, ETH_ALEN); + writel((__force u32)cpu_to_be32(temp_mac[0]), + fep->hwp + FEC_ADDR_LOW); + writel((__force u32)cpu_to_be32(temp_mac[1]), + fep->hwp + FEC_ADDR_HIGH); + + /* Clear any outstanding interrupt, except MDIO. */ + writel((0xffffffff & ~FEC_ENET_MII), fep->hwp + FEC_IEVENT); + + fec_enet_bd_init(ndev); + + writel(fep->rx_queue->bd.dma, fep->hwp + FEC_R_DES_START(0)); + writel(PKT_MAXBUF_SIZE, fep->hwp + FEC_R_BUFF_SIZE(0)); + + writel(fep->tx_queue->bd.dma, fep->hwp + FEC_X_DES_START(0)); + + if (fep->quirks & FEC_QUIRK_HAS_AVB) + writel(FEC_RX_FLUSH(0) | FEC_TX_SCHEME_CB, + fep->hwp + FEC_QOS_SCHEME); + + /* Enable MII mode */ + if (fep->full_duplex == DUPLEX_FULL) { + /* FD enable */ + writel(0x04, fep->hwp + FEC_X_CNTRL); + } else { + /* No Rcv on Xmit */ + rcntl |= 0x02; + writel(0x0, fep->hwp + FEC_X_CNTRL); + } + + /* Set MII speed */ + writel(fep->phy_speed, fep->hwp + FEC_MII_SPEED); + +#if !defined(CONFIG_M5272) + if (fep->quirks & FEC_QUIRK_HAS_RACC) { + u32 val = readl(fep->hwp + FEC_RACC); + + /* align IP header */ + val |= FEC_RACC_SHIFT16; + if (fep->csum_flags & FLAG_RX_CSUM_ENABLED) + /* set RX checksum */ + val |= FEC_RACC_OPTIONS; + else + val &= ~FEC_RACC_OPTIONS; + writel(val, fep->hwp + FEC_RACC); + writel(PKT_MAXBUF_SIZE, fep->hwp + FEC_FTRL); + } +#endif + + /* + * The phy interface and speed need to get configured + * differently on enet-mac. + */ + if (fep->quirks & FEC_QUIRK_ENET_MAC) { + /* Enable flow control and length check */ + rcntl |= 0x40000000 | 0x00000020; + + /* RGMII, RMII or MII */ + if (fep->phy_interface == PHY_INTERFACE_MODE_RGMII || + fep->phy_interface == PHY_INTERFACE_MODE_RGMII_ID || + fep->phy_interface == PHY_INTERFACE_MODE_RGMII_RXID || + fep->phy_interface == PHY_INTERFACE_MODE_RGMII_TXID) + rcntl |= (1 << 6); + else if (fep->phy_interface == PHY_INTERFACE_MODE_RMII) + rcntl |= (1 << 8); + else + rcntl &= ~(1 << 8); + + /* 1G, 100M or 10M */ + if (ndev->phydev) { + if (ndev->phydev->speed == SPEED_1000) + ecntl |= (1 << 5); + else if (ndev->phydev->speed == SPEED_100) + rcntl &= ~(1 << 9); + else + rcntl |= (1 << 9); + } + } else { +#ifdef FEC_MIIGSK_ENR + if (fep->quirks & FEC_QUIRK_USE_GASKET) { + u32 cfgr; + /* disable the gasket and wait */ + writel(0, fep->hwp + FEC_MIIGSK_ENR); + while (readl(fep->hwp + FEC_MIIGSK_ENR) & 4) + udelay(1); + + /* + * configure the gasket: + * RMII, 50 MHz, no loopback, no echo + * MII, 25 MHz, no loopback, no echo + */ + cfgr = (fep->phy_interface == PHY_INTERFACE_MODE_RMII) + ? BM_MIIGSK_CFGR_RMII : BM_MIIGSK_CFGR_MII; + if (ndev->phydev && ndev->phydev->speed == SPEED_10) + cfgr |= BM_MIIGSK_CFGR_FRCONT_10M; + writel(cfgr, fep->hwp + FEC_MIIGSK_CFGR); + + /* re-enable the gasket */ + writel(2, fep->hwp + FEC_MIIGSK_ENR); + } +#endif + } + +#if !defined(CONFIG_M5272) + /* enable pause frame*/ + if ((fep->pause_flag & FEC_PAUSE_FLAG_ENABLE) || + ((fep->pause_flag & FEC_PAUSE_FLAG_AUTONEG) && + ndev->phydev && ndev->phydev->pause)) { + rcntl |= FEC_ENET_FCE; + + /* set FIFO threshold parameter to reduce overrun */ + writel(FEC_ENET_RSEM_V, fep->hwp + FEC_R_FIFO_RSEM); + writel(FEC_ENET_RSFL_V, fep->hwp + FEC_R_FIFO_RSFL); + writel(FEC_ENET_RAEM_V, fep->hwp + FEC_R_FIFO_RAEM); + writel(FEC_ENET_RAFL_V, fep->hwp + FEC_R_FIFO_RAFL); + + /* OPD */ + writel(FEC_ENET_OPD_V, fep->hwp + FEC_OPD); + } else { + rcntl &= ~FEC_ENET_FCE; + } +#endif /* !defined(CONFIG_M5272) */ + + writel(rcntl, fep->hwp + FEC_R_CNTRL); + + /* Setup multicast filter. */ + set_multicast_list(ndev); +#ifndef CONFIG_M5272 + writel(0, fep->hwp + FEC_HASH_TABLE_HIGH); + writel(0, fep->hwp + FEC_HASH_TABLE_LOW); +#endif + + if (fep->quirks & FEC_QUIRK_ENET_MAC) { + /* enable ENET endian swap */ + ecntl |= (1 << 8); + /* enable ENET store and forward mode */ + writel(1 << 8, fep->hwp + FEC_X_WMRK); + } + + if (fep->quirks & FEC_QUIRK_DELAYED_CLKS_SUPPORT && + fep->rgmii_txc_dly) + ecntl |= FEC_ENET_TXC_DLY; + if (fep->quirks & FEC_QUIRK_DELAYED_CLKS_SUPPORT && + fep->rgmii_rxc_dly) + ecntl |= FEC_ENET_RXC_DLY; + +#ifndef CONFIG_M5272 + /* Enable the MIB statistic event counters */ + writel(0 << 31, fep->hwp + FEC_MIB_CTRLSTAT); +#endif + + /* And last, enable the transmit and receive processing */ + writel(ecntl, fep->hwp + FEC_ECNTRL); + writel(0, fep->rx_queue->bd.reg_desc_active); + + writel(0, fep->hwp + FEC_IMASK); +} + +static int fec_enet_ipc_handle_init(struct fec_enet_private *fep) +{ + if (!(of_machine_is_compatible("fsl,imx8qm") || + of_machine_is_compatible("fsl,imx8qxp") || + of_machine_is_compatible("fsl,imx8dxl"))) + return 0; + + return imx_scu_get_handle(&fep->ipc_handle); +} + +static void fec_enet_ipg_stop_set(struct fec_enet_private *fep, bool enabled) +{ + struct device_node *np = fep->pdev->dev.of_node; + u32 rsrc_id, val; + int idx; + + if (!np || !fep->ipc_handle) + return; + + idx = of_alias_get_id(np, "ethernet"); + if (idx < 0) + idx = 0; + rsrc_id = idx ? IMX_SC_R_ENET_1 : IMX_SC_R_ENET_0; + + val = enabled ? 1 : 0; + imx_sc_misc_set_control(fep->ipc_handle, rsrc_id, IMX_SC_C_IPG_STOP, val); +} + +static void fec_enet_stop_mode(struct fec_enet_private *fep, bool enabled) +{ + struct fec_platform_data *pdata = fep->pdev->dev.platform_data; + struct fec_stop_mode_gpr *stop_gpr = &fep->stop_gpr; + + if (stop_gpr->gpr) { + if (enabled) + regmap_update_bits(stop_gpr->gpr, stop_gpr->reg, + BIT(stop_gpr->bit), + BIT(stop_gpr->bit)); + else + regmap_update_bits(stop_gpr->gpr, stop_gpr->reg, + BIT(stop_gpr->bit), 0); + } else if (pdata && pdata->sleep_mode_enable) { + pdata->sleep_mode_enable(enabled); + } else { + fec_enet_ipg_stop_set(fep, enabled); + } +} + +static inline void fec_irqs_disable(struct net_device *ndev) +{ + struct fec_enet_private *fep = netdev_priv(ndev); + + writel(0, fep->hwp + FEC_IMASK); +} + +static void +fec_stop(struct net_device *ndev) +{ + struct fec_enet_private *fep = netdev_priv(ndev); + u32 rmii_mode = readl(fep->hwp + FEC_R_CNTRL) & (1 << 8); + + /* We cannot expect a graceful transmit stop without link !!! */ + if (fep->link) { + writel(1, fep->hwp + FEC_X_CNTRL); /* Graceful transmit stop */ + udelay(10); + if (!(readl(fep->hwp + FEC_IEVENT) & FEC_ENET_GRA)) + netdev_err(ndev, "Graceful transmit stop did not complete!\n"); + } + + writel(0, fep->hwp + FEC_ECNTRL); + + writel(0, fep->hwp + FEC_IMASK); + writel(fep->phy_speed, fep->hwp + FEC_MII_SPEED); +} + + +static void +fec_timeout(struct net_device *ndev, unsigned int txqueue) +{ + struct fec_enet_private *fep = netdev_priv(ndev); + + fec_dump(ndev); + + ndev->stats.tx_errors++; + + schedule_work(&fep->tx_timeout_work); +} + +static void fec_enet_timeout_work(struct work_struct *work) +{ + struct fec_enet_private *fep = + container_of(work, struct fec_enet_private, tx_timeout_work); + struct net_device *ndev = fep->netdev; + + if (netif_device_present(ndev) || netif_running(ndev)) { + mutex_lock(&fep->fast_ndev_lock); + + fec_restart(ndev); + mutex_unlock(&fep->fast_ndev_lock); + } +} + +// must be powers of 2 +#define MAX_TX_BUF (64) +#define MAX_RX_BUF (64) + +static int fec_ecat_recv_from_queue(struct net_device *ndev, void __user *buff, size_t len, struct sockaddr __user *addr, int *addr_len) +{ + struct fec_enet_private *fep = netdev_priv(ndev); + struct fec_enet_priv_rx_q *rxq = fep->rx_queue; + struct bufdesc *bdp; + unsigned short status; + struct sk_buff *skb = NULL; + ushort pkt_len; + __u8 *data; + int recv_len = 0; + int index = 0; + bool need_swap = fep->quirks & FEC_QUIRK_SWAP_FRAME; + int ret = 0; + +#ifdef CONFIG_M532x + flush_cache_all(); +#endif + + /* First, grab all of the stats for the incoming packet. + * These get messed up if we get called due to a busy condition. + */ + bdp = rxq->bd.cur; + + while (!((status = fec16_to_cpu(bdp->cbd_sc)) & BD_ENET_RX_EMPTY)) { + + writel(FEC_ENET_RXF_GET(0), fep->hwp + FEC_IEVENT); + + /* Check for errors. */ + status ^= BD_ENET_RX_LAST; + if (status & (BD_ENET_RX_LG | BD_ENET_RX_SH | BD_ENET_RX_NO | + BD_ENET_RX_CR | BD_ENET_RX_OV | BD_ENET_RX_LAST | + BD_ENET_RX_CL)) { + ndev->stats.rx_errors++; + if (status & BD_ENET_RX_OV) { + /* FIFO overrun */ + ndev->stats.rx_fifo_errors++; + goto rx_processing_done; + } + if (status & (BD_ENET_RX_LG | BD_ENET_RX_SH + | BD_ENET_RX_LAST)) { + /* Frame too long or too short. */ + ndev->stats.rx_length_errors++; + if (status & BD_ENET_RX_LAST) + netdev_err(ndev, "rcv is not +last\n"); + } + if (status & BD_ENET_RX_CR) /* CRC Error */ + ndev->stats.rx_crc_errors++; + /* Report late collisions as a frame error. */ + if (status & (BD_ENET_RX_NO | BD_ENET_RX_CL)) + ndev->stats.rx_frame_errors++; + goto rx_processing_done; + } + + /* Process the incoming frame. */ + ndev->stats.rx_packets++; + pkt_len = fec16_to_cpu(bdp->cbd_datlen); + ndev->stats.rx_bytes += pkt_len; + + index = fec_enet_get_bd_index(bdp, &rxq->bd); + skb = rxq->rx_skbuff[index]; + + prefetch(skb->data - NET_IP_ALIGN); + dma_sync_single_for_cpu(&fep->pdev->dev, + fec32_to_cpu(bdp->cbd_bufaddr), + FEC_ENET_RX_FRSIZE - fep->rx_align, + DMA_FROM_DEVICE); + + pkt_len -= 6; + data = skb->data; +#if !defined(CONFIG_M5272) + if (fep->quirks & FEC_QUIRK_HAS_RACC) + data += 2; +#endif + if (data[12] ==0x88 && data[13] ==0xa4) { + len = len < pkt_len? len : pkt_len; + if (!need_swap) { + copy_to_user(buff, data, len); + } + else { + swap_buffer2(buff, data, len); + } + if (addr != NULL) { + struct sockaddr_ll *sll = (struct sockaddr_ll *)addr; + sll->sll_hatype = ndev->type; + sll->sll_ifindex = ndev->ifindex; + } + recv_len = len; + } + dma_sync_single_for_device(&fep->pdev->dev, + fec32_to_cpu(bdp->cbd_bufaddr), + FEC_ENET_RX_FRSIZE - fep->rx_align, + DMA_FROM_DEVICE); +rx_processing_done: + /* Clear the status flags for this buffer */ + status &= ~BD_ENET_RX_STATS; + + /* Mark the buffer empty */ + status |= BD_ENET_RX_EMPTY; + + /* Make sure the updates to rest of the descriptor are + * performed before transferring ownership. + */ + wmb(); + bdp->cbd_sc = cpu_to_fec16(status); + + /* Update BD pointer to next entry */ + bdp = fec_enet_get_nextdesc(bdp, &rxq->bd); + + /* Doing this here will keep the FEC running while we process + * incoming frames. On a heavily loaded network, we should be + * able to keep up at the expense of system resources. + */ + writel(0, rxq->bd.reg_desc_active); + if (recv_len) { + break; + } + } + rxq->bd.cur = bdp; + return recv_len; +} + +static int fec_ecat_fast_recv(struct net_device *ndev, void __user *buff, size_t len, struct sockaddr __user *addr, int *addr_len) +{ + struct fec_enet_private *fep = netdev_priv(ndev); + int ret; + + if (!mutex_trylock(&fep->fast_ndev_lock)) { + return -EBUSY; + } + + ret = fec_ecat_recv_from_queue(ndev, buff, len, addr, addr_len); + + mutex_unlock(&fep->fast_ndev_lock); + return ret; +} + +/* ------------------------------------------------------------------------- */ +static int fec_get_mac(struct net_device *ndev) +{ + struct fec_enet_private *fep = netdev_priv(ndev); + unsigned char *iap, tmpaddr[ETH_ALEN]; + int ret; + + /* + * try to get mac address in following order: + * + * 1) module parameter via kernel command line in form + * fec.macaddr=0x00,0x04,0x9f,0x01,0x30,0xe0 + */ + iap = macaddr; + + /* + * 2) from device tree data + */ + if (!is_valid_ether_addr(iap)) { + struct device_node *np = fep->pdev->dev.of_node; + if (np) { + ret = of_get_mac_address(np, tmpaddr); + if (!ret) + iap = tmpaddr; + else if (ret == -EPROBE_DEFER) + return ret; + } + } + + /* + * 3) from flash or fuse (via platform data) + */ + if (!is_valid_ether_addr(iap)) { +#ifdef CONFIG_M5272 + if (FEC_FLASHMAC) + iap = (unsigned char *)FEC_FLASHMAC; +#else + struct fec_platform_data *pdata = dev_get_platdata(&fep->pdev->dev); + + if (pdata) + iap = (unsigned char *)&pdata->mac; +#endif + } + + /* + * 4) FEC mac registers set by bootloader + */ + if (!is_valid_ether_addr(iap)) { + *((__be32 *) &tmpaddr[0]) = + cpu_to_be32(readl(fep->hwp + FEC_ADDR_LOW)); + *((__be16 *) &tmpaddr[4]) = + cpu_to_be16(readl(fep->hwp + FEC_ADDR_HIGH) >> 16); + iap = &tmpaddr[0]; + } + + /* + * 5) random mac address + */ + if (!is_valid_ether_addr(iap)) { + /* Report it and use a random ethernet address instead */ + dev_err(&fep->pdev->dev, "Invalid MAC address: %pM\n", iap); + eth_hw_addr_random(ndev); + dev_info(&fep->pdev->dev, "Using random MAC address: %pM\n", + ndev->dev_addr); + return 0; + } + + /* Adjust MAC if using macaddr */ + eth_hw_addr_gen(ndev, iap, iap == macaddr ? fep->dev_id : 0); + + return 0; +} + +/* ------------------------------------------------------------------------- */ + +/* + * Phy section + */ +static void fec_enet_adjust_link(struct net_device *ndev) +{ + struct fec_enet_private *fep = netdev_priv(ndev); + struct phy_device *phy_dev = ndev->phydev; + int status_change = 0; + + /* + * If the netdev is down, or is going down, we're not interested + * in link state events, so just mark our idea of the link as down + * and ignore the event. + */ + if (!netif_running(ndev) || !netif_device_present(ndev)) { + fep->link = 0; + } else if (phy_dev->link) { + if (!fep->link) { + fep->link = phy_dev->link; + status_change = 1; + } + + if (fep->full_duplex != phy_dev->duplex) { + fep->full_duplex = phy_dev->duplex; + status_change = 1; + } + + if (phy_dev->speed != fep->speed) { + fep->speed = phy_dev->speed; + status_change = 1; + } + + switch (fep->speed) { + case SPEED_100: + fep->rx_tstamp_latency = fep->rx_delay_100; + fep->tx_tstamp_latency = fep->tx_delay_100; + break; + case SPEED_1000: + fep->rx_tstamp_latency = fep->rx_delay_1000; + fep->tx_tstamp_latency = fep->tx_delay_1000; + break; + default: + fep->rx_tstamp_latency = 0; + fep->tx_tstamp_latency = 0; + } + + /* if any of the above changed restart the FEC */ + if (status_change) { + mutex_lock(&fep->fast_ndev_lock); + fec_restart(ndev); + mutex_unlock(&fep->fast_ndev_lock); + } + } else { + if (fep->link) { + mutex_lock(&fep->fast_ndev_lock); + fec_stop(ndev); + mutex_unlock(&fep->fast_ndev_lock); + fep->link = phy_dev->link; + status_change = 1; + } + } + + if (status_change) + phy_print_status(phy_dev); +} + +static int fec_enet_mdio_wait(struct fec_enet_private *fep) +{ + uint ievent; + int ret; + + ret = readl_poll_timeout_atomic(fep->hwp + FEC_IEVENT, ievent, + ievent & FEC_ENET_MII, 2, 30000); + + if (!ret) + writel(FEC_ENET_MII, fep->hwp + FEC_IEVENT); + + return ret; +} + +static int fec_enet_mdio_read(struct mii_bus *bus, int mii_id, int regnum) +{ + struct fec_enet_private *fep = bus->priv; + struct device *dev = &fep->pdev->dev; + int ret = 0, frame_start, frame_addr, frame_op; + bool is_c45 = !!(regnum & MII_ADDR_C45); + + ret = pm_runtime_resume_and_get(dev); + if (ret < 0) + return ret; + + if (is_c45) { + frame_start = FEC_MMFR_ST_C45; + + /* write address */ + frame_addr = (regnum >> 16); + writel(frame_start | FEC_MMFR_OP_ADDR_WRITE | + FEC_MMFR_PA(mii_id) | FEC_MMFR_RA(frame_addr) | + FEC_MMFR_TA | (regnum & 0xFFFF), + fep->hwp + FEC_MII_DATA); + + /* wait for end of transfer */ + ret = fec_enet_mdio_wait(fep); + if (ret) { + netdev_err(fep->netdev, "MDIO address write timeout\n"); + goto out; + } + + frame_op = FEC_MMFR_OP_READ_C45; + + } else { + /* C22 read */ + frame_op = FEC_MMFR_OP_READ; + frame_start = FEC_MMFR_ST; + frame_addr = regnum; + } + + /* start a read op */ + writel(frame_start | frame_op | + FEC_MMFR_PA(mii_id) | FEC_MMFR_RA(frame_addr) | + FEC_MMFR_TA, fep->hwp + FEC_MII_DATA); + + /* wait for end of transfer */ + ret = fec_enet_mdio_wait(fep); + if (ret) { + netdev_err(fep->netdev, "MDIO read timeout\n"); + goto out; + } + + ret = FEC_MMFR_DATA(readl(fep->hwp + FEC_MII_DATA)); + +out: + pm_runtime_mark_last_busy(dev); + pm_runtime_put_autosuspend(dev); + + return ret; +} + +static int fec_enet_mdio_write(struct mii_bus *bus, int mii_id, int regnum, + u16 value) +{ + struct fec_enet_private *fep = bus->priv; + struct device *dev = &fep->pdev->dev; + int ret, frame_start, frame_addr; + bool is_c45 = !!(regnum & MII_ADDR_C45); + + ret = pm_runtime_resume_and_get(dev); + if (ret < 0) + return ret; + + if (is_c45) { + frame_start = FEC_MMFR_ST_C45; + + /* write address */ + frame_addr = (regnum >> 16); + writel(frame_start | FEC_MMFR_OP_ADDR_WRITE | + FEC_MMFR_PA(mii_id) | FEC_MMFR_RA(frame_addr) | + FEC_MMFR_TA | (regnum & 0xFFFF), + fep->hwp + FEC_MII_DATA); + + /* wait for end of transfer */ + ret = fec_enet_mdio_wait(fep); + if (ret) { + netdev_err(fep->netdev, "MDIO address write timeout\n"); + goto out; + } + } else { + /* C22 write */ + frame_start = FEC_MMFR_ST; + frame_addr = regnum; + } + + /* start a write op */ + writel(frame_start | FEC_MMFR_OP_WRITE | + FEC_MMFR_PA(mii_id) | FEC_MMFR_RA(frame_addr) | + FEC_MMFR_TA | FEC_MMFR_DATA(value), + fep->hwp + FEC_MII_DATA); + + /* wait for end of transfer */ + ret = fec_enet_mdio_wait(fep); + if (ret) + netdev_err(fep->netdev, "MDIO write timeout\n"); + +out: + pm_runtime_mark_last_busy(dev); + pm_runtime_put_autosuspend(dev); + + return ret; +} + +static void fec_enet_phy_reset_after_clk_enable(struct net_device *ndev) +{ + struct fec_enet_private *fep = netdev_priv(ndev); + struct phy_device *phy_dev = ndev->phydev; + + if (phy_dev) { + phy_reset_after_clk_enable(phy_dev); + } else if (fep->phy_node) { + /* + * If the PHY still is not bound to the MAC, but there is + * OF PHY node and a matching PHY device instance already, + * use the OF PHY node to obtain the PHY device instance, + * and then use that PHY device instance when triggering + * the PHY reset. + */ + phy_dev = of_phy_find_device(fep->phy_node); + phy_reset_after_clk_enable(phy_dev); + put_device(&phy_dev->mdio.dev); + } +} + +static int fec_enet_clk_enable(struct net_device *ndev, bool enable) +{ + struct fec_enet_private *fep = netdev_priv(ndev); + int ret; + + if (enable) { + ret = clk_prepare_enable(fep->clk_enet_out); + if (ret) + return ret; + + if (fep->clk_ptp) { + mutex_lock(&fep->ptp_clk_mutex); + ret = clk_prepare_enable(fep->clk_ptp); + if (ret) { + mutex_unlock(&fep->ptp_clk_mutex); + goto failed_clk_ptp; + } else { + fep->ptp_clk_on = true; + } + mutex_unlock(&fep->ptp_clk_mutex); + } + + ret = clk_prepare_enable(fep->clk_ref); + if (ret) + goto failed_clk_ref; + + ret = clk_prepare_enable(fep->clk_2x_txclk); + if (ret) + goto failed_clk_2x_txclk; + + fec_enet_phy_reset_after_clk_enable(ndev); + } else { + clk_disable_unprepare(fep->clk_enet_out); + if (fep->clk_ptp) { + mutex_lock(&fep->ptp_clk_mutex); + clk_disable_unprepare(fep->clk_ptp); + fep->ptp_clk_on = false; + mutex_unlock(&fep->ptp_clk_mutex); + } + clk_disable_unprepare(fep->clk_ref); + clk_disable_unprepare(fep->clk_2x_txclk); + } + + return 0; + +failed_clk_2x_txclk: + if (fep->clk_ref) + clk_disable_unprepare(fep->clk_ref); +failed_clk_ref: + if (fep->clk_ptp) { + mutex_lock(&fep->ptp_clk_mutex); + clk_disable_unprepare(fep->clk_ptp); + fep->ptp_clk_on = false; + mutex_unlock(&fep->ptp_clk_mutex); + } +failed_clk_ptp: + clk_disable_unprepare(fep->clk_enet_out); + + return ret; +} + +static int fec_enet_parse_rgmii_delay(struct fec_enet_private *fep, + struct device_node *np) +{ + u32 rgmii_tx_delay, rgmii_rx_delay; + + /* For rgmii tx internal delay, valid values are 0ps and 2000ps */ + if (!of_property_read_u32(np, "tx-internal-delay-ps", &rgmii_tx_delay)) { + if (rgmii_tx_delay != 0 && rgmii_tx_delay != 2000) { + dev_err(&fep->pdev->dev, "The only allowed RGMII TX delay values are: 0ps, 2000ps"); + return -EINVAL; + } else if (rgmii_tx_delay == 2000) { + fep->rgmii_txc_dly = true; + } + } + + /* For rgmii rx internal delay, valid values are 0ps and 2000ps */ + if (!of_property_read_u32(np, "rx-internal-delay-ps", &rgmii_rx_delay)) { + if (rgmii_rx_delay != 0 && rgmii_rx_delay != 2000) { + dev_err(&fep->pdev->dev, "The only allowed RGMII RX delay values are: 0ps, 2000ps"); + return -EINVAL; + } else if (rgmii_rx_delay == 2000) { + fep->rgmii_rxc_dly = true; + } + } + + return 0; +} + +static int fec_restore_mii_bus(struct net_device *ndev) +{ + struct fec_enet_private *fep = netdev_priv(ndev); + int ret; + + ret = pm_runtime_get_sync(&fep->pdev->dev); + if (ret < 0) + return ret; + + writel(0xffc00000, fep->hwp + FEC_IEVENT); + writel(fep->phy_speed, fep->hwp + FEC_MII_SPEED); + writel(FEC_ENET_MII, fep->hwp + FEC_IMASK); + writel(FEC_ENET_ETHEREN, fep->hwp + FEC_ECNTRL); + + pm_runtime_mark_last_busy(&fep->pdev->dev); + pm_runtime_put_autosuspend(&fep->pdev->dev); + + return 0; +} + +static int fec_enet_mii_probe(struct net_device *ndev) +{ + struct fec_enet_private *fep = netdev_priv(ndev); + struct phy_device *phy_dev = NULL; + char mdio_bus_id[MII_BUS_ID_SIZE]; + char phy_name[MII_BUS_ID_SIZE + 3]; + int phy_id; + int dev_id = fep->dev_id; + + if (fep->phy_node) { + phy_dev = of_phy_connect(ndev, fep->phy_node, + &fec_enet_adjust_link, 0, + fep->phy_interface); + if (!phy_dev) { + netdev_err(ndev, "Unable to connect to phy\n"); + return -ENODEV; + } + } else { + /* check for attached phy */ + for (phy_id = 0; (phy_id < PHY_MAX_ADDR); phy_id++) { + if (!mdiobus_is_registered_device(fep->mii_bus, phy_id)) + continue; + if (dev_id--) + continue; + strlcpy(mdio_bus_id, fep->mii_bus->id, MII_BUS_ID_SIZE); + break; + } + + if (phy_id >= PHY_MAX_ADDR) { + netdev_info(ndev, "no PHY, assuming direct connection to switch\n"); + strlcpy(mdio_bus_id, "fixed-0", MII_BUS_ID_SIZE); + phy_id = 0; + } + + snprintf(phy_name, sizeof(phy_name), + PHY_ID_FMT, mdio_bus_id, phy_id); + phy_dev = phy_connect(ndev, phy_name, &fec_enet_adjust_link, + fep->phy_interface); + } + + if (IS_ERR(phy_dev)) { + netdev_err(ndev, "could not attach to PHY\n"); + return PTR_ERR(phy_dev); + } + + /* mask with MAC supported features */ + if (fep->quirks & FEC_QUIRK_HAS_GBIT) { + phy_set_max_speed(phy_dev, 1000); + phy_remove_link_mode(phy_dev, + ETHTOOL_LINK_MODE_1000baseT_Half_BIT); +#if !defined(CONFIG_M5272) + phy_support_sym_pause(phy_dev); +#endif + } + else + phy_set_max_speed(phy_dev, 100); + + fep->link = 0; + fep->full_duplex = 0; + + phy_dev->mac_managed_pm = 1; + + phy_attached_info(phy_dev); + + return 0; +} + +static int fec_enet_mii_init(struct platform_device *pdev) +{ + static struct mii_bus *fec0_mii_bus; + static bool *fec_mii_bus_share; + struct net_device *ndev = platform_get_drvdata(pdev); + struct fec_enet_private *fep = netdev_priv(ndev); + bool suppress_preamble = false; + struct device_node *node; + int err = -ENXIO; + u32 mii_speed, holdtime; + u32 bus_freq; + + /* + * The i.MX28 dual fec interfaces are not equal. + * Here are the differences: + * + * - fec0 supports MII & RMII modes while fec1 only supports RMII + * - fec0 acts as the 1588 time master while fec1 is slave + * - external phys can only be configured by fec0 + * + * That is to say fec1 can not work independently. It only works + * when fec0 is working. The reason behind this design is that the + * second interface is added primarily for Switch mode. + * + * Because of the last point above, both phys are attached on fec0 + * mdio interface in board design, and need to be configured by + * fec0 mii_bus. + */ + if ((fep->quirks & FEC_QUIRK_SINGLE_MDIO) && fep->dev_id > 0) { + /* fec1 uses fec0 mii_bus */ + if (mii_cnt && fec0_mii_bus) { + fep->mii_bus = fec0_mii_bus; + *fec_mii_bus_share = true; + mii_cnt++; + return 0; + } + return -ENOENT; + } + + bus_freq = 2500000; /* 2.5MHz by default */ + node = of_get_child_by_name(pdev->dev.of_node, "mdio"); + if (node) { + of_property_read_u32(node, "clock-frequency", &bus_freq); + suppress_preamble = of_property_read_bool(node, + "suppress-preamble"); + } + + /* + * Set MII speed (= clk_get_rate() / 2 * phy_speed) + * + * The formula for FEC MDC is 'ref_freq / (MII_SPEED x 2)' while + * for ENET-MAC is 'ref_freq / ((MII_SPEED + 1) x 2)'. The i.MX28 + * Reference Manual has an error on this, and gets fixed on i.MX6Q + * document. + */ + mii_speed = DIV_ROUND_UP(clk_get_rate(fep->clk_ipg), bus_freq * 2); + if (fep->quirks & FEC_QUIRK_ENET_MAC) + mii_speed--; + if (mii_speed > 63) { + dev_err(&pdev->dev, + "fec clock (%lu) too fast to get right mii speed\n", + clk_get_rate(fep->clk_ipg)); + err = -EINVAL; + goto err_out; + } + + /* + * The i.MX28 and i.MX6 types have another filed in the MSCR (aka + * MII_SPEED) register that defines the MDIO output hold time. Earlier + * versions are RAZ there, so just ignore the difference and write the + * register always. + * The minimal hold time according to IEE802.3 (clause 22) is 10 ns. + * HOLDTIME + 1 is the number of clk cycles the fec is holding the + * output. + * The HOLDTIME bitfield takes values between 0 and 7 (inclusive). + * Given that ceil(clkrate / 5000000) <= 64, the calculation for + * holdtime cannot result in a value greater than 3. + */ + holdtime = DIV_ROUND_UP(clk_get_rate(fep->clk_ipg), 100000000) - 1; + + fep->phy_speed = mii_speed << 1 | holdtime << 8; + + if (suppress_preamble) + fep->phy_speed |= BIT(7); + + if (fep->quirks & FEC_QUIRK_CLEAR_SETUP_MII) { + /* Clear MMFR to avoid to generate MII event by writing MSCR. + * MII event generation condition: + * - writing MSCR: + * - mmfr[31:0]_not_zero & mscr[7:0]_is_zero & + * mscr_reg_data_in[7:0] != 0 + * - writing MMFR: + * - mscr[7:0]_not_zero + */ + writel(0, fep->hwp + FEC_MII_DATA); + } + + writel(fep->phy_speed, fep->hwp + FEC_MII_SPEED); + + /* Clear any pending transaction complete indication */ + writel(FEC_ENET_MII, fep->hwp + FEC_IEVENT); + + fep->mii_bus = mdiobus_alloc(); + if (fep->mii_bus == NULL) { + err = -ENOMEM; + goto err_out; + } + + fep->mii_bus->name = "fec_enet_mii_bus"; + fep->mii_bus->read = fec_enet_mdio_read; + fep->mii_bus->write = fec_enet_mdio_write; + snprintf(fep->mii_bus->id, MII_BUS_ID_SIZE, "%s-%x", + pdev->name, fep->dev_id + 1); + fep->mii_bus->priv = fep; + fep->mii_bus->parent = &pdev->dev; + + err = of_mdiobus_register(fep->mii_bus, node); + if (err) + goto err_out_free_mdiobus; + of_node_put(node); + + mii_cnt++; + + /* save fec0 mii_bus */ + if (fep->quirks & FEC_QUIRK_SINGLE_MDIO) { + fec0_mii_bus = fep->mii_bus; + fec_mii_bus_share = &fep->mii_bus_share; + } + + return 0; + +err_out_free_mdiobus: + mdiobus_free(fep->mii_bus); +err_out: + of_node_put(node); + return err; +} + +static void fec_enet_mii_remove(struct fec_enet_private *fep) +{ + if (--mii_cnt == 0) { + mdiobus_unregister(fep->mii_bus); + mdiobus_free(fep->mii_bus); + } +} + +static void fec_enet_get_drvinfo(struct net_device *ndev, + struct ethtool_drvinfo *info) +{ + struct fec_enet_private *fep = netdev_priv(ndev); + + strlcpy(info->driver, fep->pdev->dev.driver->name, + sizeof(info->driver)); + strlcpy(info->bus_info, dev_name(&ndev->dev), sizeof(info->bus_info)); +} + +static int fec_enet_get_regs_len(struct net_device *ndev) +{ + struct fec_enet_private *fep = netdev_priv(ndev); + struct resource *r; + int s = 0; + + r = platform_get_resource(fep->pdev, IORESOURCE_MEM, 0); + if (r) + s = resource_size(r); + + return s; +} + +/* List of registers that can be safety be read to dump them with ethtool */ +#if defined(CONFIG_M523x) || defined(CONFIG_M527x) || defined(CONFIG_M528x) || \ + defined(CONFIG_M520x) || defined(CONFIG_M532x) || defined(CONFIG_ARM) || \ + defined(CONFIG_ARM64) || defined(CONFIG_COMPILE_TEST) +static __u32 fec_enet_register_version = 2; +static u32 fec_enet_register_offset[] = { + FEC_IEVENT, FEC_IMASK, FEC_R_DES_ACTIVE_0, FEC_X_DES_ACTIVE_0, + FEC_ECNTRL, FEC_MII_DATA, FEC_MII_SPEED, FEC_MIB_CTRLSTAT, FEC_R_CNTRL, + FEC_X_CNTRL, FEC_ADDR_LOW, FEC_ADDR_HIGH, FEC_OPD, FEC_TXIC0, FEC_TXIC1, + FEC_TXIC2, FEC_RXIC0, FEC_RXIC1, FEC_RXIC2, FEC_HASH_TABLE_HIGH, + FEC_HASH_TABLE_LOW, FEC_GRP_HASH_TABLE_HIGH, FEC_GRP_HASH_TABLE_LOW, + FEC_X_WMRK, FEC_R_BOUND, FEC_R_FSTART, FEC_R_DES_START_1, + FEC_X_DES_START_1, FEC_R_BUFF_SIZE_1, FEC_R_DES_START_2, + FEC_X_DES_START_2, FEC_R_BUFF_SIZE_2, FEC_R_DES_START_0, + FEC_X_DES_START_0, FEC_R_BUFF_SIZE_0, FEC_R_FIFO_RSFL, FEC_R_FIFO_RSEM, + FEC_R_FIFO_RAEM, FEC_R_FIFO_RAFL, FEC_RACC, FEC_RCMR_1, FEC_RCMR_2, + FEC_DMA_CFG_1, FEC_DMA_CFG_2, FEC_R_DES_ACTIVE_1, FEC_X_DES_ACTIVE_1, + FEC_R_DES_ACTIVE_2, FEC_X_DES_ACTIVE_2, FEC_QOS_SCHEME, + RMON_T_DROP, RMON_T_PACKETS, RMON_T_BC_PKT, RMON_T_MC_PKT, + RMON_T_CRC_ALIGN, RMON_T_UNDERSIZE, RMON_T_OVERSIZE, RMON_T_FRAG, + RMON_T_JAB, RMON_T_COL, RMON_T_P64, RMON_T_P65TO127, RMON_T_P128TO255, + RMON_T_P256TO511, RMON_T_P512TO1023, RMON_T_P1024TO2047, + RMON_T_P_GTE2048, RMON_T_OCTETS, + IEEE_T_DROP, IEEE_T_FRAME_OK, IEEE_T_1COL, IEEE_T_MCOL, IEEE_T_DEF, + IEEE_T_LCOL, IEEE_T_EXCOL, IEEE_T_MACERR, IEEE_T_CSERR, IEEE_T_SQE, + IEEE_T_FDXFC, IEEE_T_OCTETS_OK, + RMON_R_PACKETS, RMON_R_BC_PKT, RMON_R_MC_PKT, RMON_R_CRC_ALIGN, + RMON_R_UNDERSIZE, RMON_R_OVERSIZE, RMON_R_FRAG, RMON_R_JAB, + RMON_R_RESVD_O, RMON_R_P64, RMON_R_P65TO127, RMON_R_P128TO255, + RMON_R_P256TO511, RMON_R_P512TO1023, RMON_R_P1024TO2047, + RMON_R_P_GTE2048, RMON_R_OCTETS, + IEEE_R_DROP, IEEE_R_FRAME_OK, IEEE_R_CRC, IEEE_R_ALIGN, IEEE_R_MACERR, + IEEE_R_FDXFC, IEEE_R_OCTETS_OK +}; +#else +static __u32 fec_enet_register_version = 1; +static u32 fec_enet_register_offset[] = { + FEC_ECNTRL, FEC_IEVENT, FEC_IMASK, FEC_IVEC, FEC_R_DES_ACTIVE_0, + FEC_R_DES_ACTIVE_1, FEC_R_DES_ACTIVE_2, FEC_X_DES_ACTIVE_0, + FEC_X_DES_ACTIVE_1, FEC_X_DES_ACTIVE_2, FEC_MII_DATA, FEC_MII_SPEED, + FEC_R_BOUND, FEC_R_FSTART, FEC_X_WMRK, FEC_X_FSTART, FEC_R_CNTRL, + FEC_MAX_FRM_LEN, FEC_X_CNTRL, FEC_ADDR_LOW, FEC_ADDR_HIGH, + FEC_GRP_HASH_TABLE_HIGH, FEC_GRP_HASH_TABLE_LOW, FEC_R_DES_START_0, + FEC_R_DES_START_1, FEC_R_DES_START_2, FEC_X_DES_START_0, + FEC_X_DES_START_1, FEC_X_DES_START_2, FEC_R_BUFF_SIZE_0, + FEC_R_BUFF_SIZE_1, FEC_R_BUFF_SIZE_2 +}; +#endif + +static void fec_enet_get_regs(struct net_device *ndev, + struct ethtool_regs *regs, void *regbuf) +{ + struct fec_enet_private *fep = netdev_priv(ndev); + u32 __iomem *theregs = (u32 __iomem *)fep->hwp; + struct device *dev = &fep->pdev->dev; + u32 *buf = (u32 *)regbuf; + u32 i, off; + int ret; + + ret = pm_runtime_resume_and_get(dev); + if (ret < 0) + return; + + regs->version = fec_enet_register_version; + + memset(buf, 0, regs->len); + + for (i = 0; i < ARRAY_SIZE(fec_enet_register_offset); i++) { + off = fec_enet_register_offset[i]; + + if ((off == FEC_R_BOUND || off == FEC_R_FSTART) && + !(fep->quirks & FEC_QUIRK_HAS_FRREG)) + continue; + + off >>= 2; + buf[off] = readl(&theregs[off]); + } + + pm_runtime_mark_last_busy(dev); + pm_runtime_put_autosuspend(dev); +} + + +#if !defined(CONFIG_M5272) + +static void fec_enet_get_pauseparam(struct net_device *ndev, + struct ethtool_pauseparam *pause) +{ + struct fec_enet_private *fep = netdev_priv(ndev); + + pause->autoneg = (fep->pause_flag & FEC_PAUSE_FLAG_AUTONEG) != 0; + pause->tx_pause = (fep->pause_flag & FEC_PAUSE_FLAG_ENABLE) != 0; + pause->rx_pause = pause->tx_pause; +} + +static int fec_enet_set_pauseparam(struct net_device *ndev, + struct ethtool_pauseparam *pause) +{ + struct fec_enet_private *fep = netdev_priv(ndev); + + if (!ndev->phydev) + return -ENODEV; + + if (pause->tx_pause != pause->rx_pause) { + netdev_info(ndev, + "hardware only support enable/disable both tx and rx"); + return -EINVAL; + } + + fep->pause_flag = 0; + + /* tx pause must be same as rx pause */ + fep->pause_flag |= pause->rx_pause ? FEC_PAUSE_FLAG_ENABLE : 0; + fep->pause_flag |= pause->autoneg ? FEC_PAUSE_FLAG_AUTONEG : 0; + + phy_set_sym_pause(ndev->phydev, pause->rx_pause, pause->tx_pause, + pause->autoneg); + + if (pause->autoneg) { + if (netif_running(ndev)) + fec_stop(ndev); + phy_start_aneg(ndev->phydev); + } + if (netif_running(ndev)) { + mutex_lock(&fep->fast_ndev_lock); + + netif_tx_lock_bh(ndev); + //netif_tx_wake_all_queues(ndev); + + mutex_unlock(&fep->fast_ndev_lock); + } + + return 0; +} + +static const struct fec_stat { + char name[ETH_GSTRING_LEN]; + u16 offset; +} fec_stats[] = { + /* RMON TX */ + { "tx_dropped", RMON_T_DROP }, + { "tx_packets", RMON_T_PACKETS }, + { "tx_broadcast", RMON_T_BC_PKT }, + { "tx_multicast", RMON_T_MC_PKT }, + { "tx_crc_errors", RMON_T_CRC_ALIGN }, + { "tx_undersize", RMON_T_UNDERSIZE }, + { "tx_oversize", RMON_T_OVERSIZE }, + { "tx_fragment", RMON_T_FRAG }, + { "tx_jabber", RMON_T_JAB }, + { "tx_collision", RMON_T_COL }, + { "tx_64byte", RMON_T_P64 }, + { "tx_65to127byte", RMON_T_P65TO127 }, + { "tx_128to255byte", RMON_T_P128TO255 }, + { "tx_256to511byte", RMON_T_P256TO511 }, + { "tx_512to1023byte", RMON_T_P512TO1023 }, + { "tx_1024to2047byte", RMON_T_P1024TO2047 }, + { "tx_GTE2048byte", RMON_T_P_GTE2048 }, + { "tx_octets", RMON_T_OCTETS }, + + /* IEEE TX */ + { "IEEE_tx_drop", IEEE_T_DROP }, + { "IEEE_tx_frame_ok", IEEE_T_FRAME_OK }, + { "IEEE_tx_1col", IEEE_T_1COL }, + { "IEEE_tx_mcol", IEEE_T_MCOL }, + { "IEEE_tx_def", IEEE_T_DEF }, + { "IEEE_tx_lcol", IEEE_T_LCOL }, + { "IEEE_tx_excol", IEEE_T_EXCOL }, + { "IEEE_tx_macerr", IEEE_T_MACERR }, + { "IEEE_tx_cserr", IEEE_T_CSERR }, + { "IEEE_tx_sqe", IEEE_T_SQE }, + { "IEEE_tx_fdxfc", IEEE_T_FDXFC }, + { "IEEE_tx_octets_ok", IEEE_T_OCTETS_OK }, + + /* RMON RX */ + { "rx_packets", RMON_R_PACKETS }, + { "rx_broadcast", RMON_R_BC_PKT }, + { "rx_multicast", RMON_R_MC_PKT }, + { "rx_crc_errors", RMON_R_CRC_ALIGN }, + { "rx_undersize", RMON_R_UNDERSIZE }, + { "rx_oversize", RMON_R_OVERSIZE }, + { "rx_fragment", RMON_R_FRAG }, + { "rx_jabber", RMON_R_JAB }, + { "rx_64byte", RMON_R_P64 }, + { "rx_65to127byte", RMON_R_P65TO127 }, + { "rx_128to255byte", RMON_R_P128TO255 }, + { "rx_256to511byte", RMON_R_P256TO511 }, + { "rx_512to1023byte", RMON_R_P512TO1023 }, + { "rx_1024to2047byte", RMON_R_P1024TO2047 }, + { "rx_GTE2048byte", RMON_R_P_GTE2048 }, + { "rx_octets", RMON_R_OCTETS }, + + /* IEEE RX */ + { "IEEE_rx_drop", IEEE_R_DROP }, + { "IEEE_rx_frame_ok", IEEE_R_FRAME_OK }, + { "IEEE_rx_crc", IEEE_R_CRC }, + { "IEEE_rx_align", IEEE_R_ALIGN }, + { "IEEE_rx_macerr", IEEE_R_MACERR }, + { "IEEE_rx_fdxfc", IEEE_R_FDXFC }, + { "IEEE_rx_octets_ok", IEEE_R_OCTETS_OK }, +}; + +#define FEC_STATS_SIZE (ARRAY_SIZE(fec_stats) * sizeof(u64)) + +static void fec_enet_update_ethtool_stats(struct net_device *dev) +{ + struct fec_enet_private *fep = netdev_priv(dev); + int i; + + for (i = 0; i < ARRAY_SIZE(fec_stats); i++) + fep->ethtool_stats[i] = readl(fep->hwp + fec_stats[i].offset); +} + +static void fec_enet_get_ethtool_stats(struct net_device *dev, + struct ethtool_stats *stats, u64 *data) +{ + struct fec_enet_private *fep = netdev_priv(dev); + + if (netif_running(dev)) + fec_enet_update_ethtool_stats(dev); + + memcpy(data, fep->ethtool_stats, FEC_STATS_SIZE); +} + +static void fec_enet_get_strings(struct net_device *netdev, + u32 stringset, u8 *data) +{ + int i; + switch (stringset) { + case ETH_SS_STATS: + for (i = 0; i < ARRAY_SIZE(fec_stats); i++) + memcpy(data + i * ETH_GSTRING_LEN, + fec_stats[i].name, ETH_GSTRING_LEN); + break; + case ETH_SS_TEST: + net_selftest_get_strings(data); + break; + } +} + +static int fec_enet_get_sset_count(struct net_device *dev, int sset) +{ + switch (sset) { + case ETH_SS_STATS: + return ARRAY_SIZE(fec_stats); + case ETH_SS_TEST: + return net_selftest_get_count(); + default: + return -EOPNOTSUPP; + } +} + +static void fec_enet_clear_ethtool_stats(struct net_device *dev) +{ + struct fec_enet_private *fep = netdev_priv(dev); + int i; + + /* Disable MIB statistics counters */ + writel(FEC_MIB_CTRLSTAT_DISABLE, fep->hwp + FEC_MIB_CTRLSTAT); + + for (i = 0; i < ARRAY_SIZE(fec_stats); i++) + writel(0, fep->hwp + fec_stats[i].offset); + + /* Don't disable MIB statistics counters */ + writel(0, fep->hwp + FEC_MIB_CTRLSTAT); +} + +#else /* !defined(CONFIG_M5272) */ +#define FEC_STATS_SIZE 0 +static inline void fec_enet_update_ethtool_stats(struct net_device *dev) +{ +} + +static inline void fec_enet_clear_ethtool_stats(struct net_device *dev) +{ +} +#endif /* !defined(CONFIG_M5272) */ + +/* ITR clock source is enet system clock (clk_ahb). + * TCTT unit is cycle_ns * 64 cycle + * So, the ICTT value = X us / (cycle_ns * 64) + */ +static int fec_enet_us_to_itr_clock(struct net_device *ndev, int us) +{ + struct fec_enet_private *fep = netdev_priv(ndev); + + return us * (fep->itr_clk_rate / 64000) / 1000; +} + + +/* LPI Sleep Ts count base on tx clk (clk_ref). + * The lpi sleep cnt value = X us / (cycle_ns). + */ +static int fec_enet_us_to_tx_cycle(struct net_device *ndev, int us) +{ + struct fec_enet_private *fep = netdev_priv(ndev); + + return us * (fep->clk_ref_rate / 1000) / 1000; +} + +static int fec_enet_eee_mode_set(struct net_device *ndev, bool enable) +{ + struct fec_enet_private *fep = netdev_priv(ndev); + struct ethtool_eee *p = &fep->eee; + unsigned int sleep_cycle, wake_cycle; + int ret = 0; + + if (enable) { + ret = phy_init_eee(ndev->phydev, 0); + if (ret) + return ret; + + sleep_cycle = fec_enet_us_to_tx_cycle(ndev, p->tx_lpi_timer); + wake_cycle = sleep_cycle; + } else { + sleep_cycle = 0; + wake_cycle = 0; + } + + p->tx_lpi_enabled = enable; + p->eee_enabled = enable; + p->eee_active = enable; + + writel(sleep_cycle, fep->hwp + FEC_LPI_SLEEP); + writel(wake_cycle, fep->hwp + FEC_LPI_WAKE); + + return 0; +} + +static int +fec_enet_get_eee(struct net_device *ndev, struct ethtool_eee *edata) +{ + struct fec_enet_private *fep = netdev_priv(ndev); + struct ethtool_eee *p = &fep->eee; + + if (!(fep->quirks & FEC_QUIRK_HAS_EEE)) + return -EOPNOTSUPP; + + if (!netif_running(ndev)) + return -ENETDOWN; + + edata->eee_enabled = p->eee_enabled; + edata->eee_active = p->eee_active; + edata->tx_lpi_timer = p->tx_lpi_timer; + edata->tx_lpi_enabled = p->tx_lpi_enabled; + + return phy_ethtool_get_eee(ndev->phydev, edata); +} + +static int +fec_enet_set_eee(struct net_device *ndev, struct ethtool_eee *edata) +{ + struct fec_enet_private *fep = netdev_priv(ndev); + struct ethtool_eee *p = &fep->eee; + int ret = 0; + + if (!(fep->quirks & FEC_QUIRK_HAS_EEE)) + return -EOPNOTSUPP; + + if (!netif_running(ndev)) + return -ENETDOWN; + + p->tx_lpi_timer = edata->tx_lpi_timer; + + if (!edata->eee_enabled || !edata->tx_lpi_enabled || + !edata->tx_lpi_timer) + ret = fec_enet_eee_mode_set(ndev, false); + else + ret = fec_enet_eee_mode_set(ndev, true); + + if (ret) + return ret; + + return phy_ethtool_set_eee(ndev->phydev, edata); +} + +static const struct ethtool_ops fec_enet_ethtool_ops = { + .supported_coalesce_params = ETHTOOL_COALESCE_USECS | + ETHTOOL_COALESCE_MAX_FRAMES, + .get_drvinfo = fec_enet_get_drvinfo, + .get_regs_len = fec_enet_get_regs_len, + .get_regs = fec_enet_get_regs, + .nway_reset = phy_ethtool_nway_reset, + .get_link = ethtool_op_get_link, +#ifndef CONFIG_M5272 + .get_strings = fec_enet_get_strings, + .get_ethtool_stats = fec_enet_get_ethtool_stats, + .get_sset_count = fec_enet_get_sset_count, +#endif + .get_eee = fec_enet_get_eee, + .set_eee = fec_enet_set_eee, + .get_link_ksettings = phy_ethtool_get_link_ksettings, + .set_link_ksettings = phy_ethtool_set_link_ksettings, + .self_test = net_selftest, +}; + +static int fec_enet_ioctl(struct net_device *ndev, struct ifreq *rq, int cmd) +{ + struct fec_enet_private *fep = netdev_priv(ndev); + struct phy_device *phydev = ndev->phydev; + + if (!netif_running(ndev)) + return -EINVAL; + + if (!phydev) + return -ENODEV; + + return phy_mii_ioctl(phydev, rq, cmd); +} + +static void fec_enet_free_buffers(struct net_device *ndev) +{ + struct fec_enet_private *fep = netdev_priv(ndev); + unsigned int i; + struct sk_buff *skb; + struct bufdesc *bdp; + struct fec_enet_priv_tx_q *txq = fep->tx_queue; + struct fec_enet_priv_rx_q *rxq = fep->rx_queue; + unsigned int q; + + bdp = rxq->bd.base; + for (i = 0; i < rxq->bd.ring_size; i++) { + skb = rxq->rx_skbuff[i]; + rxq->rx_skbuff[i] = NULL; + if (skb) { + dma_unmap_single(&fep->pdev->dev, + fec32_to_cpu(bdp->cbd_bufaddr), + FEC_ENET_RX_FRSIZE - fep->rx_align, + DMA_FROM_DEVICE); + dev_kfree_skb(skb); + } + } + + for (i = 0; i < txq->bd.ring_size; i++) { + skb = txq->tx_skbuff[i]; + txq->tx_skbuff[i] = NULL; + if (skb) { + dma_unmap_single(&fep->pdev->dev, + fec32_to_cpu(bdp->cbd_bufaddr), + FEC_ENET_RX_FRSIZE - fep->rx_align, + DMA_FROM_DEVICE); + dev_kfree_skb(skb); + } + } +} + +static void fec_enet_free_queue(struct net_device *ndev) +{ + struct fec_enet_private *fep = netdev_priv(ndev); + + if (fep->rx_queue) + kfree(fep->rx_queue); + if (fep->tx_queue) + kfree(fep->tx_queue); +} + +static int fec_enet_alloc_queue(struct net_device *ndev) +{ + struct fec_enet_private *fep = netdev_priv(ndev); + int ret = 0; + + fep->tx_queue = kzalloc(sizeof(*fep->tx_queue), GFP_KERNEL); + if (!fep->tx_queue) { + ret = -ENOMEM; + goto alloc_failed; + } + + fep->tx_queue->bd.ring_size = FEC_TX_RING_SIZE; + fep->total_tx_ring_size += fep->tx_queue->bd.ring_size; + + + fep->rx_queue = kzalloc(sizeof(*fep->rx_queue), GFP_KERNEL); + if (!fep->rx_queue) { + ret = -ENOMEM; + goto alloc_failed; + } + + fep->rx_queue->bd.ring_size = FEC_RX_RING_SIZE; + fep->total_rx_ring_size += fep->rx_queue->bd.ring_size; + return ret; + +alloc_failed: + fec_enet_free_queue(ndev); + return ret; +} + +static int fec_enet_alloc_buffers(struct net_device *ndev) +{ + struct fec_enet_private *fep = netdev_priv(ndev); + unsigned int i; + struct sk_buff *skb; + struct bufdesc *bdp; + int off; + struct fec_enet_priv_rx_q *rxq; + struct fec_enet_priv_tx_q *txq; + + rxq = fep->rx_queue; + txq = fep->tx_queue; + bdp = rxq->bd.base; + for (i = 0; i < rxq->bd.ring_size; i++) { + skb = netdev_alloc_skb(ndev, FEC_ENET_RX_FRSIZE); + if (!skb) + goto err_alloc; + + if ((off = ((unsigned long)skb->data) & fep->rx_align) > 0) + skb_reserve(skb, fep->rx_align + 1 - off); + + bdp->cbd_bufaddr = cpu_to_fec32(dma_map_single(&fep->pdev->dev, skb->data, FEC_ENET_RX_FRSIZE - fep->rx_align, DMA_FROM_DEVICE)); + if (dma_mapping_error(&fep->pdev->dev, fec32_to_cpu(bdp->cbd_bufaddr))) { + dev_kfree_skb(skb); + goto err_alloc; + } + + rxq->rx_skbuff[i] = skb; + bdp->cbd_sc = cpu_to_fec16(BD_ENET_RX_EMPTY); + + bdp = fec_enet_get_nextdesc(bdp, &rxq->bd); + } + + /* Set the last buffer to wrap. */ + bdp = fec_enet_get_prevdesc(bdp, &rxq->bd); + bdp->cbd_sc |= cpu_to_fec16(BD_SC_WRAP); + + bdp = txq->bd.base; + txq->bd.cur = bdp; + + for (i = 0; i < txq->bd.ring_size; i++) { + bdp->cbd_sc = cpu_to_fec16(0); + skb = netdev_alloc_skb(ndev, FEC_ENET_TX_FRSIZE); + if (!skb) + goto err_alloc; + + if ((off = ((unsigned long)skb->data) & fep->tx_align) > 0) + skb_reserve(skb, fep->tx_align + 1 - off); + + bdp->cbd_bufaddr = cpu_to_fec32(dma_map_single(&fep->pdev->dev, skb->data, FEC_ENET_TX_FRSIZE - fep->tx_align, DMA_TO_DEVICE)); + if (dma_mapping_error(&fep->pdev->dev, fec32_to_cpu(bdp->cbd_bufaddr))) { + goto err_alloc; + } + txq->tx_skbuff[i] = skb; + bdp = fec_enet_get_nextdesc(bdp, &txq->bd); + } + + /* Set the last buffer to wrap. */ + bdp = fec_enet_get_prevdesc(bdp, &txq->bd); + bdp->cbd_sc |= cpu_to_fec16(BD_SC_WRAP); + txq->dirty_tx = bdp; + return 0; + + err_alloc: + fec_enet_free_buffers(ndev); + return -ENOMEM; +} + +static int +fec_enet_open(struct net_device *ndev) +{ + struct fec_enet_private *fep = netdev_priv(ndev); + int ret; + bool reset_again; + int i; + + ret = pm_runtime_resume_and_get(&fep->pdev->dev); + if (ret < 0) + return ret; + + pinctrl_pm_select_default_state(&fep->pdev->dev); + + /* During the first fec_enet_open call the PHY isn't probed at this + * point. Therefore the phy_reset_after_clk_enable() call within + * fec_enet_clk_enable() fails. As we need this reset in order to be + * sure the PHY is working correctly we check if we need to reset again + * later when the PHY is probed + */ + if (ndev->phydev && ndev->phydev->drv) + reset_again = false; + else + reset_again = true; + + /* I should reset the ring buffers here, but I don't yet know + * a simple way to do that. + */ + + ret = fec_enet_alloc_buffers(ndev); + if (ret) + goto err_enet_alloc; + + /* Init MAC prior to mii bus probe */ + fec_restart(ndev); + + /* Call phy_reset_after_clk_enable() again if it failed during + * phy_reset_after_clk_enable() before because the PHY wasn't probed. + */ + if (reset_again) + fec_enet_phy_reset_after_clk_enable(ndev); + + /* Probe and connect to PHY when open the interface */ + ret = fec_enet_mii_probe(ndev); + if (ret) + goto err_enet_mii_probe; + + if (fep->quirks & FEC_QUIRK_ERR006687) + imx6q_cpuidle_fec_irqs_used(); + if (fep->quirks & FEC_QUIRK_HAS_PMQOS) + cpu_latency_qos_add_request(&fep->pm_qos_req, 0); + + phy_start(ndev->phydev); + mutex_unlock(&fep->fast_ndev_lock); + return 0; + +err_enet_mii_probe: + fec_enet_free_buffers(ndev); +err_enet_alloc: + pm_runtime_mark_last_busy(&fep->pdev->dev); + pm_runtime_put_autosuspend(&fep->pdev->dev); + if (!fep->mii_bus_share) + pinctrl_pm_select_sleep_state(&fep->pdev->dev); + + return ret; +} + +static int +fec_enet_close(struct net_device *ndev) +{ + struct fec_enet_private *fep = netdev_priv(ndev); + + phy_stop(ndev->phydev); + + if (netif_device_present(ndev)) { + netif_tx_disable(ndev); + fec_stop(ndev); + } + + phy_disconnect(ndev->phydev); + ndev->phydev = NULL; + + if (fep->quirks & FEC_QUIRK_ERR006687) + imx6q_cpuidle_fec_irqs_unused(); + + fec_enet_update_ethtool_stats(ndev); + + if (!fep->mii_bus_share) + pinctrl_pm_select_sleep_state(&fep->pdev->dev); + pm_runtime_mark_last_busy(&fep->pdev->dev); + pm_runtime_put_autosuspend(&fep->pdev->dev); + + fec_enet_free_buffers(ndev); + mutex_lock(&fep->fast_ndev_lock); + return 0; +} + +/* Set or clear the multicast filter for this adaptor. + * Skeleton taken from sunlance driver. + * The CPM Ethernet implementation allows Multicast as well as individual + * MAC address filtering. Some of the drivers check to make sure it is + * a group multicast address, and discard those that are not. I guess I + * will do the same for now, but just remove the test if you want + * individual filtering as well (do the upper net layers want or support + * this kind of feature?). + */ + +#define FEC_HASH_BITS 6 /* #bits in hash */ + +static void set_multicast_list(struct net_device *ndev) +{ + struct fec_enet_private *fep = netdev_priv(ndev); + struct netdev_hw_addr *ha; + unsigned int crc, tmp; + unsigned char hash; + unsigned int hash_high = 0, hash_low = 0; + + if (ndev->flags & IFF_PROMISC) { + tmp = readl(fep->hwp + FEC_R_CNTRL); + tmp |= 0x8; + writel(tmp, fep->hwp + FEC_R_CNTRL); + return; + } + + tmp = readl(fep->hwp + FEC_R_CNTRL); + tmp &= ~0x8; + writel(tmp, fep->hwp + FEC_R_CNTRL); + + if (ndev->flags & IFF_ALLMULTI) { + /* Catch all multicast addresses, so set the + * filter to all 1's + */ + writel(0xffffffff, fep->hwp + FEC_GRP_HASH_TABLE_HIGH); + writel(0xffffffff, fep->hwp + FEC_GRP_HASH_TABLE_LOW); + + return; + } + + /* Add the addresses in hash register */ + netdev_for_each_mc_addr(ha, ndev) { + /* calculate crc32 value of mac address */ + crc = ether_crc_le(ndev->addr_len, ha->addr); + + /* only upper 6 bits (FEC_HASH_BITS) are used + * which point to specific bit in the hash registers + */ + hash = (crc >> (32 - FEC_HASH_BITS)) & 0x3f; + + if (hash > 31) + hash_high |= 1 << (hash - 32); + else + hash_low |= 1 << hash; + } + + writel(hash_high, fep->hwp + FEC_GRP_HASH_TABLE_HIGH); + writel(hash_low, fep->hwp + FEC_GRP_HASH_TABLE_LOW); +} + +/* Set a MAC change in hardware. */ +static int +fec_set_mac_address(struct net_device *ndev, void *p) +{ + struct fec_enet_private *fep = netdev_priv(ndev); + struct sockaddr *addr = p; + + if (addr) { + if (!is_valid_ether_addr(addr->sa_data)) + return -EADDRNOTAVAIL; + memcpy(ndev->dev_addr, addr->sa_data, ndev->addr_len); + } + + /* Add netif status check here to avoid system hang in below case: + * ifconfig ethx down; ifconfig ethx hw ether xx:xx:xx:xx:xx:xx; + * After ethx down, fec all clocks are gated off and then register + * access causes system hang. + */ + if (!netif_running(ndev)) + return 0; + + writel(ndev->dev_addr[3] | (ndev->dev_addr[2] << 8) | + (ndev->dev_addr[1] << 16) | (ndev->dev_addr[0] << 24), + fep->hwp + FEC_ADDR_LOW); + writel((ndev->dev_addr[5] << 16) | (ndev->dev_addr[4] << 24), + fep->hwp + FEC_ADDR_HIGH); + return 0; +} + +static inline void fec_enet_set_netdev_features(struct net_device *netdev, + netdev_features_t features) +{ + struct fec_enet_private *fep = netdev_priv(netdev); + netdev_features_t changed = features ^ netdev->features; + + netdev->features = features; + + /* Receive checksum has been changed */ + if (changed & NETIF_F_RXCSUM) { + if (features & NETIF_F_RXCSUM) + fep->csum_flags |= FLAG_RX_CSUM_ENABLED; + else + fep->csum_flags &= ~FLAG_RX_CSUM_ENABLED; + } +} + +static int fec_set_features(struct net_device *netdev, + netdev_features_t features) +{ + struct fec_enet_private *fep = netdev_priv(netdev); + netdev_features_t changed = features ^ netdev->features; + mutex_lock(&fep->fast_ndev_lock); + + if (netif_running(netdev) && changed & NETIF_F_RXCSUM) { + fec_stop(netdev); + fec_enet_set_netdev_features(netdev, features); + fec_restart(netdev); + } else { + fec_enet_set_netdev_features(netdev, features); + } + + mutex_unlock(&fep->fast_ndev_lock); + return 0; +} + +static netdev_tx_t +fec_enet_start_xmit(struct sk_buff *skb, struct net_device *ndev) +{ + + return NETDEV_TX_OK; +} + +static u16 fec_enet_select_queue(struct net_device *ndev, struct sk_buff *skb, + struct net_device *sb_dev) +{ + return 0; +} + +static const struct net_device_ops fec_netdev_ops = { + .ndo_open = fec_enet_open, + .ndo_stop = fec_enet_close, + .ndo_start_xmit = fec_enet_start_xmit, + .ndo_select_queue = fec_enet_select_queue, + .ndo_fast_xmit = fec_ecat_fast_xmit, + .ndo_fast_recv = fec_ecat_fast_recv, + .ndo_set_rx_mode = set_multicast_list, + .ndo_validate_addr = eth_validate_addr, + .ndo_tx_timeout = fec_timeout, + .ndo_set_mac_address = fec_set_mac_address, + .ndo_eth_ioctl = fec_enet_ioctl, + .ndo_set_features = fec_set_features, +}; + +static int fec_enet_init(struct net_device *ndev) +{ + struct fec_enet_private *fep = netdev_priv(ndev); + struct bufdesc *cbd_base; + dma_addr_t bd_dma; + int bd_size; + unsigned int i; + unsigned dsize = sizeof(struct bufdesc); + unsigned dsize_log2 = __fls(dsize); + int ret; + unsigned char *pm = NULL; + + WARN_ON(dsize != (1 << dsize_log2)); +#if defined(CONFIG_ARM) || defined(CONFIG_ARM64) + fep->rx_align = 0xf; + fep->tx_align = 0xf; +#else + fep->rx_align = 0x3; + fep->tx_align = 0x3; +#endif + + /* Check mask of the streaming and coherent API */ + ret = dma_set_mask_and_coherent(&fep->pdev->dev, DMA_BIT_MASK(32)); + if (ret < 0) { + dev_warn(&fep->pdev->dev, "No suitable DMA available\n"); + return ret; + } + + ret = fec_enet_alloc_queue(ndev); + if (ret) + return ret; + + bd_size = (fep->total_tx_ring_size + fep->total_rx_ring_size) * dsize; + + /* Allocate memory for buffer descriptors. */ + cbd_base = dmam_alloc_coherent(&fep->pdev->dev, bd_size, &bd_dma, + GFP_KERNEL); + if (!cbd_base) { + ret = -ENOMEM; + goto free_queue_mem; + } + + /* Get the Ethernet address */ + ret = fec_get_mac(ndev); + if (ret) + goto free_queue_mem; + + /* make sure MAC we just acquired is programmed into the hw */ + fec_set_mac_address(ndev, NULL); + + /* Set receive and transmit descriptor base. */ + struct fec_enet_priv_rx_q *rxq = fep->rx_queue; + unsigned size; + + size = dsize * rxq->bd.ring_size; + rxq->bd.qid = i; + rxq->bd.base = cbd_base; + rxq->bd.cur = cbd_base; + rxq->bd.dma = bd_dma; + rxq->bd.dsize = dsize; + rxq->bd.dsize_log2 = dsize_log2; + rxq->bd.reg_desc_active = fep->hwp + FEC_R_DES_ACTIVE_0; + bd_dma += size; + cbd_base = (struct bufdesc *)(((void *)cbd_base) + size); + rxq->bd.last = (struct bufdesc *)(((void *)cbd_base) - dsize); + + struct fec_enet_priv_tx_q *txq = fep->tx_queue; + size = dsize * txq->bd.ring_size; + txq->bd.qid = i; + txq->bd.base = cbd_base; + txq->bd.cur = cbd_base; + txq->bd.dma = bd_dma; + txq->bd.dsize = dsize; + txq->bd.dsize_log2 = dsize_log2; + txq->bd.reg_desc_active = fep->hwp + FEC_X_DES_ACTIVE_0; + bd_dma += size; + cbd_base = (struct bufdesc *)(((void *)cbd_base) + size); + txq->bd.last = (struct bufdesc *)(((void *)cbd_base) - dsize); + + fep->netdev = ndev; + + /* The FEC Ethernet specific entries in the device structure */ + ndev->watchdog_timeo = TX_TIMEOUT; + ndev->netdev_ops = &fec_netdev_ops; + ndev->ethtool_ops = &fec_enet_ethtool_ops; + + pm = ndev->dev_addr; + writel(0, fep->hwp + FEC_IMASK); + + if (fep->quirks & FEC_QUIRK_HAS_MULTI_QUEUES) { + fep->tx_align = 0; + fep->rx_align = 0x3f; + } + + ndev->hw_features = ndev->features; + + fec_restart(ndev); + + if (fep->quirks & FEC_QUIRK_MIB_CLEAR) + fec_enet_clear_ethtool_stats(ndev); + else + fec_enet_update_ethtool_stats(ndev); + + return 0; + +free_queue_mem: + fec_enet_free_queue(ndev); + return ret; +} + +#ifdef CONFIG_OF +static int fec_reset_phy(struct platform_device *pdev) +{ + int err, phy_reset; + bool active_high = false; + int msec = 1, phy_post_delay = 0; + struct device_node *np = pdev->dev.of_node; + + if (!np) + return 0; + + err = of_property_read_u32(np, "phy-reset-duration", &msec); + /* A sane reset duration should not be longer than 1s */ + if (!err && msec > 1000) + msec = 1; + + phy_reset = of_get_named_gpio(np, "phy-reset-gpios", 0); + if (phy_reset == -EPROBE_DEFER) + return phy_reset; + else if (!gpio_is_valid(phy_reset)) + return 0; + + err = of_property_read_u32(np, "phy-reset-post-delay", &phy_post_delay); + /* valid reset duration should be less than 1s */ + if (!err && phy_post_delay > 1000) + return -EINVAL; + + active_high = of_property_read_bool(np, "phy-reset-active-high"); + + err = devm_gpio_request_one(&pdev->dev, phy_reset, + active_high ? GPIOF_OUT_INIT_HIGH : GPIOF_OUT_INIT_LOW, + "phy-reset"); + if (err) { + dev_err(&pdev->dev, "failed to get phy-reset-gpios: %d\n", err); + return err; + } + + if (msec > 20) + msleep(msec); + else + usleep_range(msec * 1000, msec * 1000 + 1000); + + gpio_set_value_cansleep(phy_reset, !active_high); + + if (!phy_post_delay) + return 0; + + if (phy_post_delay > 20) + msleep(phy_post_delay); + else + usleep_range(phy_post_delay * 1000, + phy_post_delay * 1000 + 1000); + + return 0; +} +#else /* CONFIG_OF */ +static int fec_reset_phy(struct platform_device *pdev) +{ + /* + * In case of platform probe, the reset has been done + * by machine code. + */ + return 0; +} +#endif /* CONFIG_OF */ + +static int fec_enet_init_stop_mode(struct fec_enet_private *fep, + struct device_node *np) +{ + struct device_node *gpr_np; + u32 out_val[3]; + int ret = 0; + + gpr_np = of_parse_phandle(np, "fsl,stop-mode", 0); + if (!gpr_np) + return 0; + + ret = of_property_read_u32_array(np, "fsl,stop-mode", out_val, + ARRAY_SIZE(out_val)); + if (ret) { + dev_dbg(&fep->pdev->dev, "no stop mode property\n"); + goto out; + } + + fep->stop_gpr.gpr = syscon_node_to_regmap(gpr_np); + if (IS_ERR(fep->stop_gpr.gpr)) { + dev_err(&fep->pdev->dev, "could not find gpr regmap\n"); + ret = PTR_ERR(fep->stop_gpr.gpr); + fep->stop_gpr.gpr = NULL; + goto out; + } + + fep->stop_gpr.reg = out_val[1]; + fep->stop_gpr.bit = out_val[2]; + +out: + of_node_put(gpr_np); + + return ret; +} + +static int +fec_probe(struct platform_device *pdev) +{ + struct fec_enet_private *fep; + struct fec_platform_data *pdata; + phy_interface_t interface; + struct net_device *ndev; + int i, irq, ret = 0; + const struct of_device_id *of_id; + static int dev_id; + struct device_node *np = pdev->dev.of_node, *phy_node; + char irq_name[8]; + struct fec_devinfo *dev_info; + + /* Init network device */ + ndev = alloc_etherdev_mqs(sizeof(struct fec_enet_private) + + FEC_STATS_SIZE, 1, 1); + if (!ndev) + return -ENOMEM; + ndev->fast_raw_device = 1; + SET_NETDEV_DEV(ndev, &pdev->dev); + + /* setup board info structure */ + fep = netdev_priv(ndev); + + of_id = of_match_device(fec_dt_ids, &pdev->dev); + if (of_id) + pdev->id_entry = of_id->data; + dev_info = (struct fec_devinfo *)pdev->id_entry->driver_data; + if (dev_info) + fep->quirks = dev_info->quirks; + + fep->netdev = ndev; + mutex_init(&fep->fast_ndev_lock); +#if !defined(CONFIG_M5272) + /* default enable pause frame auto negotiation */ + if (fep->quirks & FEC_QUIRK_HAS_GBIT) + fep->pause_flag |= FEC_PAUSE_FLAG_AUTONEG; +#endif + + /* Select default pin state */ + pinctrl_pm_select_default_state(&pdev->dev); + + fep->hwp = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(fep->hwp)) { + ret = PTR_ERR(fep->hwp); + goto failed_ioremap; + } + + fep->pdev = pdev; + fep->dev_id = dev_id++; + + platform_set_drvdata(pdev, ndev); + + if ((of_machine_is_compatible("fsl,imx6q") || + of_machine_is_compatible("fsl,imx6dl")) && + !of_property_read_bool(np, "fsl,err006687-workaround-present")) + fep->quirks |= FEC_QUIRK_ERR006687; + + ret = fec_enet_ipc_handle_init(fep); + if (ret) + goto failed_ipc_init; + + ret = fec_enet_init_stop_mode(fep, np); + if (ret) + goto failed_stop_mode; + + phy_node = of_parse_phandle(np, "phy-handle", 0); + if (!phy_node && of_phy_is_fixed_link(np)) { + ret = of_phy_register_fixed_link(np); + if (ret < 0) { + dev_err(&pdev->dev, + "broken fixed-link specification\n"); + goto failed_phy; + } + phy_node = of_node_get(np); + } + fep->phy_node = phy_node; + + ret = of_get_phy_mode(pdev->dev.of_node, &interface); + if (ret) { + pdata = dev_get_platdata(&pdev->dev); + if (pdata) + fep->phy_interface = pdata->phy; + else + fep->phy_interface = PHY_INTERFACE_MODE_MII; + } else { + fep->phy_interface = interface; + } + + ret = fec_enet_parse_rgmii_delay(fep, np); + if (ret) + goto failed_rgmii_delay; + + request_bus_freq(BUS_FREQ_HIGH); + + fep->clk_ipg = devm_clk_get(&pdev->dev, "ipg"); + if (IS_ERR(fep->clk_ipg)) { + ret = PTR_ERR(fep->clk_ipg); + goto failed_clk; + } + + fep->clk_ahb = devm_clk_get(&pdev->dev, "ahb"); + if (IS_ERR(fep->clk_ahb)) { + ret = PTR_ERR(fep->clk_ahb); + goto failed_clk; + } + + fep->itr_clk_rate = clk_get_rate(fep->clk_ahb); + + /* enet_out is optional, depends on board */ + fep->clk_enet_out = devm_clk_get(&pdev->dev, "enet_out"); + if (IS_ERR(fep->clk_enet_out)) + fep->clk_enet_out = NULL; + + fep->ptp_clk_on = false; + mutex_init(&fep->ptp_clk_mutex); + + /* clk_ref is optional, depends on board */ + fep->clk_ref = devm_clk_get(&pdev->dev, "enet_clk_ref"); + if (IS_ERR(fep->clk_ref)) + fep->clk_ref = NULL; + fep->clk_ref_rate = clk_get_rate(fep->clk_ref); + + /* clk_2x_txclk is optional, depends on board */ + if (fep->rgmii_txc_dly || fep->rgmii_rxc_dly) { + fep->clk_2x_txclk = devm_clk_get(&pdev->dev, "enet_2x_txclk"); + if (IS_ERR(fep->clk_2x_txclk)) + fep->clk_2x_txclk = NULL; + } + + ret = fec_enet_clk_enable(ndev, true); + if (ret) + goto failed_clk; + + ret = clk_prepare_enable(fep->clk_ipg); + if (ret) + goto failed_clk_ipg; + ret = clk_prepare_enable(fep->clk_ahb); + if (ret) + goto failed_clk_ahb; + + fep->reg_phy = devm_regulator_get_optional(&pdev->dev, "phy"); + if (!IS_ERR(fep->reg_phy)) { + ret = regulator_enable(fep->reg_phy); + if (ret) { + dev_err(&pdev->dev, + "Failed to enable phy regulator: %d\n", ret); + goto failed_regulator; + } + } else { + if (PTR_ERR(fep->reg_phy) == -EPROBE_DEFER) { + ret = -EPROBE_DEFER; + goto failed_regulator; + } + fep->reg_phy = NULL; + } + + if (of_property_read_u32(np, "fsl,rx-phy-delay-100-ns", &fep->rx_delay_100)) + fep->rx_delay_100 = 600; + + if (of_property_read_u32(np, "fsl,tx-phy-delay-100-ns", &fep->tx_delay_100)) + fep->tx_delay_100 = 0; + + if (of_property_read_u32(np, "fsl,rx-phy-delay-1000-ns", &fep->rx_delay_1000)) + fep->rx_delay_1000 = 0; + + if (of_property_read_u32(np, "fsl,tx-phy-delay-1000-ns", &fep->tx_delay_1000)) + fep->tx_delay_1000 = 0; + + pm_runtime_set_autosuspend_delay(&pdev->dev, FEC_MDIO_PM_TIMEOUT); + pm_runtime_use_autosuspend(&pdev->dev); + pm_runtime_get_noresume(&pdev->dev); + pm_runtime_set_active(&pdev->dev); + pm_runtime_enable(&pdev->dev); + + ret = fec_reset_phy(pdev); + if (ret) + goto failed_reset; + + //irq_cnt = fec_enet_get_irq_cnt(pdev); + + ret = fec_enet_init(ndev); + if (ret) + goto failed_init; + + /* board only enable one mii bus in default */ + if (!of_get_property(np, "fsl,mii-exclusive", NULL)) + fep->quirks |= FEC_QUIRK_SINGLE_MDIO; + ret = fec_enet_mii_init(pdev); + if (ret) + goto failed_mii_init; + + /* Carrier starts down, phylib will bring it up */ + netif_carrier_off(ndev); + pinctrl_pm_select_sleep_state(&pdev->dev); + + ndev->max_mtu = PKT_MAXBUF_SIZE - ETH_HLEN - ETH_FCS_LEN; + + ret = register_netdev(ndev); + if (ret) + goto failed_register; + + device_init_wakeup(&ndev->dev, fep->wol_flag & + FEC_WOL_HAS_MAGIC_PACKET); + + + INIT_WORK(&fep->tx_timeout_work, fec_enet_timeout_work); + + pm_runtime_mark_last_busy(&pdev->dev); + pm_runtime_put_autosuspend(&pdev->dev); + mutex_lock(&fep->fast_ndev_lock); + return 0; + +failed_register: + fec_enet_mii_remove(fep); +failed_mii_init: +failed_irq: +failed_init: + //fec_ptp_stop(pdev); +failed_reset: + pm_runtime_put_noidle(&pdev->dev); + pm_runtime_disable(&pdev->dev); + if (fep->reg_phy) + regulator_disable(fep->reg_phy); +failed_regulator: + clk_disable_unprepare(fep->clk_ahb); +failed_clk_ahb: + clk_disable_unprepare(fep->clk_ipg); +failed_clk_ipg: + fec_enet_clk_enable(ndev, false); +failed_clk: + release_bus_freq(BUS_FREQ_HIGH); +failed_rgmii_delay: + if (of_phy_is_fixed_link(np)) + of_phy_deregister_fixed_link(np); + of_node_put(phy_node); +failed_stop_mode: +failed_ipc_init: +failed_phy: + dev_id--; +failed_ioremap: + free_netdev(ndev); + + return ret; +} + +static int +fec_drv_remove(struct platform_device *pdev) +{ + struct net_device *ndev = platform_get_drvdata(pdev); + struct fec_enet_private *fep = netdev_priv(ndev); + struct device_node *np = pdev->dev.of_node; + int ret; + + ret = pm_runtime_resume_and_get(&pdev->dev); + if (ret < 0) + return ret; + + cancel_work_sync(&fep->tx_timeout_work); + //fec_ptp_stop(pdev); + unregister_netdev(ndev); + fec_enet_mii_remove(fep); + if (fep->reg_phy) + regulator_disable(fep->reg_phy); + + if (of_phy_is_fixed_link(np)) + of_phy_deregister_fixed_link(np); + of_node_put(fep->phy_node); + + clk_disable_unprepare(fep->clk_ahb); + clk_disable_unprepare(fep->clk_ipg); + pm_runtime_put_noidle(&pdev->dev); + pm_runtime_disable(&pdev->dev); + + free_netdev(ndev); + mutex_unlock(&fep->fast_ndev_lock); + return 0; +} + +static int __maybe_unused fec_suspend(struct device *dev) +{ + struct net_device *ndev = dev_get_drvdata(dev); + struct fec_enet_private *fep = netdev_priv(ndev); + + if (netif_running(ndev)) { + int ret; + + phy_stop(ndev->phydev); + + netif_device_detach(ndev); + fec_stop(ndev); + fec_enet_clk_enable(ndev, false); + + fep->rpm_active = !pm_runtime_status_suspended(dev); + if (fep->rpm_active) { + ret = pm_runtime_force_suspend(dev); + if (ret < 0) + return ret; + } + } else if (fep->mii_bus_share && !ndev->phydev) { + pinctrl_pm_select_sleep_state(&fep->pdev->dev); + } + + if (fep->reg_phy && !(fep->wol_flag & FEC_WOL_FLAG_ENABLE)) + regulator_disable(fep->reg_phy); + + /* SOC supply clock to phy, when clock is disabled, phy link down + * SOC control phy regulator, when regulator is disabled, phy link down + */ + if (fep->clk_enet_out || fep->reg_phy) + fep->link = 0; + + return 0; +} + +static int __maybe_unused fec_resume(struct device *dev) +{ + struct net_device *ndev = dev_get_drvdata(dev); + struct fec_enet_private *fep = netdev_priv(ndev); + int ret = 0; + int val; + + if (fep->reg_phy && !(fep->wol_flag & FEC_WOL_FLAG_ENABLE)) { + ret = regulator_enable(fep->reg_phy); + if (ret) + return ret; + } + + if (netif_running(ndev)) { + if (fep->rpm_active) + pm_runtime_force_resume(dev); + + ret = fec_enet_clk_enable(ndev, true); + if (ret) { + rtnl_unlock(); + goto failed_clk; + } + pinctrl_pm_select_default_state(&fep->pdev->dev); + fec_restart(ndev); + netif_device_attach(ndev); + + //napi_enable(&fep->napi); + + phy_init_hw(ndev->phydev); + phy_start(ndev->phydev); + } else if (fep->mii_bus_share && !ndev->phydev) { + pinctrl_pm_select_default_state(&fep->pdev->dev); + /* And then recovery mii bus */ + ret = fec_restore_mii_bus(ndev); + } + + return ret; + +failed_clk: + if (fep->reg_phy) + regulator_disable(fep->reg_phy); + return ret; +} + +static int __maybe_unused fec_runtime_suspend(struct device *dev) +{ + struct net_device *ndev = dev_get_drvdata(dev); + struct fec_enet_private *fep = netdev_priv(ndev); + + clk_disable_unprepare(fep->clk_ahb); + clk_disable_unprepare(fep->clk_ipg); + release_bus_freq(BUS_FREQ_HIGH); + + return 0; +} + +static int __maybe_unused fec_runtime_resume(struct device *dev) +{ + struct net_device *ndev = dev_get_drvdata(dev); + struct fec_enet_private *fep = netdev_priv(ndev); + int ret; + + request_bus_freq(BUS_FREQ_HIGH); + + ret = clk_prepare_enable(fep->clk_ahb); + if (ret) + return ret; + ret = clk_prepare_enable(fep->clk_ipg); + if (ret) + goto failed_clk_ipg; + + return 0; + +failed_clk_ipg: + clk_disable_unprepare(fep->clk_ahb); + release_bus_freq(BUS_FREQ_HIGH); + return ret; +} + +static const struct dev_pm_ops fec_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(fec_suspend, fec_resume) + SET_RUNTIME_PM_OPS(fec_runtime_suspend, fec_runtime_resume, NULL) +}; + +static struct platform_driver fec_ecat_driver = { + .driver = { + .name = DRIVER_NAME, + .pm = &fec_pm_ops, + .of_match_table = fec_dt_ids, + .suppress_bind_attrs = true, + }, + .id_table = fec_devtype, + .probe = fec_probe, + .remove = fec_drv_remove, +}; + +module_platform_driver(fec_ecat_driver); + +MODULE_LICENSE("GPL"); diff --git a/drivers/net/ethernet/freescale/fec_ecat.h b/drivers/net/ethernet/freescale/fec_ecat.h new file mode 100644 index 000000000000..87db4c85db24 --- /dev/null +++ b/drivers/net/ethernet/freescale/fec_ecat.h @@ -0,0 +1,713 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2023 NXP + */ + +/****************************************************************************/ +#ifndef FEC_ECAT_H +#define FEC_ECAT_H +/****************************************************************************/ + +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(CONFIG_M523x) || defined(CONFIG_M527x) || defined(CONFIG_M528x) || \ + defined(CONFIG_M520x) || defined(CONFIG_M532x) || defined(CONFIG_ARM) || \ + defined(CONFIG_ARM64) || defined(CONFIG_COMPILE_TEST) +/* + * Just figures, Motorola would have to change the offsets for + * registers in the same peripheral device on different models + * of the ColdFire! + */ +#define FEC_IEVENT 0x004 /* Interrupt event reg */ +#define FEC_IMASK 0x008 /* Interrupt mask reg */ +#define FEC_R_DES_ACTIVE_0 0x010 /* Receive descriptor reg */ +#define FEC_X_DES_ACTIVE_0 0x014 /* Transmit descriptor reg */ +#define FEC_ECNTRL 0x024 /* Ethernet control reg */ +#define FEC_MII_DATA 0x040 /* MII manage frame reg */ +#define FEC_MII_SPEED 0x044 /* MII speed control reg */ +#define FEC_MIB_CTRLSTAT 0x064 /* MIB control/status reg */ +#define FEC_R_CNTRL 0x084 /* Receive control reg */ +#define FEC_X_CNTRL 0x0c4 /* Transmit Control reg */ +#define FEC_ADDR_LOW 0x0e4 /* Low 32bits MAC address */ +#define FEC_ADDR_HIGH 0x0e8 /* High 16bits MAC address */ +#define FEC_OPD 0x0ec /* Opcode + Pause duration */ +#define FEC_TXIC0 0x0f0 /* Tx Interrupt Coalescing for ring 0 */ +#define FEC_TXIC1 0x0f4 /* Tx Interrupt Coalescing for ring 1 */ +#define FEC_TXIC2 0x0f8 /* Tx Interrupt Coalescing for ring 2 */ +#define FEC_RXIC0 0x100 /* Rx Interrupt Coalescing for ring 0 */ +#define FEC_RXIC1 0x104 /* Rx Interrupt Coalescing for ring 1 */ +#define FEC_RXIC2 0x108 /* Rx Interrupt Coalescing for ring 2 */ +#define FEC_HASH_TABLE_HIGH 0x118 /* High 32bits hash table */ +#define FEC_HASH_TABLE_LOW 0x11c /* Low 32bits hash table */ +#define FEC_GRP_HASH_TABLE_HIGH 0x120 /* High 32bits hash table */ +#define FEC_GRP_HASH_TABLE_LOW 0x124 /* Low 32bits hash table */ +#define FEC_X_WMRK 0x144 /* FIFO transmit water mark */ +#define FEC_R_BOUND 0x14c /* FIFO receive bound reg */ +#define FEC_R_FSTART 0x150 /* FIFO receive start reg */ +#define FEC_R_DES_START_1 0x160 /* Receive descriptor ring 1 */ +#define FEC_X_DES_START_1 0x164 /* Transmit descriptor ring 1 */ +#define FEC_R_BUFF_SIZE_1 0x168 /* Maximum receive buff ring1 size */ +#define FEC_R_DES_START_2 0x16c /* Receive descriptor ring 2 */ +#define FEC_X_DES_START_2 0x170 /* Transmit descriptor ring 2 */ +#define FEC_R_BUFF_SIZE_2 0x174 /* Maximum receive buff ring2 size */ +#define FEC_R_DES_START_0 0x180 /* Receive descriptor ring */ +#define FEC_X_DES_START_0 0x184 /* Transmit descriptor ring */ +#define FEC_R_BUFF_SIZE_0 0x188 /* Maximum receive buff size */ +#define FEC_R_FIFO_RSFL 0x190 /* Receive FIFO section full threshold */ +#define FEC_R_FIFO_RSEM 0x194 /* Receive FIFO section empty threshold */ +#define FEC_R_FIFO_RAEM 0x198 /* Receive FIFO almost empty threshold */ +#define FEC_R_FIFO_RAFL 0x19c /* Receive FIFO almost full threshold */ +#define FEC_FTRL 0x1b0 /* Frame truncation receive length*/ +#define FEC_RACC 0x1c4 /* Receive Accelerator function */ +#define FEC_RCMR_1 0x1c8 /* Receive classification match ring 1 */ +#define FEC_RCMR_2 0x1cc /* Receive classification match ring 2 */ +#define FEC_DMA_CFG_1 0x1d8 /* DMA class configuration for ring 1 */ +#define FEC_DMA_CFG_2 0x1dc /* DMA class Configuration for ring 2 */ +#define FEC_R_DES_ACTIVE_1 0x1e0 /* Rx descriptor active for ring 1 */ +#define FEC_X_DES_ACTIVE_1 0x1e4 /* Tx descriptor active for ring 1 */ +#define FEC_R_DES_ACTIVE_2 0x1e8 /* Rx descriptor active for ring 2 */ +#define FEC_X_DES_ACTIVE_2 0x1ec /* Tx descriptor active for ring 2 */ +#define FEC_QOS_SCHEME 0x1f0 /* Set multi queues Qos scheme */ +#define FEC_LPI_SLEEP 0x1f4 /* Set IEEE802.3az LPI Sleep Ts time */ +#define FEC_LPI_WAKE 0x1f8 /* Set IEEE802.3az LPI Wake Tw time */ +#define FEC_MIIGSK_CFGR 0x300 /* MIIGSK Configuration reg */ +#define FEC_MIIGSK_ENR 0x308 /* MIIGSK Enable reg */ + +#define BM_MIIGSK_CFGR_MII 0x00 +#define BM_MIIGSK_CFGR_RMII 0x01 +#define BM_MIIGSK_CFGR_FRCONT_10M 0x40 + +#define RMON_T_DROP 0x200 /* Count of frames not cntd correctly */ +#define RMON_T_PACKETS 0x204 /* RMON TX packet count */ +#define RMON_T_BC_PKT 0x208 /* RMON TX broadcast pkts */ +#define RMON_T_MC_PKT 0x20c /* RMON TX multicast pkts */ +#define RMON_T_CRC_ALIGN 0x210 /* RMON TX pkts with CRC align err */ +#define RMON_T_UNDERSIZE 0x214 /* RMON TX pkts < 64 bytes, good CRC */ +#define RMON_T_OVERSIZE 0x218 /* RMON TX pkts > MAX_FL bytes good CRC */ +#define RMON_T_FRAG 0x21c /* RMON TX pkts < 64 bytes, bad CRC */ +#define RMON_T_JAB 0x220 /* RMON TX pkts > MAX_FL bytes, bad CRC */ +#define RMON_T_COL 0x224 /* RMON TX collision count */ +#define RMON_T_P64 0x228 /* RMON TX 64 byte pkts */ +#define RMON_T_P65TO127 0x22c /* RMON TX 65 to 127 byte pkts */ +#define RMON_T_P128TO255 0x230 /* RMON TX 128 to 255 byte pkts */ +#define RMON_T_P256TO511 0x234 /* RMON TX 256 to 511 byte pkts */ +#define RMON_T_P512TO1023 0x238 /* RMON TX 512 to 1023 byte pkts */ +#define RMON_T_P1024TO2047 0x23c /* RMON TX 1024 to 2047 byte pkts */ +#define RMON_T_P_GTE2048 0x240 /* RMON TX pkts > 2048 bytes */ +#define RMON_T_OCTETS 0x244 /* RMON TX octets */ +#define IEEE_T_DROP 0x248 /* Count of frames not counted crtly */ +#define IEEE_T_FRAME_OK 0x24c /* Frames tx'd OK */ +#define IEEE_T_1COL 0x250 /* Frames tx'd with single collision */ +#define IEEE_T_MCOL 0x254 /* Frames tx'd with multiple collision */ +#define IEEE_T_DEF 0x258 /* Frames tx'd after deferral delay */ +#define IEEE_T_LCOL 0x25c /* Frames tx'd with late collision */ +#define IEEE_T_EXCOL 0x260 /* Frames tx'd with excesv collisions */ +#define IEEE_T_MACERR 0x264 /* Frames tx'd with TX FIFO underrun */ +#define IEEE_T_CSERR 0x268 /* Frames tx'd with carrier sense err */ +#define IEEE_T_SQE 0x26c /* Frames tx'd with SQE err */ +#define IEEE_T_FDXFC 0x270 /* Flow control pause frames tx'd */ +#define IEEE_T_OCTETS_OK 0x274 /* Octet count for frames tx'd w/o err */ +#define RMON_R_PACKETS 0x284 /* RMON RX packet count */ +#define RMON_R_BC_PKT 0x288 /* RMON RX broadcast pkts */ +#define RMON_R_MC_PKT 0x28c /* RMON RX multicast pkts */ +#define RMON_R_CRC_ALIGN 0x290 /* RMON RX pkts with CRC alignment err */ +#define RMON_R_UNDERSIZE 0x294 /* RMON RX pkts < 64 bytes, good CRC */ +#define RMON_R_OVERSIZE 0x298 /* RMON RX pkts > MAX_FL bytes good CRC */ +#define RMON_R_FRAG 0x29c /* RMON RX pkts < 64 bytes, bad CRC */ +#define RMON_R_JAB 0x2a0 /* RMON RX pkts > MAX_FL bytes, bad CRC */ +#define RMON_R_RESVD_O 0x2a4 /* Reserved */ +#define RMON_R_P64 0x2a8 /* RMON RX 64 byte pkts */ +#define RMON_R_P65TO127 0x2ac /* RMON RX 65 to 127 byte pkts */ +#define RMON_R_P128TO255 0x2b0 /* RMON RX 128 to 255 byte pkts */ +#define RMON_R_P256TO511 0x2b4 /* RMON RX 256 to 511 byte pkts */ +#define RMON_R_P512TO1023 0x2b8 /* RMON RX 512 to 1023 byte pkts */ +#define RMON_R_P1024TO2047 0x2bc /* RMON RX 1024 to 2047 byte pkts */ +#define RMON_R_P_GTE2048 0x2c0 /* RMON RX pkts > 2048 bytes */ +#define RMON_R_OCTETS 0x2c4 /* RMON RX octets */ +#define IEEE_R_DROP 0x2c8 /* Count frames not counted correctly */ +#define IEEE_R_FRAME_OK 0x2cc /* Frames rx'd OK */ +#define IEEE_R_CRC 0x2d0 /* Frames rx'd with CRC err */ +#define IEEE_R_ALIGN 0x2d4 /* Frames rx'd with alignment err */ +#define IEEE_R_MACERR 0x2d8 /* Receive FIFO overflow count */ +#define IEEE_R_FDXFC 0x2dc /* Flow control pause frames rx'd */ +#define IEEE_R_OCTETS_OK 0x2e0 /* Octet cnt for frames rx'd w/o err */ + +#else + +#define FEC_ECNTRL 0x000 /* Ethernet control reg */ +#define FEC_IEVENT 0x004 /* Interrupt even reg */ +#define FEC_IMASK 0x008 /* Interrupt mask reg */ +#define FEC_IVEC 0x00c /* Interrupt vec status reg */ +#define FEC_R_DES_ACTIVE_0 0x010 /* Receive descriptor reg */ +#define FEC_R_DES_ACTIVE_1 FEC_R_DES_ACTIVE_0 +#define FEC_R_DES_ACTIVE_2 FEC_R_DES_ACTIVE_0 +#define FEC_X_DES_ACTIVE_0 0x014 /* Transmit descriptor reg */ +#define FEC_X_DES_ACTIVE_1 FEC_X_DES_ACTIVE_0 +#define FEC_X_DES_ACTIVE_2 FEC_X_DES_ACTIVE_0 +#define FEC_MII_DATA 0x040 /* MII manage frame reg */ +#define FEC_MII_SPEED 0x044 /* MII speed control reg */ +#define FEC_R_BOUND 0x08c /* FIFO receive bound reg */ +#define FEC_R_FSTART 0x090 /* FIFO receive start reg */ +#define FEC_X_WMRK 0x0a4 /* FIFO transmit water mark */ +#define FEC_X_FSTART 0x0ac /* FIFO transmit start reg */ +#define FEC_R_CNTRL 0x104 /* Receive control reg */ +#define FEC_MAX_FRM_LEN 0x108 /* Maximum frame length reg */ +#define FEC_X_CNTRL 0x144 /* Transmit Control reg */ +#define FEC_ADDR_LOW 0x3c0 /* Low 32bits MAC address */ +#define FEC_ADDR_HIGH 0x3c4 /* High 16bits MAC address */ +#define FEC_GRP_HASH_TABLE_HIGH 0x3c8 /* High 32bits hash table */ +#define FEC_GRP_HASH_TABLE_LOW 0x3cc /* Low 32bits hash table */ +#define FEC_R_DES_START_0 0x3d0 /* Receive descriptor ring */ +#define FEC_R_DES_START_1 FEC_R_DES_START_0 +#define FEC_R_DES_START_2 FEC_R_DES_START_0 +#define FEC_X_DES_START_0 0x3d4 /* Transmit descriptor ring */ +#define FEC_X_DES_START_1 FEC_X_DES_START_0 +#define FEC_X_DES_START_2 FEC_X_DES_START_0 +#define FEC_R_BUFF_SIZE_0 0x3d8 /* Maximum receive buff size */ +#define FEC_R_BUFF_SIZE_1 FEC_R_BUFF_SIZE_0 +#define FEC_R_BUFF_SIZE_2 FEC_R_BUFF_SIZE_0 +#define FEC_FIFO_RAM 0x400 /* FIFO RAM buffer */ +/* Not existed in real chip + * Just for pass build. + */ +#define FEC_RCMR_1 0xfff +#define FEC_RCMR_2 0xfff +#define FEC_DMA_CFG_1 0xfff +#define FEC_DMA_CFG_2 0xfff +#define FEC_TXIC0 0xfff +#define FEC_TXIC1 0xfff +#define FEC_TXIC2 0xfff +#define FEC_RXIC0 0xfff +#define FEC_RXIC1 0xfff +#define FEC_RXIC2 0xfff +#define FEC_LPI_SLEEP 0xfff +#define FEC_LPI_WAKE 0xfff +#endif /* CONFIG_M5272 */ + + +/* + * Define the buffer descriptor structure. + * + * Evidently, ARM SoCs have the FEC block generated in a + * little endian mode so adjust endianness accordingly. + */ +#if defined(CONFIG_ARM) || defined(CONFIG_ARM64) +#define fec32_to_cpu le32_to_cpu +#define fec16_to_cpu le16_to_cpu +#define cpu_to_fec32 cpu_to_le32 +#define cpu_to_fec16 cpu_to_le16 +#define __fec32 __le32 +#define __fec16 __le16 + +struct bufdesc { + __fec16 cbd_datlen; /* Data length */ + __fec16 cbd_sc; /* Control and status info */ + __fec32 cbd_bufaddr; /* Buffer address */ +}; +#else +#define fec32_to_cpu be32_to_cpu +#define fec16_to_cpu be16_to_cpu +#define cpu_to_fec32 cpu_to_be32 +#define cpu_to_fec16 cpu_to_be16 +#define __fec32 __be32 +#define __fec16 __be16 + +struct bufdesc { + __fec16 cbd_sc; /* Control and status info */ + __fec16 cbd_datlen; /* Data length */ + __fec32 cbd_bufaddr; /* Buffer address */ +}; +#endif + +struct bufdesc_ex { + struct bufdesc desc; + __fec32 cbd_esc; + __fec32 cbd_prot; + __fec32 cbd_bdu; + __fec32 ts; + __fec16 res0[4]; +}; + +/* + * The following definitions courtesy of commproc.h, which where + * Copyright (c) 1997 Dan Malek (dmalek@jlc.net). + */ +#define BD_SC_EMPTY ((ushort)0x8000) /* Receive is empty */ +#define BD_SC_READY ((ushort)0x8000) /* Transmit is ready */ +#define BD_SC_WRAP ((ushort)0x2000) /* Last buffer descriptor */ +#define BD_SC_INTRPT ((ushort)0x1000) /* Interrupt on change */ +#define BD_SC_CM ((ushort)0x0200) /* Continuous mode */ +#define BD_SC_ID ((ushort)0x0100) /* Rec'd too many idles */ +#define BD_SC_P ((ushort)0x0100) /* xmt preamble */ +#define BD_SC_BR ((ushort)0x0020) /* Break received */ +#define BD_SC_FR ((ushort)0x0010) /* Framing error */ +#define BD_SC_PR ((ushort)0x0008) /* Parity error */ +#define BD_SC_OV ((ushort)0x0002) /* Overrun */ +#define BD_SC_CD ((ushort)0x0001) /* ?? */ + +/* Buffer descriptor control/status used by Ethernet receive. + */ +#define BD_ENET_RX_EMPTY ((ushort)0x8000) +#define BD_ENET_RX_WRAP ((ushort)0x2000) +#define BD_ENET_RX_INTR ((ushort)0x1000) +#define BD_ENET_RX_LAST ((ushort)0x0800) +#define BD_ENET_RX_FIRST ((ushort)0x0400) +#define BD_ENET_RX_MISS ((ushort)0x0100) +#define BD_ENET_RX_LG ((ushort)0x0020) +#define BD_ENET_RX_NO ((ushort)0x0010) +#define BD_ENET_RX_SH ((ushort)0x0008) +#define BD_ENET_RX_CR ((ushort)0x0004) +#define BD_ENET_RX_OV ((ushort)0x0002) +#define BD_ENET_RX_CL ((ushort)0x0001) +#define BD_ENET_RX_STATS ((ushort)0x013f) /* All status bits */ + +/* Enhanced buffer descriptor control/status used by Ethernet receive */ +#define BD_ENET_RX_VLAN 0x00000004 + +/* Buffer descriptor control/status used by Ethernet transmit. + */ +#define BD_ENET_TX_READY ((ushort)0x8000) +#define BD_ENET_TX_PAD ((ushort)0x4000) +#define BD_ENET_TX_WRAP ((ushort)0x2000) +#define BD_ENET_TX_INTR ((ushort)0x1000) +#define BD_ENET_TX_LAST ((ushort)0x0800) +#define BD_ENET_TX_TC ((ushort)0x0400) +#define BD_ENET_TX_DEF ((ushort)0x0200) +#define BD_ENET_TX_HB ((ushort)0x0100) +#define BD_ENET_TX_LC ((ushort)0x0080) +#define BD_ENET_TX_RL ((ushort)0x0040) +#define BD_ENET_TX_RCMASK ((ushort)0x003c) +#define BD_ENET_TX_UN ((ushort)0x0002) +#define BD_ENET_TX_CSL ((ushort)0x0001) +#define BD_ENET_TX_STATS ((ushort)0x0fff) /* All status bits */ + +/* enhanced buffer descriptor control/status used by Ethernet transmit */ +#define BD_ENET_TX_INT 0x40000000 +#define BD_ENET_TX_TS 0x20000000 +#define BD_ENET_TX_PINS 0x10000000 +#define BD_ENET_TX_IINS 0x08000000 + + +/* This device has up to three irqs on some platforms */ +#define FEC_IRQ_NUM 3 + +/* Maximum number of queues supported + * ENET with AVB IP can support up to 3 independent tx queues and rx queues. + * User can point the queue number that is less than or equal to 3. + */ +#define FEC_ENET_MAX_TX_QS 3 +#define FEC_ENET_MAX_RX_QS 3 + +#define FEC_R_DES_START(X) (((X) == 1) ? FEC_R_DES_START_1 : \ + (((X) == 2) ? \ + FEC_R_DES_START_2 : FEC_R_DES_START_0)) +#define FEC_X_DES_START(X) (((X) == 1) ? FEC_X_DES_START_1 : \ + (((X) == 2) ? \ + FEC_X_DES_START_2 : FEC_X_DES_START_0)) +#define FEC_R_BUFF_SIZE(X) (((X) == 1) ? FEC_R_BUFF_SIZE_1 : \ + (((X) == 2) ? \ + FEC_R_BUFF_SIZE_2 : FEC_R_BUFF_SIZE_0)) + +#define FEC_DMA_CFG(X) (((X) == 2) ? FEC_DMA_CFG_2 : FEC_DMA_CFG_1) + +#define DMA_CLASS_EN (1 << 16) +#define FEC_RCMR(X) (((X) == 2) ? FEC_RCMR_2 : FEC_RCMR_1) +#define IDLE_SLOPE_MASK 0xffff +#define IDLE_SLOPE_1 0x200 /* BW fraction: 0.5 */ +#define IDLE_SLOPE_2 0x200 /* BW fraction: 0.5 */ +#define IDLE_SLOPE(X) (((X) == 1) ? \ + (IDLE_SLOPE_1 & IDLE_SLOPE_MASK) : \ + (IDLE_SLOPE_2 & IDLE_SLOPE_MASK)) +#define RCMR_MATCHEN (0x1 << 16) +#define RCMR_CMP_CFG(v, n) (((v) & 0x7) << (n << 2)) +#ifdef CONFIG_AVB_SUPPORT +#define SR_CLASS_A_PRIORITY 3 +#define SR_CLASS_B_PRIORITY 2 +#define RCMR_CMP_1 (RCMR_CMP_CFG(SR_CLASS_A_PRIORITY, 0) | \ + RCMR_CMP_CFG(SR_CLASS_A_PRIORITY, 1) | \ + RCMR_CMP_CFG(SR_CLASS_A_PRIORITY, 2) | \ + RCMR_CMP_CFG(SR_CLASS_A_PRIORITY, 3)) +#define RCMR_CMP_2 (RCMR_CMP_CFG(SR_CLASS_B_PRIORITY, 0) | \ + RCMR_CMP_CFG(SR_CLASS_B_PRIORITY, 1) | \ + RCMR_CMP_CFG(SR_CLASS_B_PRIORITY, 2) | \ + RCMR_CMP_CFG(SR_CLASS_B_PRIORITY, 3)) +#else +#define RCMR_CMP_1 (RCMR_CMP_CFG(0, 0) | RCMR_CMP_CFG(1, 1) | \ + RCMR_CMP_CFG(2, 2) | RCMR_CMP_CFG(3, 3)) +#define RCMR_CMP_2 (RCMR_CMP_CFG(4, 0) | RCMR_CMP_CFG(5, 1) | \ + RCMR_CMP_CFG(6, 2) | RCMR_CMP_CFG(7, 3)) +#endif +#define RCMR_CMP(X) (((X) == 1) ? RCMR_CMP_1 : RCMR_CMP_2) +#define FEC_TX_BD_FTYPE(X) (((X) & 0xf) << 20) + +#define FEC_RX_FLUSH(X) (1 << ((X) + 3)) + +#define FEC_TX_SCHEME_CB 0x0 /* Credit based */ +#define FEC_TX_SCHEME_RR 0x1 /* Round-robin */ + +#define BD_ENET_RX_INT 0x00800000 +#define BD_ENET_RX_PTP ((ushort)0x0400) +#define BD_ENET_RX_ICE 0x00000020 +#define BD_ENET_RX_PCR 0x00000010 +#define FLAG_RX_CSUM_ENABLED (BD_ENET_RX_ICE | BD_ENET_RX_PCR) +#define FLAG_RX_CSUM_ERROR (BD_ENET_RX_ICE | BD_ENET_RX_PCR) + +/* Interrupt events/masks. */ +#define FEC_ENET_HBERR ((uint)0x80000000) /* Heartbeat error */ +#define FEC_ENET_BABR ((uint)0x40000000) /* Babbling receiver */ +#define FEC_ENET_BABT ((uint)0x20000000) /* Babbling transmitter */ +#define FEC_ENET_GRA ((uint)0x10000000) /* Graceful stop complete */ +#define FEC_ENET_TXF_0 ((uint)0x08000000) /* Full frame transmitted */ +#define FEC_ENET_TXF_1 ((uint)0x00000008) /* Full frame transmitted */ +#define FEC_ENET_TXF_2 ((uint)0x00000080) /* Full frame transmitted */ +#define FEC_ENET_TXB ((uint)0x04000000) /* A buffer was transmitted */ +#define FEC_ENET_RXF_0 ((uint)0x02000000) /* Full frame received */ +#define FEC_ENET_RXF_1 ((uint)0x00000002) /* Full frame received */ +#define FEC_ENET_RXF_2 ((uint)0x00000020) /* Full frame received */ +#define FEC_ENET_RXB ((uint)0x01000000) /* A buffer was received */ +#define FEC_ENET_MII ((uint)0x00800000) /* MII interrupt */ +#define FEC_ENET_EBERR ((uint)0x00400000) /* SDMA bus error */ +#define FEC_ENET_WAKEUP ((uint)0x00020000) /* Wakeup request */ +#define FEC_ENET_TXF (FEC_ENET_TXF_0 | FEC_ENET_TXF_1 | FEC_ENET_TXF_2) +#define FEC_ENET_RXF (FEC_ENET_RXF_0 | FEC_ENET_RXF_1 | FEC_ENET_RXF_2) +#define FEC_ENET_RXF_GET(X) (((X) == 0) ? FEC_ENET_RXF_0 : \ + (((X) == 1) ? FEC_ENET_RXF_1 : \ + FEC_ENET_RXF_2)) +#define FEC_ENET_TS_AVAIL ((uint)0x00010000) +#define FEC_ENET_TS_TIMER ((uint)0x00008000) + +#define FEC_DEFAULT_IMASK (FEC_ENET_TXF | FEC_ENET_RXF) +#define FEC_RX_DISABLED_IMASK (FEC_DEFAULT_IMASK & (~FEC_ENET_RXF)) + +#define FEC_ENET_ETHEREN ((uint)0x00000002) +#define FEC_ENET_TXC_DLY ((uint)0x00010000) +#define FEC_ENET_RXC_DLY ((uint)0x00020000) + +/* ENET interrupt coalescing macro define */ +#define FEC_ITR_CLK_SEL (0x1 << 30) +#define FEC_ITR_EN (0x1 << 31) +#define FEC_ITR_ICFT(X) (((X) & 0xff) << 20) +#define FEC_ITR_ICTT(X) ((X) & 0xffff) +#define FEC_ITR_ICFT_DEFAULT 200 /* Set 200 frame count threshold */ +#define FEC_ITR_ICTT_DEFAULT 1000 /* Set 1000us timer threshold */ + +#define FEC_VLAN_TAG_LEN 0x04 +#define FEC_ETHTYPE_LEN 0x02 + +/* Controller is ENET-MAC */ +#define FEC_QUIRK_ENET_MAC (1 << 0) +/* Controller needs driver to swap frame */ +#define FEC_QUIRK_SWAP_FRAME (1 << 1) +/* Controller uses gasket */ +#define FEC_QUIRK_USE_GASKET (1 << 2) +/* Controller has GBIT support */ +#define FEC_QUIRK_HAS_GBIT (1 << 3) +/* Controller has extend desc buffer */ +#define FEC_QUIRK_HAS_BUFDESC_EX (1 << 4) +/* Controller has hardware checksum support */ +#define FEC_QUIRK_HAS_CSUM (1 << 5) +/* Controller has hardware vlan support */ +#define FEC_QUIRK_HAS_VLAN (1 << 6) +/* ENET IP errata ERR006358 + * + * If the ready bit in the transmit buffer descriptor (TxBD[R]) is previously + * detected as not set during a prior frame transmission, then the + * ENET_TDAR[TDAR] bit is cleared at a later time, even if additional TxBDs + * were added to the ring and the ENET_TDAR[TDAR] bit is set. This results in + * frames not being transmitted until there is a 0-to-1 transition on + * ENET_TDAR[TDAR]. + */ +#define FEC_QUIRK_ERR006358 (1 << 7) +/* ENET IP hw AVB + * + * i.MX6SX ENET IP add Audio Video Bridging (AVB) feature support. + * - Two class indicators on receive with configurable priority + * - Two class indicators and line speed timer on transmit allowing + * implementation class credit based shapers externally + * - Additional DMA registers provisioned to allow managing up to 3 + * independent rings + */ +#define FEC_QUIRK_HAS_AVB (1 << 8) +/* There is a TDAR race condition for mutliQ when the software sets TDAR + * and the UDMA clears TDAR simultaneously or in a small window (2-4 cycles). + * This will cause the udma_tx and udma_tx_arbiter state machines to hang. + * The issue exist at i.MX6SX enet IP. + */ +#define FEC_QUIRK_ERR007885 (1 << 9) +/* ENET Block Guide/ Chapter for the iMX6SX (PELE) address one issue: + * After set ENET_ATCR[Capture], there need some time cycles before the counter + * value is capture in the register clock domain. + * The wait-time-cycles is at least 6 clock cycles of the slower clock between + * the register clock and the 1588 clock. The 1588 ts_clk is fixed to 25Mhz, + * register clock is 66Mhz, so the wait-time-cycles must be greater than 240ns + * (40ns * 6). + */ +#define FEC_QUIRK_BUG_CAPTURE (1 << 10) +/* Controller has only one MDIO bus */ +#define FEC_QUIRK_SINGLE_MDIO (1 << 11) +/* Controller supports RACC register */ +#define FEC_QUIRK_HAS_RACC (1 << 12) +/* Controller supports interrupt coalesc */ +#define FEC_QUIRK_HAS_COALESCE (1 << 13) +/* Interrupt doesn't wake CPU from deep idle */ +#define FEC_QUIRK_ERR006687 (1 << 14) +/* The MIB counters should be cleared and enabled during + * initialisation. + */ +#define FEC_QUIRK_MIB_CLEAR (1 << 15) +/* Only i.MX25/i.MX27/i.MX28 controller supports FRBR,FRSR registers, + * those FIFO receive registers are resolved in other platforms. + */ +#define FEC_QUIRK_HAS_FRREG (1 << 16) + +/* Some FEC hardware blocks need the MMFR cleared at setup time to avoid + * the generation of an MII event. This must be avoided in the older + * FEC blocks where it will stop MII events being generated. + */ +#define FEC_QUIRK_CLEAR_SETUP_MII (1 << 17) + +/* Some link partners do not tolerate the momentary reset of the REF_CLK + * frequency when the RNCTL register is cleared by hardware reset. + */ +#define FEC_QUIRK_NO_HARD_RESET (1 << 18) + +/* i.MX6SX ENET IP supports multiple queues (3 queues), use this quirk to + * represents this ENET IP. + */ +#define FEC_QUIRK_HAS_MULTI_QUEUES (1 << 19) + +/* i.MX8MQ ENET IP version add new feature to support IEEE 802.3az EEE + * standard. For the transmission, MAC supply two user registers to set + * Sleep (TS) and Wake (TW) time. + */ +#define FEC_QUIRK_HAS_EEE (1 << 20) + +/* i.MX8QM ENET IP version add new feture to generate delayed TXC/RXC + * as an alternative option to make sure it works well with various PHYs. + * For the implementation of delayed clock, ENET takes synchronized 250MHz + * clocks to generate 2ns delay. + */ +#define FEC_QUIRK_DELAYED_CLKS_SUPPORT (1 << 21) + + +/* i.MX8MQ SoC integration mix wakeup interrupt signal into "int2" interrupt line. */ +#define FEC_QUIRK_WAKEUP_FROM_INT2 (1 << 22) + +/* request pmqos during low power */ +#define FEC_QUIRK_HAS_PMQOS (1 << 23) + +struct bufdesc_prop { + int qid; + /* Address of Rx and Tx buffers */ + struct bufdesc *base; + struct bufdesc *last; + struct bufdesc *cur; + void __iomem *reg_desc_active; + dma_addr_t dma; + unsigned short ring_size; + unsigned char dsize; + unsigned char dsize_log2; +}; + +struct fec_enet_priv_tx_q { + struct bufdesc_prop bd; + struct bufdesc *dirty_tx; + struct sk_buff *tx_skbuff[FEC_TX_RING_SIZE]; + + unsigned int tx_bounce_size; + +#ifdef CONFIG_AVB_SUPPORT + unsigned int tx_index; + unsigned char *tx_bounce[FEC_TX_RING_SIZE + 32]; +#else + unsigned char *tx_bounce[FEC_TX_RING_SIZE]; +#endif + unsigned short tx_stop_threshold; + unsigned short tx_wake_threshold; + + char *tso_hdrs; + dma_addr_t tso_hdrs_dma; + +#ifdef CONFIG_AVB_SUPPORT + unsigned long tx_idle_slope; +#endif +}; + +struct fec_enet_priv_rx_q { + struct bufdesc_prop bd; + struct sk_buff *rx_skbuff[FEC_RX_RING_SIZE]; +}; + +struct fec_stop_mode_gpr { + struct regmap *gpr; + u8 reg; + u8 bit; +}; + +#ifdef CONFIG_AVB_SUPPORT +#define AVB_DMA_MAPPING 1 +#endif + +/* The FEC buffer descriptors track the ring buffers. The rx_bd_base and + * tx_bd_base always point to the base of the buffer descriptors. The + * cur_rx and cur_tx point to the currently available buffer. + * The dirty_tx tracks the current buffer that is being sent by the + * controller. The cur_tx and dirty_tx are equal under both completely + * empty and completely full conditions. The empty/ready indicator in + * the buffer descriptor determines the actual condition. + */ +struct fec_enet_private { + /* Hardware registers of the FEC device */ + void __iomem *hwp; + + struct net_device *netdev; + + struct clk *clk_ipg; + struct clk *clk_ahb; + struct clk *clk_ref; + struct clk *clk_enet_out; + struct clk *clk_ptp; + struct clk *clk_2x_txclk; + + bool ptp_clk_on; + struct mutex ptp_clk_mutex; + unsigned int num_tx_queues; + unsigned int num_rx_queues; + + /* The saved address of a sent-in-place packet/buffer, for skfree(). */ + struct fec_enet_priv_tx_q *tx_queue; + struct fec_enet_priv_rx_q *rx_queue; + + unsigned int total_tx_ring_size; + unsigned int total_rx_ring_size; + +#ifdef CONFIG_AVB_SUPPORT + const struct avb_ops *avb; + void *avb_data; + unsigned int avb_enabled; + __ETHTOOL_DECLARE_LINK_MODE_MASK(phy_advertising); +#endif + struct platform_device *pdev; + + int dev_id; + + /* Phylib and MDIO interface */ + struct mii_bus *mii_bus; + uint phy_speed; + phy_interface_t phy_interface; + struct device_node *phy_node; + bool rgmii_txc_dly; + bool rgmii_rxc_dly; + bool mii_bus_share; + bool rpm_active; + int link; + int full_duplex; + int speed; + int irq[FEC_IRQ_NUM]; + bool bufdesc_ex; + int pause_flag; + int wol_flag; + int wake_irq; + u32 quirks; + + struct napi_struct napi; + int csum_flags; + + struct work_struct tx_timeout_work; + + struct ptp_clock *ptp_clock; + struct ptp_clock_info ptp_caps; + unsigned long last_overflow_check; + raw_spinlock_t tmreg_lock; + struct cyclecounter cc; + struct timecounter tc; + int rx_hwtstamp_filter; + u32 base_incval; + u32 cycle_speed; + int hwts_rx_en; + int hwts_tx_en; + + /* Transmit and receive latency, depending on link speed, for + * packets timestamps in ns + */ + u32 rx_tstamp_latency; + u32 tx_tstamp_latency; + + struct delayed_work time_keep; + struct regulator *reg_phy; + struct fec_stop_mode_gpr stop_gpr; + struct pm_qos_request pm_qos_req; + + unsigned int tx_align; + unsigned int rx_align; + + /* hw interrupt coalesce */ + unsigned int rx_pkts_itr; + unsigned int rx_time_itr; + unsigned int tx_pkts_itr; + unsigned int tx_time_itr; + unsigned int itr_clk_rate; + + /* tx lpi eee mode */ + struct ethtool_eee eee; + unsigned int clk_ref_rate; + + u32 rx_copybreak; + + /* ptp clock period in ns*/ + unsigned int ptp_inc; + + /* pps */ + int pps_channel; + unsigned int reload_period; + int pps_enable; + unsigned int next_counter; + + struct mutex fast_ndev_lock; + struct imx_sc_ipc *ipc_handle; + + /* Configured rx/tx timestamps delays for different link speeds + * to compensate for FEC-PHY latency in ns + */ + u32 rx_delay_100; + u32 tx_delay_100; + u32 rx_delay_1000; + u32 tx_delay_1000; + +#ifdef CONFIG_AVB_SUPPORT + int rec_channel; + int rec_enable; +#endif + + u64 ethtool_stats[]; +}; + +#ifdef CONFIG_AVB_SUPPORT +#define FEC_MAX_RATE 400 /* Mbps */ +#define FEC_MAX_RATE_HAS_AVB 1000 /* Mbps */ + +static inline int fec_max_rate(struct fec_enet_private *fep) +{ + int max_rate = (fep->quirks & FEC_QUIRK_HAS_AVB) ? FEC_MAX_RATE_HAS_AVB : FEC_MAX_RATE; + return min(max_rate, fep->speed); +} + +#define IDLE_SLOPE_DIVISOR 512 +#endif + +void fec_ptp_init(struct platform_device *pdev, int irq_idx); +void fec_ptp_stop(struct platform_device *pdev); +void fec_ptp_start_cyclecounter(struct net_device *ndev); +void fec_ptp_disable_hwts(struct net_device *ndev); +int fec_ptp_set(struct net_device *ndev, struct ifreq *ifr); +int fec_ptp_get(struct net_device *ndev, struct ifreq *ifr); + +/****************************************************************************/ +#endif /* FEC_H */ diff --git a/drivers/net/ethernet/freescale/fec_main.c b/drivers/net/ethernet/freescale/fec_main.c index 23f7604d9dbc..290d6cfa7fc8 100644 --- a/drivers/net/ethernet/freescale/fec_main.c +++ b/drivers/net/ethernet/freescale/fec_main.c @@ -77,17 +77,25 @@ static void set_multicast_list(struct net_device *ndev); static void fec_enet_itr_coal_set(struct net_device *ndev); +#ifndef CONFIG_AVB_SUPPORT static int fec_enet_xdp_tx_xmit(struct net_device *ndev, struct xdp_buff *xdp); static int fec_enet_xdp_xmit(struct net_device *dev, int num_frames, struct xdp_frame **frames, u32 flags); +#endif #define DRIVER_NAME "fec" static const u16 fec_enet_vlan_pri_to_queue[8] = {0, 0, 1, 1, 1, 2, 2, 2}; +#ifdef CONFIG_AVB_SUPPORT +/* Idle Slope values specific to AVB capable boards */ +static const unsigned short idle_slope_values[] = {1, 2, 4, 8, 16, 32, 64, 128, + 256, 384, 512, 640, 768, 896, 1024, 1152, 1280, 1408, 1536}; +#endif + /* Pause frame feild and FIFO threshold */ #define FEC_ENET_FCE (1 << 5) #define FEC_ENET_RSEM_V 0x84 @@ -339,6 +347,24 @@ MODULE_PARM_DESC(macaddr, "FEC Ethernet MAC address"); static int mii_cnt; +#ifdef CONFIG_AVB_SUPPORT +static inline void read16(void *dst, void *src) +{ +#ifdef CONFIG_ARM64 + asm volatile ( "ldp x10, x11, [%1]\n\t" + "stp x10, x11, [%0]\n\t" + : : "r" (dst), "r" (src) : "x10", "x11", "memory"); +#elif CONFIG_ARM + asm volatile ( "ldmia %1, {r5-r8}\n\t" + "stmia %0, {r5-r8}\n\t" + : : "r" (dst), "r" (src) : "r5", "r6", "r7", "r8", "memory"); +#else + ((u64 *)dst)[0] = ((u64 *)src)[0]; + ((u64 *)dst)[1] = ((u64 *)src)[1]; +#endif +} +#endif /*CONFIG_AVB_SUPPORT*/ + static struct bufdesc *fec_enet_get_nextdesc(struct bufdesc *bdp, struct bufdesc_prop *bd) { @@ -470,6 +496,253 @@ fec_enet_create_page_pool(struct fec_enet_private *fep, return err; } +#ifdef CONFIG_AVB_SUPPORT +/* + * Sends an AVB buffer on the network. + */ +int fec_enet_start_xmit_avb(void *data, struct avb_tx_desc *desc) +{ + struct fec_enet_private *fep = data; + struct bufdesc *bdp; + unsigned short status; + unsigned int index; + struct bufdesc_ex *ebdp; + unsigned long cbd_esc; + unsigned short queue_id = desc->queue_id; + struct fec_enet_priv_tx_q *txq = fep->tx_queue[queue_id]; + + /* ring buffer base address */ + /* registers base address */ + /* current descriptor pointer */ + bdp = txq->bd.cur; + + if (bdp == txq->dirty_tx) + return -2; + + status = fec16_to_cpu(bdp->cbd_sc); + + if (status & BD_ENET_TX_READY) + return -2; + + /* Clear all of the status flags */ + status &= ~BD_ENET_TX_STATS; + + index = fec_enet_get_bd_index(bdp, &txq->bd); + + /* Save desc pointer */ + txq->tx_buf[index].skb = (void *)desc; + + bdp->cbd_datlen = cpu_to_fec16(desc->common.len); + bdp->cbd_bufaddr = cpu_to_fec32(desc->dma_addr); + + ebdp = (struct bufdesc_ex *)bdp; + + ebdp->cbd_bdu = cpu_to_fec32(0); + + if (desc->common.flags & AVB_TX_FLAG_HW_TS) + cbd_esc = BD_ENET_TX_TS | desc->esc; + else + cbd_esc = desc->esc; + + if (desc->common.flags & AVB_TX_FLAG_HW_CSUM) + cbd_esc |= BD_ENET_TX_PINS | BD_ENET_TX_IINS; + + if (fep->quirks & FEC_QUIRK_HAS_AVB) + cbd_esc |= FEC_TX_BD_FTYPE(txq->bd.qid); + + ebdp->cbd_esc = cpu_to_fec32(cbd_esc); + + wmb(); + + bdp->cbd_sc = cpu_to_fec16(status | (BD_ENET_TX_READY | BD_ENET_TX_LAST | BD_ENET_TX_TC)); + + /* If this was the last BD in the ring, start at the beginning again. */ + bdp = fec_enet_get_nextdesc(bdp, &txq->bd); + + txq->bd.cur = bdp; + + /* Trigger transmission start */ + if (!(fep->quirks & FEC_QUIRK_ERR006358)) + if (!(fep->quirks & FEC_QUIRK_ERR007885) || + !readl(txq->bd.reg_desc_active) || + !readl(txq->bd.reg_desc_active) || + !readl(txq->bd.reg_desc_active) || + !readl(txq->bd.reg_desc_active)) + writel(0, txq->bd.reg_desc_active); + + if (bdp == txq->dirty_tx) + return -1; + + return 0; +} +EXPORT_SYMBOL(fec_enet_start_xmit_avb); + +void fec_enet_finish_xmit_avb(void *data, unsigned int queue_id) +{ + struct fec_enet_private *fep = data; + + /* Trigger transmission start */ + if (fep->quirks & FEC_QUIRK_ERR006358) + if (!(fep->quirks & FEC_QUIRK_ERR007885) || + !readl(fep->tx_queue[queue_id]->bd.reg_desc_active) || + !readl(fep->tx_queue[queue_id]->bd.reg_desc_active) || + !readl(fep->tx_queue[queue_id]->bd.reg_desc_active) || + !readl(fep->tx_queue[queue_id]->bd.reg_desc_active)) + writel(0, fep->tx_queue[queue_id]->bd.reg_desc_active); +} +EXPORT_SYMBOL(fec_enet_finish_xmit_avb); + +/* + * When AVB is enabled, it is the transmit function for the regular + * network traffic. It does not support any SG/TSO skb. + * Frames are posted to the AVB module for further scheduling. + */ +static int fec_enet_start_xmit_best_effort(struct fec_enet_priv_tx_q *txq, + struct netdev_queue *nq, struct sk_buff *skb, struct net_device *ndev) +{ + struct fec_enet_private *fep = netdev_priv(ndev); + void *bufaddr; + struct avb_tx_desc *desc; + int rc; + + if (fep->avb->tx_full(fep->avb_data)) { + netdev_err(ndev, "tx queue full!\n"); + return NETDEV_TX_BUSY; + } + + /* Protocol checksum off-load for TCP and UDP. */ + if (fec_enet_clear_csum(skb, ndev)) { + dev_kfree_skb_any(skb); + ndev->stats.tx_dropped++; + return NETDEV_TX_OK; + } + + if (skb_headroom(skb) < sizeof(struct avb_tx_desc)) { + if (pskb_expand_head(skb, sizeof(struct avb_tx_desc), 0, GFP_ATOMIC)) { + dev_kfree_skb_any(skb); + ndev->stats.tx_dropped++; + return NETDEV_TX_OK; + } + } + + bufaddr = skb->data; + desc = (struct avb_tx_desc *)(skb->data - sizeof(struct avb_tx_desc)); + + /* Set buffer length and buffer pointer */ + desc->common.offset = sizeof(struct avb_tx_desc); + desc->common.len = skb->len; + desc->queue_id = txq->bd.qid; + + if ((((unsigned long) bufaddr) & fep->tx_align) || + (fep->quirks & FEC_QUIRK_SWAP_FRAME)) { + memcpy(txq->tx_bounce[txq->tx_index], skb->data, skb->len); + bufaddr = txq->tx_bounce[txq->tx_index]; + + txq->tx_index++; + if (txq->tx_index >= txq->tx_bounce_size) + txq->tx_index = 0; + + if (fep->quirks & FEC_QUIRK_SWAP_FRAME) + swap_buffer(bufaddr, skb->len); + } + + /* Save skb pointer */ + desc->data = skb; + desc->common.flags = AVB_TX_FLAG_SKB; + + /* Push the data cache so the CPM does not get stale memory data. */ + desc->dma_addr = dma_map_single(&fep->pdev->dev, bufaddr, skb->len, DMA_TO_DEVICE); + if (dma_mapping_error(&fep->pdev->dev, desc->dma_addr)) { + dev_kfree_skb_any(skb); + if (net_ratelimit()) + netdev_err(ndev, "Tx DMA memory map failed\n"); + return NETDEV_TX_OK; + } + + desc->esc = 0; + + if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP && + fep->hwts_tx_en)) { + skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; + desc->esc |= BD_ENET_TX_TS; + } + + if (skb->ip_summed == CHECKSUM_PARTIAL) + desc->esc |= BD_ENET_TX_PINS | BD_ENET_TX_IINS; + + skb_tx_timestamp(skb); + + if ((rc = fep->avb->tx(fep->avb_data, desc)) < 0) { + if (rc < -1) { + kfree_skb(skb); + ndev->stats.tx_dropped++; + } + + netif_tx_stop_queue(nq); + return NETDEV_TX_OK; + } + + return NETDEV_TX_OK; +} + +int fec_enet_set_idle_slope(void *data, unsigned int queue_id, + u32 desired_rate) +{ + struct fec_enet_private *fep = data; + struct fec_enet_priv_tx_q *txq; + u64 idle_slope; + u32 line_rate; + int i; + + if (!fep) + return -EINVAL; + + /* Nothing to be done for non-AVB boards */ + if (!(fep->quirks & FEC_QUIRK_HAS_AVB)) + return 0; + + if ((queue_id == 0) || (queue_id >= fep->num_tx_queues)) + return -EINVAL; + + if ((fep->speed != SPEED_100) && (fep->speed != SPEED_1000)) + return -EOPNOTSUPP; + + line_rate = fep->speed * 1000000ULL; + if (desired_rate > line_rate) + return -EINVAL; + + txq = fep->tx_queue[queue_id]; + + /* + * Compute the desired Idle-Slope based on the desired rate and use + * the round up to the next integer. + */ + idle_slope = (u64)desired_rate * IDLE_SLOPE_DIVISOR + line_rate - desired_rate - 1; + idle_slope = div_u64(idle_slope, line_rate - desired_rate); + + for (i = 0; i < ARRAY_SIZE(idle_slope_values); i++) + if (idle_slope <= idle_slope_values[i]) + break; + + if (i >= ARRAY_SIZE(idle_slope_values)) + return -EINVAL; + /* + * If the desired rate is higher than the last available bandwidth + * threshold then we should not configure the Credit-Based shaper + * at all. + */ + if (idle_slope > idle_slope_values[i]) + return -EINVAL; + + txq->tx_idle_slope = idle_slope_values[i]; + writel(DMA_CLASS_EN | txq->tx_idle_slope, + fep->hwp + FEC_DMA_CFG(queue_id)); + + return 0; +} +EXPORT_SYMBOL(fec_enet_set_idle_slope); +#endif + static struct bufdesc * fec_enet_txq_submit_frag_skb(struct fec_enet_priv_tx_q *txq, struct sk_buff *skb, @@ -681,7 +954,12 @@ static int fec_enet_txq_submit_skb(struct fec_enet_priv_tx_q *txq, txq->bd.cur = bdp; /* Trigger transmission start */ - writel(0, txq->bd.reg_desc_active); + if (!(fep->quirks & FEC_QUIRK_ERR007885) || + !readl(txq->bd.reg_desc_active) || + !readl(txq->bd.reg_desc_active) || + !readl(txq->bd.reg_desc_active) || + !readl(txq->bd.reg_desc_active)) + writel(0, txq->bd.reg_desc_active); return 0; } @@ -898,6 +1176,11 @@ fec_enet_start_xmit(struct sk_buff *skb, struct net_device *ndev) txq = fep->tx_queue[queue]; nq = netdev_get_tx_queue(ndev, queue); +#ifdef CONFIG_AVB_SUPPORT + if (fep->avb_enabled) + return fec_enet_start_xmit_best_effort(txq, nq, skb, ndev); +#endif + if (skb_is_gso(skb)) ret = fec_enet_txq_submit_tso(txq, skb, ndev); else @@ -922,6 +1205,9 @@ static void fec_enet_bd_init(struct net_device *dev) struct bufdesc *bdp; unsigned int i; unsigned int q; +#ifdef CONFIG_AVB_SUPPORT + struct sk_buff *skb; +#endif for (q = 0; q < fep->num_rx_queues; q++) { /* Initialize the receive buffer descriptors. */ @@ -945,6 +1231,23 @@ static void fec_enet_bd_init(struct net_device *dev) rxq->bd.cur = rxq->bd.base; } +#ifdef CONFIG_AVB_SUPPORT + if (fep->avb_enabled) { + struct avb_tx_desc *desc; + + while ((desc = fep->avb->tx_cleanup_dequeue(fep->avb_data)) != (void *) -1) { + + skb = desc->data; + + dma_unmap_single(&fep->pdev->dev, desc->bufaddr, + skb->len, DMA_TO_DEVICE); + + /* Free the sk buffer associated with this last transmit */ + dev_kfree_skb_any(skb); + } + } +#endif + for (q = 0; q < fep->num_tx_queues; q++) { /* ...and the same for transmit */ txq = fep->tx_queue[q]; @@ -954,6 +1257,42 @@ static void fec_enet_bd_init(struct net_device *dev) for (i = 0; i < txq->bd.ring_size; i++) { /* Initialize the BD for every fragment in the page. */ bdp->cbd_sc = cpu_to_fec16(0); +#ifdef CONFIG_AVB_SUPPORT + /* AVB not compatible with XDP: all buffers are either avb descriptors or skbs. */ + if (txq->tx_buf[i].skb) { + skb = NULL; + if (fep->avb_enabled) { + struct avb_tx_desc *desc = (struct avb_tx_desc *)txq->tx_buf[i].skb; + + if (!(desc->common.flags & AVB_TX_FLAG_SKB)) { + fep->avb->free(fep->avb_data, &desc->common); + /* Avoid unmapping AVB buffers below */ + bdp->cbd_bufaddr = cpu_to_fec32(0); + } else { + skb = desc->data; + } + } else + skb = txq->tx_buf[i].skb; + + if (skb) { + dev_kfree_skb_any(skb); + } + + txq->tx_buf[i].skb = NULL; + } + + if (bdp->cbd_bufaddr && + !IS_TSO_HEADER(txq, fec32_to_cpu(bdp->cbd_bufaddr))) + dma_unmap_single(&fep->pdev->dev, + fec32_to_cpu(bdp->cbd_bufaddr), + fec16_to_cpu(bdp->cbd_datlen), + DMA_TO_DEVICE); + if (txq->tx_buf[i].skb) { + dev_kfree_skb_any(txq->tx_buf[i].skb); + txq->tx_buf[i].skb = NULL; + } + +#else if (txq->tx_buf[i].type == FEC_TXBUF_T_SKB) { if (bdp->cbd_bufaddr && !IS_TSO_HEADER(txq, fec32_to_cpu(bdp->cbd_bufaddr))) @@ -981,7 +1320,7 @@ static void fec_enet_bd_init(struct net_device *dev) txq->tx_buf[i].type = FEC_TXBUF_T_SKB; } - +#endif bdp->cbd_bufaddr = cpu_to_fec32(0); bdp = fec_enet_get_nextdesc(bdp, &txq->bd); } @@ -1007,6 +1346,7 @@ static void fec_enet_enable_ring(struct net_device *ndev) struct fec_enet_private *fep = netdev_priv(ndev); struct fec_enet_priv_tx_q *txq; struct fec_enet_priv_rx_q *rxq; + unsigned long idle_slope; int i; for (i = 0; i < fep->num_rx_queues; i++) { @@ -1025,10 +1365,25 @@ static void fec_enet_enable_ring(struct net_device *ndev) writel(txq->bd.dma, fep->hwp + FEC_X_DES_START(i)); /* enable DMA1/2 */ - if (i) - writel(DMA_CLASS_EN | IDLE_SLOPE(i), + if (i) { +#ifdef CONFIG_AVB_SUPPORT + idle_slope = txq->tx_idle_slope; +#else + idle_slope = IDLE_SLOPE(i); +#endif + + writel(DMA_CLASS_EN | idle_slope, fep->hwp + FEC_DMA_CFG(i)); + } } + + /* + * For AVB capable devices we should enable RX flushing for the + * best effort queue (ring 0) and also the TX credit based shaper. + */ + if (fep->quirks & FEC_QUIRK_HAS_AVB) + writel(FEC_RX_FLUSH(0) | FEC_TX_SCHEME_CB, + fep->hwp + FEC_QOS_SCHEME); } /* @@ -1044,17 +1399,12 @@ fec_restart(struct net_device *ndev) u32 rcntl = OPT_FRAME_SIZE | 0x04; u32 ecntl = 0x2; /* ETHEREN */ - /* Whack a reset. We should wait for this. - * For i.MX6SX SOC, enet use AXI bus, we use disable MAC - * instead of reset MAC itself. + /* Always use disable MAC instead of MAC reset to: + * - Keep the ENET counter running + * - Avoid dead system bus for SoCs using the ENET-AXI bus + * and not the AHB bus, like the i.MX6SX */ - if (fep->quirks & FEC_QUIRK_HAS_MULTI_QUEUES || - ((fep->quirks & FEC_QUIRK_NO_HARD_RESET) && fep->link)) { - writel(0, fep->hwp + FEC_ECNTRL); - } else { - writel(1, fep->hwp + FEC_ECNTRL); - udelay(10); - } + writel(0, fep->hwp + FEC_ECNTRL); /* * enet-mac reset will reset mac address registers too, @@ -1211,9 +1561,6 @@ fec_restart(struct net_device *ndev) writel(ecntl, fep->hwp + FEC_ECNTRL); fec_enet_active_rxring(ndev); - if (fep->bufdesc_ex) - fec_ptp_start_cyclecounter(ndev); - /* Enable interrupts we wish to service */ if (fep->link) writel(FEC_DEFAULT_IMASK, fep->hwp + FEC_IMASK); @@ -1303,17 +1650,14 @@ fec_stop(struct net_device *ndev) netdev_err(ndev, "Graceful transmit stop did not complete!\n"); } - /* Whack a reset. We should wait for this. - * For i.MX6SX SOC, enet use AXI bus, we use disable MAC - * instead of reset MAC itself. - */ if (!(fep->wol_flag & FEC_WOL_FLAG_SLEEP_ON)) { - if (fep->quirks & FEC_QUIRK_HAS_MULTI_QUEUES) { - writel(0, fep->hwp + FEC_ECNTRL); - } else { - writel(1, fep->hwp + FEC_ECNTRL); - udelay(10); - } + /* Always use disable MAC instead of MAC reset to: + * - Keep the ENET counter running + * - Avoid dead system bus for SoCs using the ENET-AXI bus + * and not the AHB bus, like the i.MX6SX + */ + writel(0, fep->hwp + FEC_ECNTRL); + } else { val = readl(fep->hwp + FEC_ECNTRL); val |= (FEC_ECR_MAGICEN | FEC_ECR_SLEEP); @@ -1353,7 +1697,15 @@ static void fec_enet_timeout_work(struct work_struct *work) if (netif_device_present(ndev) || netif_running(ndev)) { napi_disable(&fep->napi); netif_tx_lock_bh(ndev); +#ifdef CONFIG_AVB_SUPPORT + if (fep->avb_enabled) + fep->avb->close(fep->avb_data); +#endif fec_restart(ndev); +#ifdef CONFIG_AVB_SUPPORT + if (fep->avb_enabled) + fep->avb->open(fep->avb_data, fep, fec_max_rate(fep)); +#endif netif_tx_wake_all_queues(ndev); netif_tx_unlock_bh(ndev); napi_enable(&fep->napi); @@ -1368,14 +1720,194 @@ fec_enet_hwtstamp(struct fec_enet_private *fep, unsigned ts, unsigned long flags; u64 ns; - spin_lock_irqsave(&fep->tmreg_lock, flags); + raw_spin_lock_irqsave(&fep->tmreg_lock, flags); ns = timecounter_cyc2time(&fep->tc, ts); - spin_unlock_irqrestore(&fep->tmreg_lock, flags); + raw_spin_unlock_irqrestore(&fep->tmreg_lock, flags); memset(hwtstamps, 0, sizeof(*hwtstamps)); hwtstamps->hwtstamp = ns_to_ktime(ns); } +#ifdef CONFIG_AVB_SUPPORT +static int +fec_enet_tx_queue_avb(struct net_device *ndev, u16 queue_id) +{ + struct fec_enet_private *fep = netdev_priv(ndev); + struct bufdesc *bdp; + struct bufdesc_ex local_ebdp; + struct avb_tx_desc *desc; + unsigned short status; + struct fec_enet_priv_tx_q *txq; + struct netdev_queue *nq; + int index = 0; + int rc = 0; + unsigned int total_tx_packets = 0; + unsigned int total_tx_bytes = 0; + u16 tx_tstamp_latency = fep->tx_tstamp_latency; + + txq = fep->tx_queue[queue_id]; + nq = netdev_get_tx_queue(ndev, queue_id); + + bdp = txq->dirty_tx; + + /* get next bdp of dirty_tx */ + bdp = fec_enet_get_nextdesc(bdp, &txq->bd); + + while (bdp != READ_ONCE(txq->bd.cur)) { + /* Order the load of cur_tx and cbd_sc */ + rmb(); + + /* Read the first 16 bytes of the descriptor at once to avoid + * multiple reads of non cacheable memory from RAM */ + read16(&local_ebdp, bdp); + + status = fec16_to_cpu(local_ebdp.desc.cbd_sc); + if (status & BD_ENET_TX_READY) + break; + + index = fec_enet_get_bd_index(bdp, &txq->bd); + desc = (struct avb_tx_desc *)txq->tx_buf[index].skb; + + if (!(desc->common.flags & AVB_TX_FLAG_SKB)) { + if ((desc->common.flags & AVB_TX_FLAG_HW_TS)) { + struct bufdesc_ex *ebdp = (struct bufdesc_ex *)bdp; + + desc->common.ts = ebdp->ts + tx_tstamp_latency; + + /* upper layer will retrieve the timestamp and free the descriptor */ + rc |= fep->avb->tx_ts(fep->avb_data, &desc->common); + } + else + fep->avb->free(fep->avb_data, &desc->common); + + total_tx_packets++; + total_tx_bytes += desc->datlen; + } else { + /* Backup hardware descriptor fields in software descriptor */ + desc->sc = status; + desc->datlen = fec16_to_cpu(local_ebdp.desc.cbd_datlen); + desc->bufaddr = fec32_to_cpu(local_ebdp.desc.cbd_bufaddr); + desc->common.ts = fec32_to_cpu(((struct bufdesc_ex *)bdp)->ts); + + if (fep->avb->tx_cleanup(fep->avb_data, desc) < 0) + BUG(); + } + + txq->tx_buf[index].skb = NULL; + bdp->cbd_bufaddr = cpu_to_fec32(0); + txq->dirty_tx = bdp; + + /* Update pointer to next buffer descriptor to be transmitted */ + bdp = fec_enet_get_nextdesc(bdp, &txq->bd); + } + + /* schedule tx napi, based on level of tx cleanup queue or time passed */ + if (fep->avb->tx_cleanup_ready(fep->avb_data) || netif_tx_queue_stopped(nq)) { + if (napi_schedule_prep(&fep->napi)) { + __napi_schedule(&fep->napi); + } + } + + /* Update stats*/ + ndev->stats.tx_packets += total_tx_packets; + ndev->stats.tx_bytes += total_tx_bytes; + + return rc; +} + +int +fec_enet_tx_avb(void *data) +{ + struct fec_enet_private *fep = data; + u16 queue_id; + int rc = 0; + + for (queue_id = 0; queue_id < fep->num_tx_queues; queue_id++) + rc |= fec_enet_tx_queue_avb(fep->netdev, queue_id); + + return rc; +} +EXPORT_SYMBOL(fec_enet_tx_avb); + +static void +fec_enet_tx_best_effort(struct net_device *ndev) +{ + struct fec_enet_private *fep; + unsigned short status; + struct avb_tx_desc *desc; + struct sk_buff *skb; + struct fec_enet_priv_tx_q *txq; + struct netdev_queue *nq; + + fep = netdev_priv(ndev); + + while ((desc = fep->avb->tx_cleanup_dequeue(fep->avb_data)) != (void *) -1) { + + txq = fep->tx_queue[desc->queue_id]; + nq = netdev_get_tx_queue(ndev, desc->queue_id); + + status = desc->sc; + + /* Check for errors. */ + if (status & (BD_ENET_TX_HB | BD_ENET_TX_LC | + BD_ENET_TX_RL | BD_ENET_TX_UN | + BD_ENET_TX_CSL)) { + ndev->stats.tx_errors++; + if (status & BD_ENET_TX_HB) /* No heartbeat */ + ndev->stats.tx_heartbeat_errors++; + if (status & BD_ENET_TX_LC) /* Late collision */ + ndev->stats.tx_window_errors++; + if (status & BD_ENET_TX_RL) /* Retrans limit */ + ndev->stats.tx_aborted_errors++; + if (status & BD_ENET_TX_UN) /* Underrun */ + ndev->stats.tx_fifo_errors++; + if (status & BD_ENET_TX_CSL) /* Carrier lost */ + ndev->stats.tx_carrier_errors++; + } else { + ndev->stats.tx_packets++; + ndev->stats.tx_bytes += desc->datlen; + } + + skb = desc->data; + + if (!IS_TSO_HEADER(txq, desc->bufaddr)) + dma_unmap_single(&fep->pdev->dev, desc->bufaddr, + desc->datlen, DMA_TO_DEVICE); + + if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS)) { + struct skb_shared_hwtstamps shhwtstamps; + + desc->common.ts += fep->tx_tstamp_latency; + fec_enet_hwtstamp(fep, desc->common.ts, + &shhwtstamps); + skb_tstamp_tx(skb, &shhwtstamps); + } + + /* Deferred means some collisions occurred during transmit, + * but we eventually sent the packet OK. + */ + if (status & BD_ENET_TX_DEF) + ndev->stats.collisions++; + + /* Free the sk buffer associated with this last transmit */ + dev_kfree_skb_any(skb); + + /* Make sure the update to bdp and tx_buf are performed + * before dirty_tx + */ + wmb(); + + //FIXME add treshold + if (!fep->avb->tx_full(fep->avb_data)) { + if (netif_tx_queue_stopped(nq)) { + //` netdev_info(ndev, "wake queue\n"); + netif_tx_wake_queue(nq); + } + } + } +} +#endif + static void fec_enet_tx_queue(struct net_device *ndev, u16 queue_id) { @@ -1474,6 +2006,11 @@ fec_enet_tx_queue(struct net_device *ndev, u16 queue_id) struct bufdesc_ex *ebdp = (struct bufdesc_ex *)bdp; fec_enet_hwtstamp(fep, fec32_to_cpu(ebdp->ts), &shhwtstamps); + /* Adjust for TX MAC-PHY latency + */ + shhwtstamps.hwtstamp = + ktime_add_ns(shhwtstamps.hwtstamp, fep->tx_tstamp_latency); + skb_tstamp_tx(skb, &shhwtstamps); } @@ -1538,6 +2075,7 @@ static void fec_enet_update_cbd(struct fec_enet_priv_rx_q *rxq, bdp->cbd_bufaddr = cpu_to_fec32(phys_addr); } +#ifndef CONFIG_AVB_SUPPORT static u32 fec_enet_run_xdp(struct fec_enet_private *fep, struct bpf_prog *prog, struct xdp_buff *xdp, struct fec_enet_priv_rx_q *rxq, int index) @@ -1572,32 +2110,302 @@ fec_enet_run_xdp(struct fec_enet_private *fep, struct bpf_prog *prog, } break; - default: - bpf_warn_invalid_xdp_action(fep->netdev, prog, act); - fallthrough; + default: + bpf_warn_invalid_xdp_action(fep->netdev, prog, act); + fallthrough; + + case XDP_TX: + ret = fec_enet_xdp_tx_xmit(fep->netdev, xdp); + if (ret == FEC_ENET_XDP_CONSUMED) { + page = virt_to_head_page(xdp->data); + page_pool_put_page(rxq->page_pool, page, sync, true); + } else { + ret = FEC_ENET_XDP_TX; + } + break; + case XDP_ABORTED: + fallthrough; /* handle aborts by dropping packet */ + + case XDP_DROP: + rxq->stats[RX_XDP_DROP]++; + ret = FEC_ENET_XDP_CONSUMED; + page = virt_to_head_page(xdp->data); + page_pool_put_page(rxq->page_pool, page, sync, true); + break; + } + + return ret; +} +#else /* CONFIG_AVB_SUPPORT */ + +static int +fec_enet_rx_best_effort(struct net_device *ndev, int budget) +{ + struct fec_enet_private *fep = netdev_priv(ndev); + struct avb_rx_desc *desc; + struct sk_buff *skb; + ushort pkt_len; + __u8 *data; + bool vlan_packet_rcvd = false; + u16 vlan_tag; + int pkt_received = 0; + + do { + desc = fep->avb->dequeue(fep->avb_data); + if (desc == (void *)-1) + break; + + /* Process the incoming frame. */ + pkt_len = desc->common.len; + data = (u8 *)desc + desc->common.offset; + + skb = netdev_alloc_skb(ndev, pkt_len - 4); + if (unlikely(!skb)) { + ndev->stats.rx_dropped++; + goto rx_processing_done; + } + else { + /* Make some room minus FCS */ + skb_put(skb, pkt_len - 4); + + /* Copy AVB buffer to skb */ + skb_copy_to_linear_data(skb, data, pkt_len - 4); + data = skb->data; + + /* Get receive timestamp from the skb */ + if (fep->hwts_rx_en) { + skb_reset_mac_header(skb); + fec_enet_hwtstamp(fep, desc->common.ts, + skb_hwtstamps(skb)); + } + + /* If this is a VLAN packet remove the VLAN Tag */ + vlan_packet_rcvd = false; + if (desc->common.private & BD_ENET_RX_VLAN) { + /* Push and remove the vlan tag */ + struct vlan_hdr *vlan_header = + (struct vlan_hdr *) (data + ETH_HLEN); + vlan_tag = ntohs(vlan_header->h_vlan_TCI); + + vlan_packet_rcvd = true; + + memmove(skb->data + VLAN_HLEN, data, ETH_ALEN * 2); + + skb_pull(skb, VLAN_HLEN); + } + + skb->protocol = eth_type_trans(skb, ndev); + + if (fep->csum_flags & FLAG_RX_CSUM_ENABLED) { + if (!(desc->common.private & FLAG_RX_CSUM_ERROR)) { + /* don't check it */ + skb->ip_summed = CHECKSUM_UNNECESSARY; + } else { + skb_checksum_none_assert(skb); + } + } + + /* Handle received VLAN packets */ + if (vlan_packet_rcvd) + __vlan_hwaccel_put_tag(skb, + htons(ETH_P_8021Q), + vlan_tag); + + napi_gro_receive(&fep->napi, skb); + } +rx_processing_done: + fep->avb->free(fep->avb_data, &desc->common); + + } while (++pkt_received < budget); + + return pkt_received; +} + +static unsigned int +fec_enet_rx_queue_avb(struct net_device *ndev, u16 queue_id) +{ + struct fec_enet_private *fep = netdev_priv(ndev); + struct fec_enet_priv_rx_q *rxq; + struct bufdesc *bdp; + unsigned short status; + ushort pkt_len; + __u8 *data, *new_data; + int index = 0; + struct bufdesc_ex *ebdp = NULL; + struct bufdesc_ex local_ebdp; + struct avb_rx_desc *desc; + unsigned int rc = 0; + unsigned int net_data_offset; + unsigned int count = 0; + unsigned int total_rx_packets = 0; + unsigned int total_rx_bytes = 0; + u16 rx_tstamp_latency = fep->rx_tstamp_latency; + bool need_swap = fep->quirks & FEC_QUIRK_SWAP_FRAME; + bool has_racc = fep->quirks & FEC_QUIRK_HAS_RACC; + + rxq = fep->rx_queue[queue_id]; + + /* First, grab all of the stats for the incoming packet. + * These get messed up if we get called due to a busy condition. + */ + bdp = rxq->bd.cur; + + /* 20 packets per 125us > 64 bytes packets @ 100Mbps */ + while (!((status = fec16_to_cpu(bdp->cbd_sc)) & BD_ENET_RX_EMPTY) && (count++ < 20)) { + + writel(FEC_ENET_RXF, fep->hwp + FEC_IEVENT); + + /* Read the first 16 bytes of the descriptor at once to avoid + * multiple reads of non cacheable memory from RAM */ + read16(&local_ebdp, bdp); + + /* Check for errors. */ + status ^= BD_ENET_RX_LAST; + if (unlikely(status & (BD_ENET_RX_LG | BD_ENET_RX_SH | BD_ENET_RX_NO | + BD_ENET_RX_CR | BD_ENET_RX_OV | BD_ENET_RX_LAST | + BD_ENET_RX_CL))) { + ndev->stats.rx_errors++; + if (status & BD_ENET_RX_OV) { + /* FIFO overrun */ + ndev->stats.rx_fifo_errors++; + goto rx_processing_done; + } + if (status & (BD_ENET_RX_LG | BD_ENET_RX_SH + | BD_ENET_RX_LAST)) { + /* Frame too long or too short. */ + ndev->stats.rx_length_errors++; + if (status & BD_ENET_RX_LAST) + netdev_err(ndev, "rcv is not +last\n"); + } + if (status & BD_ENET_RX_CR) /* CRC Error */ + ndev->stats.rx_crc_errors++; + /* Report late collisions as a frame error. */ + if (status & (BD_ENET_RX_NO | BD_ENET_RX_CL)) + ndev->stats.rx_frame_errors++; + goto rx_processing_done; + } + + new_data = fep->avb->alloc(fep->avb_data); + if (unlikely(!new_data)) { + ndev->stats.rx_dropped++; + goto rx_processing_done; + } + + /* Process the incoming frame. */ + total_rx_packets++; + pkt_len = fec16_to_cpu(local_ebdp.desc.cbd_datlen); + total_rx_bytes += pkt_len; + index = fec_enet_get_bd_index(bdp, &rxq->bd); + data = (__u8 *)rxq->rx_skb_info[index].skb; + + /* FIXME, skip unmap of audio data */ + dma_sync_single_for_cpu(&fep->pdev->dev, fec32_to_cpu(local_ebdp.desc.cbd_bufaddr), + L1_CACHE_ALIGN(pkt_len), DMA_FROM_DEVICE); + + desc = (struct avb_rx_desc *)data; + + net_data_offset = desc->common.offset; + +#if !defined(CONFIG_M5272) + if (has_racc) + net_data_offset -= 2; +#endif + prefetch(data + net_data_offset); + + if (need_swap) + swap_buffer(data, pkt_len); + + desc->common.len = pkt_len; + desc->sc = fec16_to_cpu(local_ebdp.desc.cbd_sc); + + /* Extract the enhanced buffer descriptor */ + ebdp = (struct bufdesc_ex *)bdp; + + desc->common.ts = ebdp->ts - rx_tstamp_latency; + desc->common.private = fec32_to_cpu(local_ebdp.cbd_esc); + + rc |= fep->avb->rx(fep->avb_data, desc); + + data = new_data; + + desc = (struct avb_rx_desc *)data; + + desc->common.len = 0; + desc->queue_id = queue_id; + + bdp->cbd_bufaddr = cpu_to_fec32((dma_addr_t)(desc->dma_addr)); + +#if !defined(CONFIG_M5272) + if (has_racc) + desc->common.offset += 2; +#endif + rxq->rx_skb_info[index].skb = (void *)data; +rx_processing_done: + /* Clear the status flags for this buffer */ + status &= ~BD_ENET_RX_STATS; + + /* Mark the buffer empty */ + status |= BD_ENET_RX_EMPTY; + bdp->cbd_sc = cpu_to_fec16(status); + + ebdp = (struct bufdesc_ex *)bdp; + + ebdp->cbd_esc = cpu_to_fec32(0); + ebdp->cbd_prot = cpu_to_fec32(0); + ebdp->cbd_bdu = cpu_to_fec32(0); + + /* Update BD pointer to next entry */ + + bdp = fec_enet_get_nextdesc(bdp, &rxq->bd); + } + + /* If the receive ring buffer can hold at least double the maximum + number of packets per polling period (18.2 packets @ 100Mbps), it's + ok to only re-enable receive after processing all current packets */ + + writel(0, rxq->bd.reg_desc_active); + + rxq->bd.cur = bdp; + + /*Update stats*/ + ndev->stats.rx_packets += total_rx_packets; + ndev->stats.rx_bytes += total_rx_bytes; + + return rc; +} + +static unsigned int +fec_enet_rx_avb(struct net_device *ndev) +{ + u16 queue_id; + struct fec_enet_private *fep = netdev_priv(ndev); + unsigned int rc = 0; + + for (queue_id = 0; queue_id < fep->num_rx_queues; queue_id++) + rc |= fec_enet_rx_queue_avb(ndev, queue_id); + + return rc; +} + +int fec_enet_rx_poll_avb(void *data) +{ + struct fec_enet_private *fep = data; + struct net_device *ndev = fep->netdev; + unsigned int rc; - case XDP_TX: - ret = fec_enet_xdp_tx_xmit(fep->netdev, xdp); - if (ret == FEC_ENET_XDP_CONSUMED) { - page = virt_to_head_page(xdp->data); - page_pool_put_page(rxq->page_pool, page, sync, true); - } else { - ret = FEC_ENET_XDP_TX; - } - break; - case XDP_ABORTED: - fallthrough; /* handle aborts by dropping packet */ + rc = fec_enet_rx_avb(ndev); - case XDP_DROP: - rxq->stats[RX_XDP_DROP]++; - ret = FEC_ENET_XDP_CONSUMED; - page = virt_to_head_page(xdp->data); - page_pool_put_page(rxq->page_pool, page, sync, true); - break; + if (rc & AVB_WAKE_NAPI) { + /* Best effort packets were posted, schedule napi if not scheduled yet. */ + if (napi_schedule_prep(&fep->napi)) + __napi_schedule(&fep->napi); } - return ret; + return rc; } +EXPORT_SYMBOL(fec_enet_rx_poll_avb); + +#endif /* CONFIG_AVB_SUPPORT */ /* During a receive, the bd_rx.cur points to the current incoming buffer. * When we update through the ring, if the next incoming buffer has @@ -1620,10 +2428,12 @@ fec_enet_rx_queue(struct net_device *ndev, int budget, u16 queue_id) u16 vlan_tag; int index = 0; bool need_swap = fep->quirks & FEC_QUIRK_SWAP_FRAME; +#ifndef CONFIG_AVB_SUPPORT struct bpf_prog *xdp_prog = READ_ONCE(fep->xdp_prog); u32 ret, xdp_result = FEC_ENET_XDP_PASS; - u32 data_start = FEC_ENET_XDP_HEADROOM; struct xdp_buff xdp; +#endif + u32 data_start = FEC_ENET_XDP_HEADROOM; struct page *page; u32 sub_len = 4; @@ -1646,7 +2456,9 @@ fec_enet_rx_queue(struct net_device *ndev, int budget, u16 queue_id) * These get messed up if we get called due to a busy condition. */ bdp = rxq->bd.cur; +#ifndef CONFIG_AVB_SUPPORT xdp_init_buff(&xdp, PAGE_SIZE, &rxq->xdp_rxq); +#endif while (!((status = fec16_to_cpu(bdp->cbd_sc)) & BD_ENET_RX_EMPTY)) { @@ -1696,6 +2508,7 @@ fec_enet_rx_queue(struct net_device *ndev, int budget, u16 queue_id) prefetch(page_address(page)); fec_enet_update_cbd(rxq, bdp, index); +#ifndef CONFIG_AVB_SUPPORT if (xdp_prog) { xdp_buff_clear_frags_flag(&xdp); /* subtract 16bit shift and FCS */ @@ -1706,6 +2519,7 @@ fec_enet_rx_queue(struct net_device *ndev, int budget, u16 queue_id) if (ret != FEC_ENET_XDP_PASS) goto rx_processing_done; } +#endif /* The packet length includes FCS, but we don't want to * include that when passing upstream as it messes up @@ -1754,10 +2568,16 @@ fec_enet_rx_queue(struct net_device *ndev, int budget, u16 queue_id) skb->protocol = eth_type_trans(skb, ndev); /* Get receive timestamp from the skb */ - if (fep->hwts_rx_en && fep->bufdesc_ex) + if (fep->hwts_rx_en && fep->bufdesc_ex) { fec_enet_hwtstamp(fep, fec32_to_cpu(ebdp->ts), skb_hwtstamps(skb)); + /* Adjust for RX MAC-PHY latency + */ + skb_hwtstamps(skb)->hwtstamp = + ktime_sub_ns(skb_hwtstamps(skb)->hwtstamp, fep->rx_tstamp_latency); + } + if (fep->bufdesc_ex && (fep->csum_flags & FLAG_RX_CSUM_ENABLED)) { if (!(ebdp->cbd_esc & cpu_to_fec32(FLAG_RX_CSUM_ERROR))) { @@ -1808,8 +2628,10 @@ fec_enet_rx_queue(struct net_device *ndev, int budget, u16 queue_id) } rxq->bd.cur = bdp; +#ifndef CONFIG_AVB_SUPPORT if (xdp_result & FEC_ENET_XDP_REDIR) xdp_do_flush_map(); +#endif return pkt_received; } @@ -1850,6 +2672,10 @@ fec_enet_interrupt(int irq, void *dev_id) if (fec_enet_collect_events(fep) && fep->link) { ret = IRQ_HANDLED; +#ifdef CONFIG_AVB_SUPPORT + if (fep->avb_enabled) + dev_err(&fep->pdev->dev, "Rx/Tx IRQ with AVB enabled, should not happen\n"); +#endif if (napi_schedule_prep(&fep->napi)) { /* Disable interrupts */ writel(0, fep->hwp + FEC_IMASK); @@ -1866,10 +2692,18 @@ static int fec_enet_rx_napi(struct napi_struct *napi, int budget) struct fec_enet_private *fep = netdev_priv(ndev); int done = 0; - do { - done += fec_enet_rx(ndev, budget - done); - fec_enet_tx(ndev); - } while ((done < budget) && fec_enet_collect_events(fep)); +#ifdef CONFIG_AVB_SUPPORT + if (fep->avb_enabled) { + done = fec_enet_rx_best_effort(ndev, budget); + fec_enet_tx_best_effort(ndev); + } else +#endif + { + do { + done += fec_enet_rx(ndev, budget - done); + fec_enet_tx(ndev); + } while ((done < budget) && fec_enet_collect_events(fep)); + } if (done < budget) { napi_complete_done(napi, done); @@ -1986,11 +2820,33 @@ static void fec_enet_adjust_link(struct net_device *ndev) status_change = 1; } + switch (fep->speed) { + case SPEED_100: + fep->rx_tstamp_latency = fep->rx_delay_100; + fep->tx_tstamp_latency = fep->tx_delay_100; + break; + case SPEED_1000: + fep->rx_tstamp_latency = fep->rx_delay_1000; + fep->tx_tstamp_latency = fep->tx_delay_1000; + break; + default: + fep->rx_tstamp_latency = 0; + fep->tx_tstamp_latency = 0; + } + /* if any of the above changed restart the FEC */ if (status_change) { napi_disable(&fep->napi); netif_tx_lock_bh(ndev); +#ifdef CONFIG_AVB_SUPPORT + if (fep->avb_enabled) + fep->avb->close(fep->avb_data); +#endif fec_restart(ndev); +#ifdef CONFIG_AVB_SUPPORT + if (fep->avb_enabled) + fep->avb->open(fep->avb_data, fep, fec_max_rate(fep)); +#endif netif_tx_wake_all_queues(ndev); netif_tx_unlock_bh(ndev); napi_enable(&fep->napi); @@ -1999,7 +2855,15 @@ static void fec_enet_adjust_link(struct net_device *ndev) if (fep->link) { napi_disable(&fep->napi); netif_tx_lock_bh(ndev); +#ifdef CONFIG_AVB_SUPPORT + if (fep->avb_enabled) + fep->avb->close(fep->avb_data); +#endif fec_stop(ndev); +#ifdef CONFIG_AVB_SUPPORT + if (fep->avb_enabled) + fep->avb->open(fep->avb_data, fep, fec_max_rate(fep)); +#endif netif_tx_unlock_bh(ndev); napi_enable(&fep->napi); fep->link = phy_dev->link; @@ -2322,6 +3186,12 @@ static int fec_enet_mii_probe(struct net_device *ndev) else phy_set_max_speed(phy_dev, 100); +#ifdef CONFIG_AVB_SUPPORT + /* Restore advertising settings saved last interface close */ + if (!linkmode_empty(fep->phy_advertising)) + linkmode_copy(phy_dev->advertising, fep->phy_advertising); +#endif + fep->link = 0; fep->full_duplex = 0; @@ -3252,20 +4122,86 @@ static void fec_enet_free_buffers(struct net_device *ndev) for (q = 0; q < fep->num_rx_queues; q++) { rxq = fep->rx_queue[q]; - for (i = 0; i < rxq->bd.ring_size; i++) - page_pool_put_full_page(rxq->page_pool, rxq->rx_skb_info[i].page, false); +#ifdef CONFIG_AVB_SUPPORT + if (fep->avb_enabled) { + struct bufdesc *bdp = rxq->bd.base; + for (i = 0; i < rxq->bd.ring_size; i++) { + if (bdp->cbd_bufaddr) { + struct avb_rx_desc *desc; + + desc = (struct avb_rx_desc *)rxq->rx_skb_info[i].skb; + fep->avb->free(fep->avb_data, &desc->common); + bdp->cbd_bufaddr = cpu_to_fec32(0); + } + + bdp = fec_enet_get_nextdesc(bdp, &rxq->bd); + } + } else +#endif + { + for (i = 0; i < rxq->bd.ring_size; i++) + page_pool_put_full_page(rxq->page_pool, rxq->rx_skb_info[i].page, false); - for (i = 0; i < XDP_STATS_TOTAL; i++) - rxq->stats[i] = 0; + for (i = 0; i < XDP_STATS_TOTAL; i++) + rxq->stats[i] = 0; - if (xdp_rxq_info_is_reg(&rxq->xdp_rxq)) - xdp_rxq_info_unreg(&rxq->xdp_rxq); - page_pool_destroy(rxq->page_pool); - rxq->page_pool = NULL; + if (xdp_rxq_info_is_reg(&rxq->xdp_rxq)) + xdp_rxq_info_unreg(&rxq->xdp_rxq); + page_pool_destroy(rxq->page_pool); + rxq->page_pool = NULL; + } + } + +#ifdef CONFIG_AVB_SUPPORT + if (fep->avb_enabled) { + struct avb_rx_desc *rx_desc; + struct avb_tx_desc *tx_desc; + + while ((rx_desc = fep->avb->dequeue(fep->avb_data)) != (void *) -1) + fep->avb->free(fep->avb_data, &rx_desc->common); + + while ((tx_desc = fep->avb->tx_cleanup_dequeue(fep->avb_data)) != (void *) -1) { + + skb = tx_desc->data; + + dma_unmap_single(&fep->pdev->dev, tx_desc->bufaddr, + skb->len, DMA_TO_DEVICE); + + /* Free the sk buffer associated with this last transmit */ + dev_kfree_skb_any(skb); + } } +#endif for (q = 0; q < fep->num_tx_queues; q++) { txq = fep->tx_queue[q]; +#ifdef CONFIG_AVB_SUPPORT + for (i = 0; i < txq->bd.ring_size; i++) { + if (txq->tx_buf[i].skb) { + skb = NULL; + if (fep->avb_enabled) { + struct avb_tx_desc *desc = (struct avb_tx_desc *)txq->tx_buf[i].skb; + + if (!(desc->common.flags & AVB_TX_FLAG_SKB)) + fep->avb->free(fep->avb_data, &desc->common); + else + skb = desc->data; + } else { + skb = txq->tx_buf[i].skb; + } + + if (skb) + dev_kfree_skb(skb); + + txq->tx_buf[i].skb = NULL; + } + } + + for (i = 0; i < txq->tx_bounce_size; i++) { + kfree(txq->tx_bounce[i]); + txq->tx_bounce[i] = NULL; + } +#else for (i = 0; i < txq->bd.ring_size; i++) { kfree(txq->tx_bounce[i]); txq->tx_bounce[i] = NULL; @@ -3282,6 +4218,7 @@ static void fec_enet_free_buffers(struct net_device *ndev) txq->tx_buf[i].type = FEC_TXBUF_T_SKB; } } +#endif } } @@ -3321,10 +4258,14 @@ static int fec_enet_alloc_queue(struct net_device *ndev) } fep->tx_queue[i] = txq; - txq->bd.ring_size = TX_RING_SIZE; + txq->bd.ring_size = FEC_TX_RING_SIZE; fep->total_tx_ring_size += fep->tx_queue[i]->bd.ring_size; +#ifdef CONFIG_AVB_SUPPORT + txq->tx_stop_threshold = FEC_TX_RING_SIZE * 3/4; +#else txq->tx_stop_threshold = FEC_MAX_SKB_DESCS; +#endif txq->tx_wake_threshold = (txq->bd.ring_size - txq->tx_stop_threshold) / 2; @@ -3346,7 +4287,7 @@ static int fec_enet_alloc_queue(struct net_device *ndev) goto alloc_failed; } - fep->rx_queue[i]->bd.ring_size = RX_RING_SIZE; + fep->rx_queue[i]->bd.ring_size = FEC_RX_RING_SIZE; fep->total_rx_ring_size += fep->rx_queue[i]->bd.ring_size; } return ret; @@ -3367,32 +4308,77 @@ fec_enet_alloc_rxq_buffers(struct net_device *ndev, unsigned int queue) int i, err; rxq = fep->rx_queue[queue]; + +#ifdef CONFIG_AVB_SUPPORT bdp = rxq->bd.base; - err = fec_enet_create_page_pool(fep, rxq, rxq->bd.ring_size); - if (err < 0) { - netdev_err(ndev, "%s failed queue %d (%d)\n", __func__, queue, err); - return err; + for (i = 0; i < rxq->bd.ring_size; i++) { + bdp->cbd_bufaddr = cpu_to_fec32(0); + bdp = fec_enet_get_nextdesc(bdp, &rxq->bd); } +#endif - for (i = 0; i < rxq->bd.ring_size; i++) { - page = page_pool_dev_alloc_pages(rxq->page_pool); - if (!page) - goto err_alloc; + bdp = rxq->bd.base; +#ifdef CONFIG_AVB_SUPPORT + if (fep->avb_enabled) { + for (i = 0; i < rxq->bd.ring_size; i++) { + struct avb_rx_desc *desc; + void *buffer; - phys_addr = page_pool_get_dma_addr(page) + FEC_ENET_XDP_HEADROOM; - bdp->cbd_bufaddr = cpu_to_fec32(phys_addr); + buffer = fep->avb->alloc(fep->avb_data); + if (!buffer) + goto err_alloc; - rxq->rx_skb_info[i].page = page; - rxq->rx_skb_info[i].offset = FEC_ENET_XDP_HEADROOM; - bdp->cbd_sc = cpu_to_fec16(BD_ENET_RX_EMPTY); + desc = buffer; - if (fep->bufdesc_ex) { - struct bufdesc_ex *ebdp = (struct bufdesc_ex *)bdp; - ebdp->cbd_esc = cpu_to_fec32(BD_ENET_RX_INT); + desc->common.len = 0; + desc->queue_id = queue; + + rxq->rx_skb_info[i].skb = buffer; + bdp->cbd_bufaddr = cpu_to_fec32((dma_addr_t)(desc->dma_addr)); + +#if !defined(CONFIG_M5272) + if (fep->quirks & FEC_QUIRK_HAS_RACC) + desc->common.offset += 2; +#endif + + bdp->cbd_sc = cpu_to_fec16(BD_ENET_RX_EMPTY); + + if (fep->bufdesc_ex) { + struct bufdesc_ex *ebdp = (struct bufdesc_ex *)bdp; + ebdp->cbd_esc = cpu_to_fec32(0); + } + + bdp = fec_enet_get_nextdesc(bdp, &rxq->bd); + } + } else +#endif + { + err = fec_enet_create_page_pool(fep, rxq, rxq->bd.ring_size); + if (err < 0) { + netdev_err(ndev, "%s failed queue %d (%d)\n", __func__, queue, err); + return err; } - bdp = fec_enet_get_nextdesc(bdp, &rxq->bd); + for (i = 0; i < rxq->bd.ring_size; i++) { + page = page_pool_dev_alloc_pages(rxq->page_pool); + if (!page) + goto err_alloc; + + phys_addr = page_pool_get_dma_addr(page) + FEC_ENET_XDP_HEADROOM; + bdp->cbd_bufaddr = cpu_to_fec32(phys_addr); + + rxq->rx_skb_info[i].page = page; + rxq->rx_skb_info[i].offset = FEC_ENET_XDP_HEADROOM; + bdp->cbd_sc = cpu_to_fec16(BD_ENET_RX_EMPTY); + + if (fep->bufdesc_ex) { + struct bufdesc_ex *ebdp = (struct bufdesc_ex *)bdp; + ebdp->cbd_esc = cpu_to_fec32(BD_ENET_RX_INT); + } + + bdp = fec_enet_get_nextdesc(bdp, &rxq->bd); + } } /* Set the last buffer to wrap. */ @@ -3415,17 +4401,28 @@ fec_enet_alloc_txq_buffers(struct net_device *ndev, unsigned int queue) txq = fep->tx_queue[queue]; bdp = txq->bd.base; - for (i = 0; i < txq->bd.ring_size; i++) { + for (i = 0; i < txq->tx_bounce_size; i++) { txq->tx_bounce[i] = kmalloc(FEC_ENET_TX_FRSIZE, GFP_KERNEL); if (!txq->tx_bounce[i]) goto err_alloc; + } + + bdp = txq->bd.base; + txq->bd.cur = bdp; + for (i = 0; i < txq->bd.ring_size; i++) { bdp->cbd_sc = cpu_to_fec16(0); bdp->cbd_bufaddr = cpu_to_fec32(0); if (fep->bufdesc_ex) { struct bufdesc_ex *ebdp = (struct bufdesc_ex *)bdp; - ebdp->cbd_esc = cpu_to_fec32(BD_ENET_TX_INT); + +#ifdef CONFIG_AVB_SUPPORT + if (fep->avb_enabled) + ebdp->cbd_esc = cpu_to_fec32(0); + else +#endif + ebdp->cbd_esc = cpu_to_fec32(BD_ENET_TX_INT); } bdp = fec_enet_get_nextdesc(bdp, &txq->bd); @@ -3434,6 +4431,7 @@ fec_enet_alloc_txq_buffers(struct net_device *ndev, unsigned int queue) /* Set the last buffer to wrap. */ bdp = fec_enet_get_prevdesc(bdp, &txq->bd); bdp->cbd_sc |= cpu_to_fec16(BD_SC_WRAP); + txq->dirty_tx = bdp; return 0; @@ -3463,15 +4461,31 @@ fec_enet_open(struct net_device *ndev) struct fec_enet_private *fep = netdev_priv(ndev); int ret; bool reset_again; + int i; + +#ifdef CONFIG_AVB_SUPPORT + if (fep->avb_enabled) { + if (!try_module_get(fep->avb->owner)) { + ret = -EIO; + goto err_module_get; + } + for (i = 0; i < fep->num_tx_queues; i++) + fep->tx_queue[i]->tx_bounce_size = FEC_TX_RING_SIZE + 32; + } else +#endif + for (i = 0; i < fep->num_tx_queues; i++) + fep->tx_queue[i]->tx_bounce_size = FEC_TX_RING_SIZE; ret = pm_runtime_resume_and_get(&fep->pdev->dev); if (ret < 0) return ret; pinctrl_pm_select_default_state(&fep->pdev->dev); +#ifndef CONFIG_AVB_SUPPORT ret = fec_enet_clk_enable(ndev, true); if (ret) goto clk_enable; +#endif /* During the first fec_enet_open call the PHY isn't probed at this * point. Therefore the phy_reset_after_clk_enable() call within @@ -3514,6 +4528,12 @@ fec_enet_open(struct net_device *ndev) napi_enable(&fep->napi); phy_start(ndev->phydev); + +#ifdef CONFIG_AVB_SUPPORT + if (fep->avb_enabled) + fep->avb->open(fep->avb_data, fep, fec_max_rate(fep)); +#endif + netif_tx_start_all_queues(ndev); device_set_wakeup_enable(&ndev->dev, fep->wol_flag & @@ -3524,12 +4544,23 @@ fec_enet_open(struct net_device *ndev) err_enet_mii_probe: fec_enet_free_buffers(ndev); err_enet_alloc: +#ifndef CONFIG_AVB_SUPPORT fec_enet_clk_enable(ndev, false); clk_enable: +#endif pm_runtime_mark_last_busy(&fep->pdev->dev); pm_runtime_put_autosuspend(&fep->pdev->dev); if (!fep->mii_bus_share) pinctrl_pm_select_sleep_state(&fep->pdev->dev); + +#ifdef CONFIG_AVB_SUPPORT + if (fep->avb_enabled) + module_put(fep->avb->owner); +#endif + +#ifdef CONFIG_AVB_SUPPORT +err_module_get: +#endif return ret; } @@ -3543,9 +4574,20 @@ fec_enet_close(struct net_device *ndev) if (netif_device_present(ndev)) { napi_disable(&fep->napi); netif_tx_disable(ndev); +#ifdef CONFIG_AVB_SUPPORT + if (fep->avb_enabled) + fep->avb->close(fep->avb_data); +#endif fec_stop(ndev); } +#ifdef CONFIG_AVB_SUPPORT + /* + * Save advertising settings so there are not lost + * when opening the interface again. + */ + linkmode_copy(fep->phy_advertising, ndev->phydev->advertising); +#endif phy_disconnect(ndev->phydev); ndev->phydev = NULL; @@ -3554,7 +4596,9 @@ fec_enet_close(struct net_device *ndev) fec_enet_update_ethtool_stats(ndev); +#ifndef CONFIG_AVB_SUPPORT fec_enet_clk_enable(ndev, false); +#endif if (fep->quirks & FEC_QUIRK_HAS_PMQOS) cpu_latency_qos_remove_request(&fep->pm_qos_req); @@ -3565,6 +4609,11 @@ fec_enet_close(struct net_device *ndev) fec_enet_free_buffers(ndev); +#ifdef CONFIG_AVB_SUPPORT + if (fep->avb_enabled) + module_put(fep->avb->owner); +#endif + return 0; } @@ -3749,6 +4798,7 @@ static u16 fec_enet_select_queue(struct net_device *ndev, struct sk_buff *skb, return fec_enet_vlan_pri_to_queue[vlan_tag >> 13]; } +#ifndef CONFIG_AVB_SUPPORT static int fec_enet_bpf(struct net_device *dev, struct netdev_bpf *bpf) { struct fec_enet_private *fep = netdev_priv(dev); @@ -3936,6 +4986,7 @@ static int fec_enet_xdp_xmit(struct net_device *dev, return sent_frames; } +#endif /* !CONFIG_AVB_SUPPORT */ static const struct net_device_ops fec_netdev_ops = { .ndo_open = fec_enet_open, @@ -3951,8 +5002,11 @@ static const struct net_device_ops fec_netdev_ops = { .ndo_poll_controller = fec_poll_controller, #endif .ndo_set_features = fec_set_features, +#ifndef CONFIG_AVB_SUPPORT + /* AVB support not compatible with XDP */ .ndo_bpf = fec_enet_bpf, .ndo_xdp_xmit = fec_enet_xdp_xmit, +#endif }; static const unsigned short offset_des_active_rxq[] = { @@ -3963,6 +5017,196 @@ static const unsigned short offset_des_active_txq[] = { FEC_X_DES_ACTIVE_0, FEC_X_DES_ACTIVE_1, FEC_X_DES_ACTIVE_2 }; +#ifdef CONFIG_AVB_SUPPORT + +static struct platform_driver fec_driver; + +/* Checks if the net_device is registered by the fec */ +static bool __is_fec_net_device(struct net_device *ndev) +{ + if (!ndev) + return false; + + if (ndev->dev.parent->driver == &fec_driver.driver) + return true; + else + return false; +} + +struct device *fec_enet_avb_get_device(const char *ifname) +{ + struct net_device *ndev; + struct fec_enet_private *fep; + + ndev = dev_get_by_name(&init_net, ifname); + if (!ndev) + goto err_dev_get; + + if (!__is_fec_net_device(ndev)) + goto err_ndev; + + fep = netdev_priv(ndev); + + dev_put(ndev); + + return &fep->pdev->dev; + +err_ndev: + dev_put(ndev); + +err_dev_get: + return NULL; +} +EXPORT_SYMBOL(fec_enet_avb_get_device); + +int fec_enet_avb_register(const char *ifname, const struct avb_ops *avb, void *data) +{ + struct net_device *ndev; + struct fec_enet_private *fep; + unsigned int up; + int ifindex; + + ndev = dev_get_by_name(&init_net, ifname); + if (!ndev) + goto err_dev_get; + + if (!__is_fec_net_device(ndev)) + goto err_ndev; + + fep = netdev_priv(ndev); + + if (fep->avb) + goto err_avb; + + rtnl_lock(); + up = ndev->flags & IFF_UP; + if (up) + dev_close(ndev); + + fep->avb = avb; + fep->avb_data = data; + fep->avb_enabled = 1; + ifindex = ndev->ifindex; + + if (up) { + /* In case of error, device is closed but avb interface is registered */ + dev_open(ndev, NULL); + } + + rtnl_unlock(); + + dev_put(ndev); + + return ifindex; + +err_avb: +err_ndev: + dev_put(ndev); + +err_dev_get: + return -1; +} +EXPORT_SYMBOL(fec_enet_avb_register); + +int fec_enet_avb_unregister(int ifindex, const struct avb_ops *avb) +{ + struct net_device *ndev; + struct fec_enet_private *fep; + unsigned int up; + + ndev = dev_get_by_index(&init_net, ifindex); + if (!ndev) + goto err_dev_get; + + if (!__is_fec_net_device(ndev)) + goto err_ndev; + + fep = netdev_priv(ndev); + if (fep->avb != avb) + goto err_avb; + + rtnl_lock(); + up = ndev->flags & IFF_UP; + if (up) + dev_close(ndev); + + fep->avb = NULL; + fep->avb_data = NULL; + fep->avb_enabled = 0; + + if (up) + /* In case of error, device is closed but avb interface is unregistered */ + dev_open(ndev, NULL); + + rtnl_unlock(); + + dev_put(ndev); + + return 0; + +err_avb: +err_ndev: + dev_put(ndev); + +err_dev_get: + return -1; +} +EXPORT_SYMBOL(fec_enet_avb_unregister); + +int fec_enet_get_tx_queue_properties(int ifindex, struct tx_queue_properties *prop) +{ + struct net_device *ndev; + struct fec_enet_private *fep; + + ndev = dev_get_by_index(&init_net, ifindex); + if (!ndev) + goto err_dev_get; + + if (!__is_fec_net_device(ndev)) + goto err_ndev; + + fep = netdev_priv(ndev); + + if (fep->num_tx_queues >= TX_QUEUE_PROP_MAX) + goto err_queues; + + if (fep->quirks & FEC_QUIRK_HAS_AVB) { + prop->num_queues = fep->num_tx_queues; + prop->queue[0].priority = 0; + prop->queue[0].flags = TX_QUEUE_FLAGS_STRICT_PRIORITY; + prop->queue[1].priority = 2; + prop->queue[1].flags = TX_QUEUE_FLAGS_CREDIT_SHAPER; + prop->queue[2].priority = 1; + prop->queue[2].flags = TX_QUEUE_FLAGS_CREDIT_SHAPER; + } else { + /* + * For now, there is no MAC non-AVB capable + * with more than 1 queue. + */ + if (fep->num_tx_queues == 1) { + prop->num_queues = fep->num_tx_queues; + prop->queue[0].priority = 0; + prop->queue[0].flags = TX_QUEUE_FLAGS_STRICT_PRIORITY; + } else { + netdev_err(ndev, "%s invalid/unknown TX queues configuration\n", __func__); + goto err_queues; + } + } + + dev_put(ndev); + + return 0; + +err_queues: +err_ndev: + dev_put(ndev); + +err_dev_get: + return -1; +} +EXPORT_SYMBOL(fec_enet_get_tx_queue_properties); +#endif + /* * XXX: We need to clean up on failure exits here. * @@ -4050,10 +5294,14 @@ static int fec_enet_init(struct net_device *ndev) txq->bd.dsize_log2 = dsize_log2; txq->bd.reg_desc_active = fep->hwp + offset_des_active_txq[i]; bd_dma += size; +#ifdef CONFIG_AVB_SUPPORT + txq->tx_idle_slope = IDLE_SLOPE(i); +#endif cbd_base = (struct bufdesc *)(((void *)cbd_base) + size); txq->bd.last = (struct bufdesc *)(((void *)cbd_base) - dsize); } + fep->netdev = ndev; /* The FEC Ethernet specific entries in the device structure */ ndev->watchdog_timeo = TX_TIMEOUT; @@ -4071,8 +5319,13 @@ static int fec_enet_init(struct net_device *ndev) netif_set_tso_max_segs(ndev, FEC_MAX_TSO_SEGS); /* enable hw accelerator */ +#ifdef CONFIG_AVB_SUPPORT + /* AVB support not compatible with SG or TSO */ + ndev->features |= (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM); +#else ndev->features |= (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_TSO); +#endif fep->csum_flags |= FLAG_RX_CSUM_ENABLED; } @@ -4399,6 +5652,13 @@ fec_probe(struct platform_device *pdev) } fep->bufdesc_ex = fep->quirks & FEC_QUIRK_HAS_BUFDESC_EX; +#ifdef CONFIG_AVB_SUPPORT + if (!fep->bufdesc_ex) { + dev_err(&pdev->dev, + "Error: AVB Support requires extended buffer descriptor\n"); + goto failed_clk; + } +#endif fep->clk_ptp = devm_clk_get(&pdev->dev, "ptp"); if (IS_ERR(fep->clk_ptp)) { fep->clk_ptp = NULL; @@ -4432,12 +5692,32 @@ fec_probe(struct platform_device *pdev) fep->reg_phy = NULL; } + if (of_property_read_u32(np, "fsl,rx-phy-delay-100-ns", &fep->rx_delay_100)) + fep->rx_delay_100 = 0; + + if (of_property_read_u32(np, "fsl,tx-phy-delay-100-ns", &fep->tx_delay_100)) + fep->tx_delay_100 = 0; + + if (of_property_read_u32(np, "fsl,rx-phy-delay-1000-ns", &fep->rx_delay_1000)) + fep->rx_delay_1000 = 0; + + if (of_property_read_u32(np, "fsl,tx-phy-delay-1000-ns", &fep->tx_delay_1000)) + fep->tx_delay_1000 = 0; + pm_runtime_set_autosuspend_delay(&pdev->dev, FEC_MDIO_PM_TIMEOUT); pm_runtime_use_autosuspend(&pdev->dev); pm_runtime_get_noresume(&pdev->dev); pm_runtime_set_active(&pdev->dev); pm_runtime_enable(&pdev->dev); +#ifdef CONFIG_AVB_SUPPORT + /* + * Prevent runtime pm with avb module to keep all clocks + * running even on link down. + */ + pm_runtime_forbid(&pdev->dev); +#endif + ret = fec_reset_phy(pdev); if (ret) goto failed_reset; @@ -4480,7 +5760,9 @@ fec_probe(struct platform_device *pdev) /* Carrier starts down, phylib will bring it up */ netif_carrier_off(ndev); +#ifndef CONFIG_AVB_SUPPORT fec_enet_clk_enable(ndev, false); +#endif pinctrl_pm_select_sleep_state(&pdev->dev); ndev->max_mtu = PKT_MAXBUF_SIZE - ETH_HLEN - ETH_FCS_LEN; @@ -4590,6 +5872,10 @@ static int __maybe_unused fec_suspend(struct device *dev) netif_tx_lock_bh(ndev); netif_device_detach(ndev); netif_tx_unlock_bh(ndev); +#ifdef CONFIG_AVB_SUPPORT + if (fep->avb_enabled) + fep->avb->close(fep->avb_data); +#endif fec_stop(ndev); if (!(fep->wol_flag & FEC_WOL_FLAG_ENABLE)) { fec_irqs_disable(ndev); @@ -4674,6 +5960,10 @@ static int __maybe_unused fec_resume(struct device *dev) napi_enable(&fep->napi); phy_init_hw(ndev->phydev); phy_start(ndev->phydev); +#ifdef CONFIG_AVB_SUPPORT + if (fep->avb_enabled) + fep->avb->open(fep->avb_data, fep, fec_max_rate(fep)); +#endif } else if (fep->mii_bus_share && !ndev->phydev) { pinctrl_pm_select_default_state(&fep->pdev->dev); /* And then recovery mii bus */ diff --git a/drivers/net/ethernet/freescale/fec_ptp.c b/drivers/net/ethernet/freescale/fec_ptp.c index d15737962145..339d3f0fc3fe 100644 --- a/drivers/net/ethernet/freescale/fec_ptp.c +++ b/drivers/net/ethernet/freescale/fec_ptp.c @@ -111,7 +111,14 @@ static int fec_ptp_enable_pps(struct fec_enet_private *fep, uint enable) fep->pps_channel = DEFAULT_PPS_CHANNEL; fep->reload_period = PPS_OUPUT_RELOAD_PERIOD; - spin_lock_irqsave(&fep->tmreg_lock, flags); + raw_spin_lock_irqsave(&fep->tmreg_lock, flags); + +#ifdef CONFIG_AVB_SUPPORT + if (fep->rec_enable && (fep->pps_channel == fep->rec_channel)) { + raw_spin_unlock_irqrestore(&fep->tmreg_lock, flags); + return -EBUSY; + } +#endif if (enable) { /* clear capture or output compare interrupt status if have. @@ -196,7 +203,7 @@ static int fec_ptp_enable_pps(struct fec_enet_private *fep, uint enable) } fep->pps_enable = enable; - spin_unlock_irqrestore(&fep->tmreg_lock, flags); + raw_spin_unlock_irqrestore(&fep->tmreg_lock, flags); return 0; } @@ -207,7 +214,7 @@ static int fec_ptp_pps_perout(struct fec_enet_private *fep) u64 curr_time; unsigned long flags; - spin_lock_irqsave(&fep->tmreg_lock, flags); + raw_spin_lock_irqsave(&fep->tmreg_lock, flags); /* Update time counter */ timecounter_read(&fep->tc); @@ -230,7 +237,7 @@ static int fec_ptp_pps_perout(struct fec_enet_private *fep) */ if (fep->perout_stime < curr_time + 100 * NSEC_PER_MSEC) { dev_err(&fep->pdev->dev, "Current time is too close to the start time!\n"); - spin_unlock_irqrestore(&fep->tmreg_lock, flags); + raw_spin_unlock_irqrestore(&fep->tmreg_lock, flags); return -1; } @@ -258,7 +265,7 @@ static int fec_ptp_pps_perout(struct fec_enet_private *fep) */ writel(fep->next_counter, fep->hwp + FEC_TCCR(fep->pps_channel)); fep->next_counter = (fep->next_counter + fep->reload_period) & fep->cc.mask; - spin_unlock_irqrestore(&fep->tmreg_lock, flags); + raw_spin_unlock_irqrestore(&fep->tmreg_lock, flags); return 0; } @@ -273,6 +280,26 @@ static enum hrtimer_restart fec_ptp_pps_perout_handler(struct hrtimer *timer) return HRTIMER_NORESTART; } +#ifdef CONFIG_AVB_SUPPORT +/** + * fec_timecounter_set + * @start_tstamp: new time in ns + * + * update the FEC timecounter structure to a new time, and make sure + * the HW counter matches that value. + */ +static inline void fec_timecounter_set(struct fec_enet_private *fep, + u64 start_tstamp) +{ + u32 tempval; + + tempval = start_tstamp & fep->cc.mask; + writel(tempval, fep->hwp + FEC_ATIME); + fep->tc.cycle_last = tempval; + fep->tc.nsec = start_tstamp; +} +#endif + /** * fec_ptp_read - read raw cycle counter (to be used by time counter) * @cc: the cyclecounter structure @@ -287,14 +314,14 @@ static u64 fec_ptp_read(const struct cyclecounter *cc) container_of(cc, struct fec_enet_private, cc); u32 tempval; - tempval = readl(fep->hwp + FEC_ATIME_CTRL); + tempval = readl_relaxed(fep->hwp + FEC_ATIME_CTRL); tempval |= FEC_T_CTRL_CAPTURE; writel(tempval, fep->hwp + FEC_ATIME_CTRL); if (fep->quirks & FEC_QUIRK_BUG_CAPTURE) udelay(1); - return readl(fep->hwp + FEC_ATIME); + return readl_relaxed(fep->hwp + FEC_ATIME); } /** @@ -314,29 +341,312 @@ void fec_ptp_start_cyclecounter(struct net_device *ndev) inc = 1000000000 / fep->cycle_speed; /* grab the ptp lock */ - spin_lock_irqsave(&fep->tmreg_lock, flags); + raw_spin_lock_irqsave(&fep->tmreg_lock, flags); - /* 1ns counter */ - writel(inc << FEC_T_INC_OFFSET, fep->hwp + FEC_ATIME_INC); + /* 1ns counter, disable correction period */ + writel_relaxed(0, fep->hwp + FEC_ATIME_CORR); + writel_relaxed(inc << FEC_T_INC_OFFSET, fep->hwp + FEC_ATIME_INC); - /* use 31-bit timer counter */ - writel(FEC_COUNTER_PERIOD, fep->hwp + FEC_ATIME_EVT_PERIOD); - writel(FEC_T_CTRL_ENABLE | FEC_T_CTRL_PERIOD_RST, +#ifdef CONFIG_AVB_SUPPORT + /* use 32-bits timer counter */ + writel_relaxed(0, fep->hwp + FEC_ATIME_EVT_PERIOD); + writel_relaxed(FEC_T_CTRL_ENABLE, fep->hwp + FEC_ATIME_CTRL); +#else + /* use 31-bit timer counter */ + writel_relaxed(FEC_COUNTER_PERIOD, fep->hwp + FEC_ATIME_EVT_PERIOD); + writel_relaxed(FEC_T_CTRL_ENABLE | FEC_T_CTRL_PERIOD_RST, fep->hwp + FEC_ATIME_CTRL); +#endif memset(&fep->cc, 0, sizeof(fep->cc)); fep->cc.read = fec_ptp_read; +#ifdef CONFIG_AVB_SUPPORT + fep->cc.mask = CLOCKSOURCE_MASK(32); +#else fep->cc.mask = CLOCKSOURCE_MASK(31); +#endif fep->cc.shift = 31; fep->cc.mult = FEC_CC_MULT; /* reset the ns time counter */ +#ifdef CONFIG_AVB_SUPPORT + fep->tc.cc = &fep->cc; + fec_timecounter_set(fep, 0); +#else timecounter_init(&fep->tc, &fep->cc, 0); +#endif - spin_unlock_irqrestore(&fep->tmreg_lock, flags); + raw_spin_unlock_irqrestore(&fep->tmreg_lock, flags); } +#ifdef CONFIG_AVB_SUPPORT +/** + * fec_ptp_adjfreq - adjust ptp cycle frequency + * @ptp: the ptp clock structure + * @ppb: parts per billion adjustment from base + * + * Adjust the frequency of the ptp cycle counter by the + * indicated ppb from the base frequency. + * + * This version adjusts the HW counter. The HW implementation results in the following equation: + * fe * diff = ppb * (pc + 1) + * with: + * . fe : ENET ref clock frequency in Hz + * . diff = inc_corr - inc : difference between default increment and correction increment + * . ppb : parts per billion adjustment from base + * . pc : correction period (in number of fe clock cycles) + * + * Limitations: + * . only add or remove 1ns per correction period. This will limit jitter and improve short term + * accuracy (in particular for trigger events during clock recovery) but increase the max possible + * error, since the correction period will be the shortest possible and thus may not be optimal. + * In the case of an adjustment of about 100ppm (which should be the max if the ref clock + * is within the 802.1AS spec), the max error will be about 0.2ppm with a 50MHz ref clock, and + * 0.08ppm with a 125MHz ref clock. + * Long term accuracy will also be lower (20ns per 100ms @50MHz, 8ns per 100ms @125MHz), but this + * can be fixed by phase adjustments. + * . It seems not all period/correction values are valid. With a 1ns correction, all even + * period values return wrong timings (half the requested correction), but on the other hand odd + * values are not taken into account systematically by the hardware, so we choose the closest even + * value that matches the following equation: + * fe * diff = 2 * ppb * (pc + 1) + * . given we force abs(diff) = 1, limit max adjustment to prevent pc < 1. + * + */ +static int fec_ptp_adjfreq(struct ptp_clock_info *ptp, s32 ppb) +{ + struct fec_enet_private *fep = + container_of(ptp, struct fec_enet_private, ptp_caps); + unsigned long flags; + u32 inc, cor, pc; + + inc = fep->ptp_inc; + if (ppb == 0) { + cor = 0; + pc = 0; + } else { + if (ppb < 0) { + ppb = -ppb; + cor = inc - 1; + } + else + cor = inc + 1; + + if (ppb > fep->ptp_caps.max_adj) { + dev_err(&fep->pdev->dev, "ppb value %d outside accepted range (max_adj = %d)", (cor > inc) ? ppb : -ppb, ptp->max_adj); + return -1; + } + + pc = (((fep->cycle_speed / (2*ppb)) - 1) + 1) & ~ 0x1; // + 1) & ~ 0x1 returns the closest even value + } + + raw_spin_lock_irqsave(&fep->tmreg_lock, flags); + + writel_relaxed((cor << FEC_T_INC_CORR_OFFSET) | (inc << FEC_T_INC_OFFSET), fep->hwp + FEC_ATIME_INC); + writel_relaxed(pc, fep->hwp + FEC_ATIME_CORR); + + raw_spin_unlock_irqrestore(&fep->tmreg_lock, flags); + + return 0; +} + +/** + * fec_ptp_adjtime + * @ptp: the ptp clock structure + * @delta: offset to adjust the cycle counter by + * + * adjust the timer by updating the HW counter AND the timecounter structure + * as well. Since updating the HW register requires a read and a write, make + * a crude estimate of the read time and subtract it from the desired delta. + */ +static int fec_ptp_adjtime(struct ptp_clock_info *ptp, s64 delta) +{ + struct fec_enet_private *fep = + container_of(ptp, struct fec_enet_private, ptp_caps); + unsigned long flags; + u64 now, then; + s64 real_delta; + + raw_spin_lock_irqsave(&fep->tmreg_lock, flags); + + now = timecounter_read(&fep->tc); + then = timecounter_read(&fep->tc); + + real_delta = delta + (then - now); + + now = timecounter_read(&fep->tc); + now += real_delta; + + fec_timecounter_set(fep, now); + + raw_spin_unlock_irqrestore(&fep->tmreg_lock, flags); + + return 0; +} + +/** + * fec_ptp_settime + * @ptp: the ptp clock structure + * @ts: the timespec containing the new time for the cycle counter + * + * Update the timecounter to use a new base value instead of the kernel + * wall timer value, and update the HW counter as well. + */ +static int fec_ptp_settime(struct ptp_clock_info *ptp, + const struct timespec64 *ts) +{ + struct fec_enet_private *fep = + container_of(ptp, struct fec_enet_private, ptp_caps); + u64 ns; + unsigned long flags; + + ns = ts->tv_sec * 1000000000ULL; + ns += ts->tv_nsec; + + raw_spin_lock_irqsave(&fep->tmreg_lock, flags); + fec_timecounter_set(fep, ns); + raw_spin_unlock_irqrestore(&fep->tmreg_lock, flags); + return 0; +} + +/** + * fec_ptp_read + * @data: fec private context ptr + * @cnt: data pointer for counter value + * + * Returns status + */ +int fec_ptp_read_cnt(void *data, u32 *cnt) +{ + struct fec_enet_private *fep = data; + unsigned long flags; + + /* Note : it might be possible to avoid taking the lock */ + raw_spin_lock_irqsave(&fep->tmreg_lock, flags); + + *cnt = (u32) fec_ptp_read(&fep->cc); + + raw_spin_unlock_irqrestore(&fep->tmreg_lock, flags); + + return 0; +} +EXPORT_SYMBOL(fec_ptp_read_cnt); + +/** + * fec_ptp_tc_start + * @data: fec private context ptr + * @id: TC register ID + * @ts_0: First timestamp + * @ts_1: Second timestamp + * @tcsr_val: TCSR register value + * + * Returns 0 on success, -1 if PTP counter is not + * enabled. + */ +int fec_ptp_tc_start(void *data, u8 id, u32 ts_0, u32 ts_1, u32 tcsr_val) +{ + struct fec_enet_private *fep = data; + unsigned long flags; + u32 ctrl_val; + int rc = 0; + + if (id > MAX_TIMER_CHANNEL) { + rc = -1; + goto exit; + } + + raw_spin_lock_irqsave(&fep->tmreg_lock, flags); + + /* Simple resource sharing handling with pps */ + if (fep->pps_enable && (fep->pps_channel == id)) { + rc = -1; + goto exit_unlock; + } + + ctrl_val = readl_relaxed(fep->hwp + FEC_ATIME_CTRL); + if (!(ctrl_val & FEC_T_CTRL_ENABLE)) { + rc = -1; + goto exit_unlock; + } + + writel_relaxed(ts_0, fep->hwp + FEC_TCCR(id)); + writel_relaxed(tcsr_val, fep->hwp + FEC_TCSR(id)); + writel_relaxed(ts_1, fep->hwp + FEC_TCCR(id)); + + fep->rec_enable = 1; + fep->rec_channel = id; + +exit_unlock: + raw_spin_unlock_irqrestore(&fep->tmreg_lock, flags); +exit: + return rc; +} +EXPORT_SYMBOL(fec_ptp_tc_start); + +/** + * fec_ptp_tc_stop + * @data: fec private context ptr + * @id: TC register ID + * + * Returns none + */ +void fec_ptp_tc_stop(void *data, u8 id) +{ + struct fec_enet_private *fep = data; + unsigned long flags; + + raw_spin_lock_irqsave(&fep->tmreg_lock, flags); + + writel_relaxed(0, fep->hwp + FEC_TCCR(id)); + writel_relaxed(FEC_T_TF_MASK, fep->hwp + FEC_TCSR(id)); + + fep->rec_enable = 0; + + raw_spin_unlock_irqrestore(&fep->tmreg_lock, flags); +} +EXPORT_SYMBOL(fec_ptp_tc_stop); + +/** + * fec_ptp_tc_reload + * @data: fec private context ptr + * @id: TC register ID + * @ts: New timestamp to load + * + * Returns 0 if success, -1 if compare has not occured + * or if PTP counter is not enabled. + */ +int fec_ptp_tc_reload(void *data, u8 id, u32 ts) +{ + struct fec_enet_private *fep = data; + unsigned long flags; + u32 tcsr_val; + u32 ctrl_val; + int rc = 0; + + raw_spin_lock_irqsave(&fep->tmreg_lock, flags); + + ctrl_val = readl_relaxed(fep->hwp + FEC_ATIME_CTRL); + if (!(ctrl_val & FEC_T_CTRL_ENABLE)) { + rc = -1; + goto exit; + } + + tcsr_val = readl_relaxed(fep->hwp + FEC_TCSR(id)); + if (tcsr_val & FEC_T_TF_MASK) { + writel_relaxed(ts, fep->hwp + FEC_TCCR(id)); + writel_relaxed(tcsr_val, fep->hwp + FEC_TCSR(id)); + } + else + rc = -1; + +exit: + raw_spin_unlock_irqrestore(&fep->tmreg_lock, flags); + return rc; +} +EXPORT_SYMBOL(fec_ptp_tc_reload); +#else /* CONFIG_AVB_SUPPORT */ + /** * fec_ptp_adjfreq - adjust ptp cycle frequency * @ptp: the ptp clock structure @@ -395,7 +705,7 @@ static int fec_ptp_adjfreq(struct ptp_clock_info *ptp, s32 ppb) else corr_ns = fep->ptp_inc + corr_inc; - spin_lock_irqsave(&fep->tmreg_lock, flags); + raw_spin_lock_irqsave(&fep->tmreg_lock, flags); tmp = readl(fep->hwp + FEC_ATIME_INC) & FEC_T_INC_MASK; tmp |= corr_ns << FEC_T_INC_CORR_OFFSET; @@ -405,7 +715,7 @@ static int fec_ptp_adjfreq(struct ptp_clock_info *ptp, s32 ppb) /* dummy read to update the timer. */ timecounter_read(&fep->tc); - spin_unlock_irqrestore(&fep->tmreg_lock, flags); + raw_spin_unlock_irqrestore(&fep->tmreg_lock, flags); return 0; } @@ -423,40 +733,9 @@ static int fec_ptp_adjtime(struct ptp_clock_info *ptp, s64 delta) container_of(ptp, struct fec_enet_private, ptp_caps); unsigned long flags; - spin_lock_irqsave(&fep->tmreg_lock, flags); + raw_spin_lock_irqsave(&fep->tmreg_lock, flags); timecounter_adjtime(&fep->tc, delta); - spin_unlock_irqrestore(&fep->tmreg_lock, flags); - - return 0; -} - -/** - * fec_ptp_gettime - * @ptp: the ptp clock structure - * @ts: timespec structure to hold the current time value - * - * read the timecounter and return the correct value on ns, - * after converting it into a struct timespec. - */ -static int fec_ptp_gettime(struct ptp_clock_info *ptp, struct timespec64 *ts) -{ - struct fec_enet_private *adapter = - container_of(ptp, struct fec_enet_private, ptp_caps); - u64 ns; - unsigned long flags; - - mutex_lock(&adapter->ptp_clk_mutex); - /* Check the ptp clock */ - if (!adapter->ptp_clk_on) { - mutex_unlock(&adapter->ptp_clk_mutex); - return -EINVAL; - } - spin_lock_irqsave(&adapter->tmreg_lock, flags); - ns = timecounter_read(&adapter->tc); - spin_unlock_irqrestore(&adapter->tmreg_lock, flags); - mutex_unlock(&adapter->ptp_clk_mutex); - - *ts = ns_to_timespec64(ns); + raw_spin_unlock_irqrestore(&fep->tmreg_lock, flags); return 0; } @@ -492,21 +771,55 @@ static int fec_ptp_settime(struct ptp_clock_info *ptp, */ counter = ns & fep->cc.mask; - spin_lock_irqsave(&fep->tmreg_lock, flags); + raw_spin_lock_irqsave(&fep->tmreg_lock, flags); writel(counter, fep->hwp + FEC_ATIME); timecounter_init(&fep->tc, &fep->cc, ns); - spin_unlock_irqrestore(&fep->tmreg_lock, flags); + raw_spin_unlock_irqrestore(&fep->tmreg_lock, flags); mutex_unlock(&fep->ptp_clk_mutex); return 0; } +#endif /* CONFIG_AVB_SUPPORT */ + +/** + * fec_ptp_gettime + * @ptp: the ptp clock structure + * @ts: timespec structure to hold the current time value + * + * read the timecounter and return the correct value on ns, + * after converting it into a struct timespec. + */ +static int fec_ptp_gettime(struct ptp_clock_info *ptp, struct timespec64 *ts) +{ + struct fec_enet_private *adapter = + container_of(ptp, struct fec_enet_private, ptp_caps); + u64 ns; + u32 remainder; + unsigned long flags; + + mutex_lock(&adapter->ptp_clk_mutex); + /* Check the ptp clock */ + if (!adapter->ptp_clk_on) { + mutex_unlock(&adapter->ptp_clk_mutex); + return -EINVAL; + } + raw_spin_lock_irqsave(&adapter->tmreg_lock, flags); + ns = timecounter_read(&adapter->tc); + raw_spin_unlock_irqrestore(&adapter->tmreg_lock, flags); + mutex_unlock(&adapter->ptp_clk_mutex); + + ts->tv_sec = div_u64_rem(ns, 1000000000ULL, &remainder); + ts->tv_nsec = remainder; + + return 0; +} static int fec_ptp_pps_disable(struct fec_enet_private *fep, uint channel) { unsigned long flags; - spin_lock_irqsave(&fep->tmreg_lock, flags); + raw_spin_lock_irqsave(&fep->tmreg_lock, flags); writel(0, fep->hwp + FEC_TCSR(channel)); - spin_unlock_irqrestore(&fep->tmreg_lock, flags); + raw_spin_unlock_irqrestore(&fep->tmreg_lock, flags); return 0; } @@ -567,10 +880,10 @@ static int fec_ptp_enable(struct ptp_clock_info *ptp, mutex_unlock(&fep->ptp_clk_mutex); return -EOPNOTSUPP; } - spin_lock_irqsave(&fep->tmreg_lock, flags); + raw_spin_lock_irqsave(&fep->tmreg_lock, flags); /* Read current timestamp */ curr_time = timecounter_read(&fep->tc); - spin_unlock_irqrestore(&fep->tmreg_lock, flags); + raw_spin_unlock_irqrestore(&fep->tmreg_lock, flags); mutex_unlock(&fep->ptp_clk_mutex); /* Calculate time difference */ @@ -676,9 +989,9 @@ static void fec_time_keep(struct work_struct *work) mutex_lock(&fep->ptp_clk_mutex); if (fep->ptp_clk_on) { - spin_lock_irqsave(&fep->tmreg_lock, flags); + raw_spin_lock_irqsave(&fep->tmreg_lock, flags); timecounter_read(&fep->tc); - spin_unlock_irqrestore(&fep->tmreg_lock, flags); + raw_spin_unlock_irqrestore(&fep->tmreg_lock, flags); } mutex_unlock(&fep->ptp_clk_mutex); @@ -694,6 +1007,9 @@ static irqreturn_t fec_pps_interrupt(int irq, void *dev_id) u8 channel = fep->pps_channel; struct ptp_clock_event event; + if (fep->pps_enable) + goto exit; + val = readl(fep->hwp + FEC_TCSR(channel)); if (val & FEC_T_TF_MASK) { /* Write the next next compare(not the next according the spec) @@ -713,6 +1029,7 @@ static irqreturn_t fec_pps_interrupt(int irq, void *dev_id) return IRQ_HANDLED; } +exit: return IRQ_NONE; } @@ -736,7 +1053,18 @@ void fec_ptp_init(struct platform_device *pdev, int irq_idx) fep->ptp_caps.owner = THIS_MODULE; strscpy(fep->ptp_caps.name, "fec ptp", sizeof(fep->ptp_caps.name)); + fep->cycle_speed = clk_get_rate(fep->clk_ptp); + if (!fep->cycle_speed) { + fep->cycle_speed = NSEC_PER_SEC; + dev_err(&fep->pdev->dev, "clk_ptp clock rate is zero\n"); + } + fep->ptp_inc = NSEC_PER_SEC / fep->cycle_speed; + +#ifdef CONFIG_AVB_SUPPORT + fep->ptp_caps.max_adj = fep->cycle_speed / 2; +#else fep->ptp_caps.max_adj = 250000000; +#endif fep->ptp_caps.n_alarm = 0; fep->ptp_caps.n_ext_ts = 0; fep->ptp_caps.n_per_out = 1; @@ -748,14 +1076,7 @@ void fec_ptp_init(struct platform_device *pdev, int irq_idx) fep->ptp_caps.settime64 = fec_ptp_settime; fep->ptp_caps.enable = fec_ptp_enable; - fep->cycle_speed = clk_get_rate(fep->clk_ptp); - if (!fep->cycle_speed) { - fep->cycle_speed = NSEC_PER_SEC; - dev_err(&fep->pdev->dev, "clk_ptp clock rate is zero\n"); - } - fep->ptp_inc = NSEC_PER_SEC / fep->cycle_speed; - - spin_lock_init(&fep->tmreg_lock); + raw_spin_lock_init(&fep->tmreg_lock); fec_ptp_start_cyclecounter(ndev); diff --git a/drivers/net/ethernet/freescale/fec_uio.c b/drivers/net/ethernet/freescale/fec_uio.c index 7e93c9966c2b..b433868d4c5a 100644 --- a/drivers/net/ethernet/freescale/fec_uio.c +++ b/drivers/net/ethernet/freescale/fec_uio.c @@ -1188,8 +1188,8 @@ static int fec_enet_uio_init(struct net_device *ndev) return ret; } - tx_ring_size = RING_SIZE_TX; - rx_ring_size = RING_SIZE_RX; + tx_ring_size = FEC_TX_RING_SIZE; + rx_ring_size = FEC_RX_RING_SIZE; for (i = 0; i < FEC_MAX_Q; i++) { total_tx_ring_size += tx_ring_size; diff --git a/drivers/net/ethernet/freescale/gianfar_ethtool.c b/drivers/net/ethernet/freescale/gianfar_ethtool.c index b2b0d3c26fcc..370fe6e965b7 100644 --- a/drivers/net/ethernet/freescale/gianfar_ethtool.c +++ b/drivers/net/ethernet/freescale/gianfar_ethtool.c @@ -11,6 +11,7 @@ * Modifier: Sandeep Gopalpet * * Copyright 2003-2006, 2008-2009, 2011 Freescale Semiconductor, Inc. + * Copyright 2017-2023 NXP */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt @@ -166,6 +167,32 @@ static void gfar_gdrvinfo(struct net_device *dev, strscpy(drvinfo->driver, DRV_NAME, sizeof(drvinfo->driver)); } +static int gfar_get_eee(struct net_device *dev, struct ethtool_eee *et_eee) +{ + struct phy_device *phydev = dev->phydev; + + if (!phydev) + return -ENODEV; + + return phy_ethtool_get_eee(phydev, et_eee); +} + +static int gfar_set_eee(struct net_device *dev, struct ethtool_eee *et_eee) +{ + struct phy_device *phydev = dev->phydev; + + if (!phydev) + return -ENODEV; + + if (et_eee->eee_enabled || + et_eee->tx_lpi_enabled || + et_eee->tx_lpi_timer) { + return -EOPNOTSUPP; + } + + return phy_ethtool_set_eee(phydev, et_eee); +} + /* Return the length of the register structure */ static int gfar_reglen(struct net_device *dev) { @@ -1489,6 +1516,8 @@ static int gfar_get_ts_info(struct net_device *dev, const struct ethtool_ops gfar_ethtool_ops = { .supported_coalesce_params = ETHTOOL_COALESCE_USECS | ETHTOOL_COALESCE_MAX_FRAMES, + .get_eee = gfar_get_eee, + .set_eee = gfar_set_eee, .get_drvinfo = gfar_gdrvinfo, .get_regs_len = gfar_reglen, .get_regs = gfar_get_regs, diff --git a/drivers/net/ethernet/freescale/sdk_dpaa/Kconfig b/drivers/net/ethernet/freescale/sdk_dpaa/Kconfig index bb6693e2d702..9e219bbfd978 100644 --- a/drivers/net/ethernet/freescale/sdk_dpaa/Kconfig +++ b/drivers/net/ethernet/freescale/sdk_dpaa/Kconfig @@ -84,7 +84,7 @@ config FSL_DPAA_ETH_JUMBO_FRAME config FSL_DPAA_TS bool "Linux compliant timestamping" depends on FSL_SDK_DPAA_ETH - default n + default y help Enable Linux API compliant timestamping support. @@ -96,6 +96,15 @@ config FSL_DPAA_1588 help Enable IEEE1588 support code. +config FSL_DPAA_ETHERCAT + bool "Enable DPAA Ethercat support" + depends on FSL_SDK_DPAA_ETH + default n + help + Enable DPAA Ethercat support code, if enabling this feature, will create a new + QMan Portal for Ethercat port, and isolate the last cpu core for Ethercat traffic. + Regular ethernet traffic will NOT be in the isolated core. + config FSL_DPAA_ETH_MAX_BUF_COUNT int "Maximum number of buffers in the private bpool" depends on FSL_SDK_DPAA_ETH diff --git a/drivers/net/ethernet/freescale/sdk_dpaa/Makefile b/drivers/net/ethernet/freescale/sdk_dpaa/Makefile index a0623b24fa1c..9f9980099aad 100644 --- a/drivers/net/ethernet/freescale/sdk_dpaa/Makefile +++ b/drivers/net/ethernet/freescale/sdk_dpaa/Makefile @@ -11,6 +11,9 @@ ccflags-y += -I$(NET_DPA) obj-$(CONFIG_FSL_SDK_DPAA_ETH) += fsl_mac.o fsl_dpa.o fsl_dpa-objs += dpaa_ethtool.o dpaa_eth_sysfs.o dpaa_eth.o dpaa_eth_sg.o dpaa_eth_common.o +ifeq ($(CONFIG_FSL_DPAA_ETHERCAT),y) +fsl_dpa-objs += dpaa_ethercat.o +endif ifeq ($(CONFIG_FSL_DPAA_DBG_LOOP),y) fsl_dpa-objs += dpaa_debugfs.o endif diff --git a/drivers/net/ethernet/freescale/sdk_dpaa/dpaa_eth.h b/drivers/net/ethernet/freescale/sdk_dpaa/dpaa_eth.h index f8ab524b368c..1ccf55e475e7 100644 --- a/drivers/net/ethernet/freescale/sdk_dpaa/dpaa_eth.h +++ b/drivers/net/ethernet/freescale/sdk_dpaa/dpaa_eth.h @@ -1,5 +1,5 @@ /* Copyright 2008-2012 Freescale Semiconductor Inc. - * Copyright 2019 NXP + * Copyright 2019-2023 NXP * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -415,6 +415,12 @@ struct dpa_priv_s { #ifdef CONFIG_FSL_DPAA_CEETM bool ceetm_en; /* CEETM QoS enabled */ #endif + +#ifdef CONFIG_FSL_DPAA_ETHERCAT + uint16_t ethercat_channel; /* "fsl,qman-channel-id" */ + struct qman_portal *p; + void *ecdev; +#endif }; struct fm_port_fqs { @@ -461,6 +467,11 @@ int __hot skb_to_sg_fd(struct dpa_priv_s *priv, int __cold __attribute__((nonnull)) _dpa_fq_free(struct device *dev, struct qman_fq *fq); +#ifdef CONFIG_FSL_DPAA_ETHERCAT +int dpa_unregister_ethercat(struct net_device *net_dev); +int ec_dpaa_receive_data(void *pecdev, const void *data, size_t size); +#endif + /* Turn on HW checksum computation for this outgoing frame. * If the current protocol is not something we support in this regard * (or if the stack has already computed the SW checksum), we do nothing. diff --git a/drivers/net/ethernet/freescale/sdk_dpaa/dpaa_eth_common.c b/drivers/net/ethernet/freescale/sdk_dpaa/dpaa_eth_common.c index 8853c602d6af..641a7c0c0917 100644 --- a/drivers/net/ethernet/freescale/sdk_dpaa/dpaa_eth_common.c +++ b/drivers/net/ethernet/freescale/sdk_dpaa/dpaa_eth_common.c @@ -1,4 +1,5 @@ /* Copyright 2008-2013 Freescale Semiconductor, Inc. + * Copyright 2019-2023 NXP * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -478,7 +479,15 @@ int __cold dpa_remove(struct platform_device *of_dev) dpaa_eth_sysfs_remove(dev); dev_set_drvdata(dev, NULL); + +#ifdef CONFIG_FSL_DPAA_ETHERCAT + if (priv->ecdev) + dpa_unregister_ethercat(net_dev); + else + unregister_netdev(net_dev); +#else unregister_netdev(net_dev); +#endif err = dpa_fq_free(dev, &priv->dpa_fq_list); @@ -992,6 +1001,49 @@ void dpa_release_channel(void) } EXPORT_SYMBOL(dpa_release_channel); +#ifdef CONFIG_FSL_DPAA_ETHERCAT +static int ec_cpu_isolated[NR_CPUS]; +static int __init ethercat_cpus_setup(char *str) +{ + int last_idx = -1; + int dash_flag = 0; + int idx = 0; + int i = 0; + int j = 0; + + if (!str) + return 0; + + for (i = 0; i < strlen(str); i++) { + if (str[i] == ',') { + last_idx = -1; + dash_flag = 0; + continue; + } else if ((str[i] == '-') && (last_idx >= 0)) { + dash_flag = 1; + continue; + } + + if ((str[i] >= '0') && (str[i] <= '9')) { + idx = str[i] - '0'; + ec_cpu_isolated[idx] = 1; + + if (dash_flag) { + for (j = last_idx; j < idx; j++) + ec_cpu_isolated[j] = 1; + last_idx = -1; + dash_flag = 0; + } else { + last_idx = idx; + } + } + } + + return 1; +} +__setup("ethercat_cpus=", ethercat_cpus_setup); +#endif + void dpaa_eth_add_channel(u16 channel) { const cpumask_t *cpus = qman_affine_cpus(); @@ -1000,6 +1052,10 @@ void dpaa_eth_add_channel(u16 channel) struct qman_portal *portal; for_each_cpu(cpu, cpus) { +#ifdef CONFIG_FSL_DPAA_ETHERCAT + if (ec_cpu_isolated[cpu]) + continue; +#endif portal = (struct qman_portal *)qman_get_affine_portal(cpu); qman_p_static_dequeue_add(portal, pool); } diff --git a/drivers/net/ethernet/freescale/sdk_dpaa/dpaa_eth_sg.c b/drivers/net/ethernet/freescale/sdk_dpaa/dpaa_eth_sg.c index 24704236b500..30c66abf6a8d 100644 --- a/drivers/net/ethernet/freescale/sdk_dpaa/dpaa_eth_sg.c +++ b/drivers/net/ethernet/freescale/sdk_dpaa/dpaa_eth_sg.c @@ -1,5 +1,5 @@ /* Copyright 2012 Freescale Semiconductor Inc. - * Copyright 2019 NXP + * Copyright 2019-2023 NXP * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -608,10 +608,31 @@ void __hot _dpa_rx(struct net_device *net_dev, * which case we were in) having been removed from the pool. */ (*count_ptr)--; +#ifndef CONFIG_FSL_DPAA_ETHERCAT skb->protocol = eth_type_trans(skb, net_dev); - +#endif skb_len = skb->len; +#ifdef CONFIG_FSL_DPAA_ETHERCAT + if (priv->ecdev) { + u16 rawcpuid = 0; + u16 prot = 0; + + rawcpuid = (u16)raw_smp_processor_id(); + skb_record_rx_queue(skb, rawcpuid); + + prot = ntohs(*(u16 *)(skb->data + 12)); + if (prot == ETH_P_ETHERCAT) + ec_dpaa_receive_data(priv->ecdev, skb->data, skb->len); + else + pr_warn("invalid ethercat protocol 0x%x\n", prot); + + dev_kfree_skb(skb); + goto ethercat_tag; + } + skb->protocol = eth_type_trans(skb, net_dev); +#endif + #ifdef CONFIG_FSL_DPAA_DBG_LOOP if (dpa_skb_loop(priv, skb)) { percpu_stats->rx_packets++; @@ -635,6 +656,9 @@ void __hot _dpa_rx(struct net_device *net_dev, } else if (unlikely(netif_receive_skb(skb) == NET_RX_DROP)) return; +#ifdef CONFIG_FSL_DPAA_ETHERCAT +ethercat_tag: +#endif percpu_stats->rx_packets++; percpu_stats->rx_bytes += skb_len; diff --git a/drivers/net/ethernet/freescale/sdk_dpaa/dpaa_ethercat.c b/drivers/net/ethernet/freescale/sdk_dpaa/dpaa_ethercat.c new file mode 100644 index 000000000000..67fa72257a6f --- /dev/null +++ b/drivers/net/ethernet/freescale/sdk_dpaa/dpaa_ethercat.c @@ -0,0 +1,1217 @@ +// SPDX-License-Identifier: (GPL-2.0 OR MIT) +/* + * Copyright 2021-2023 NXP + */ + +#include +#include +#include +#include +#include +#include +#include /* arp_hdr_len() */ +#include /* VLAN_HLEN */ +#include /* struct icmphdr */ +#include /* struct iphdr */ +#include /* struct ipv6hdr */ +#include /* struct udphdr */ +#include /* struct tcphdr */ +#include /* net_ratelimit() */ +#include /* ETH_P_IP and ETH_P_IPV6 */ +#include +#include +#include +#include +#ifdef CONFIG_SOC_BUS +#include /* soc_device_match */ +#endif + +#include "fsl_fman.h" +#include "fm_ext.h" +#include "fm_port_ext.h" + +#include "mac.h" +#include "dpaa_eth.h" +#include "dpaa_eth_common.h" +#ifdef CONFIG_FSL_DPAA_DBG_LOOP +#include "dpaa_debugfs.h" +#endif /* CONFIG_FSL_DPAA_DBG_LOOP */ + +#define DPA_NAPI_WEIGHT 64 + +/* Valid checksum indication */ +#define DPA_CSUM_VALID 0xFFFF + +#define DPA_DESCRIPTION "FSL/NXP DPAA Ethercat driver" + +MODULE_LICENSE("Dual BSD/GPL"); + +MODULE_DESCRIPTION(DPA_DESCRIPTION); + +static u8 debug = -1; + +/* This has to work in tandem with the DPA_CS_THRESHOLD_xxx values. */ +static u16 tx_timeout = 1000; + +static const char rtx[][3] = { + [RX] = "RX", + [TX] = "TX" +}; + +/* BM */ + +#define DPAA_ETH_MAX_PAD (L1_CACHE_BYTES * 8) + +static u32 dpa_priv_bpid; +static u32 dpa_priv_cpuid; + +#ifdef CONFIG_FSL_DPAA_DBG_LOOP +struct net_device *dpa_loop_netdevs[20]; +#endif + +#ifdef CONFIG_PM + +static int dpaa_suspend(struct device *dev) +{ + struct net_device *net_dev; + struct dpa_priv_s *priv; + struct mac_device *mac_dev; + int err = 0; + + net_dev = dev_get_drvdata(dev); + + if (net_dev->flags & IFF_UP) { + priv = netdev_priv(net_dev); + mac_dev = priv->mac_dev; + + if (priv->wol & DPAA_WOL_MAGIC) { + err = priv->mac_dev->set_wol(mac_dev->port_dev[RX], + priv->mac_dev->get_mac_handle(mac_dev), true); + if (err) { + netdev_err(net_dev, "set_wol() = %d\n", err); + goto set_wol_failed; + } + } + + err = fm_port_suspend(mac_dev->port_dev[RX]); + if (err) { + netdev_err(net_dev, "fm_port_suspend(RX) = %d\n", err); + goto rx_port_suspend_failed; + } + + err = fm_port_suspend(mac_dev->port_dev[TX]); + if (err) { + netdev_err(net_dev, "fm_port_suspend(TX) = %d\n", err); + goto tx_port_suspend_failed; + } + } + + return 0; + +tx_port_suspend_failed: + fm_port_resume(mac_dev->port_dev[RX]); +rx_port_suspend_failed: + if (priv->wol & DPAA_WOL_MAGIC) { + priv->mac_dev->set_wol(mac_dev->port_dev[RX], + priv->mac_dev->get_mac_handle(mac_dev), false); + } +set_wol_failed: + return err; +} + +static int dpaa_resume(struct device *dev) +{ + struct net_device *net_dev; + struct dpa_priv_s *priv; + struct mac_device *mac_dev; + int err = 0; + + net_dev = dev_get_drvdata(dev); + + if (net_dev->flags & IFF_UP) { + priv = netdev_priv(net_dev); + mac_dev = priv->mac_dev; + + err = fm_mac_resume(mac_dev->get_mac_handle(mac_dev)); + if (err) { + netdev_err(net_dev, "fm_mac_resume = %d\n", err); + goto resume_failed; + } + + err = fm_port_resume(mac_dev->port_dev[TX]); + if (err) { + netdev_err(net_dev, "fm_port_resume(TX) = %d\n", err); + goto resume_failed; + } + + err = fm_port_resume(mac_dev->port_dev[RX]); + if (err) { + netdev_err(net_dev, "fm_port_resume(RX) = %d\n", err); + goto resume_failed; + } + + if (priv->wol & DPAA_WOL_MAGIC) { + err = priv->mac_dev->set_wol(mac_dev->port_dev[RX], + priv->mac_dev->get_mac_handle(mac_dev), false); + if (err) { + netdev_err(net_dev, "set_wol() = %d\n", err); + goto resume_failed; + } + } + } + + return 0; + +resume_failed: + return err; +} + +static const struct dev_pm_ops dpaa_pm_ops = { + .suspend = dpaa_suspend, + .resume = dpaa_resume, +}; + +#define DPAA_PM_OPS (&dpaa_pm_ops) + +#else /* CONFIG_PM */ + +#define DPAA_PM_OPS NULL + +#endif /* CONFIG_PM */ + +/* Checks whether the checksum field in Parse Results array is valid + * (equals 0xFFFF) and increments the .cse counter otherwise + */ +static inline void +dpa_csum_validation(const struct dpa_priv_s *priv, + struct dpa_percpu_priv_s *percpu_priv, + const struct qm_fd *fd) +{ + dma_addr_t addr = qm_fd_addr(fd); + struct dpa_bp *dpa_bp = priv->dpa_bp; + void *frm = phys_to_virt(addr); + fm_prs_result_t *parse_result; + + if (unlikely(!frm)) + return; + + dma_sync_single_for_cpu(dpa_bp->dev, addr, DPA_RX_PRIV_DATA_SIZE + + DPA_PARSE_RESULTS_SIZE, DMA_BIDIRECTIONAL); + + parse_result = (fm_prs_result_t *)(frm + DPA_RX_PRIV_DATA_SIZE); + + if (parse_result->cksum != DPA_CSUM_VALID) + percpu_priv->rx_errors.cse++; +} + +static void _dpa_rx_error(struct net_device *net_dev, + const struct dpa_priv_s *priv, + struct dpa_percpu_priv_s *percpu_priv, + const struct qm_fd *fd, + u32 fqid) +{ + /* limit common, possibly innocuous Rx FIFO Overflow errors' + * interference with zero-loss convergence benchmark results. + */ + if (likely(fd->status & FM_FD_STAT_ERR_PHYSICAL)) + pr_warn_once("fsl-dpa: non-zero error counters in fman statistics (sysfs)\n"); + else + if (netif_msg_hw(priv) && net_ratelimit()) + netdev_dbg(net_dev, "Err FD status = 0x%08x\n", + fd->status & FM_FD_STAT_RX_ERRORS); +#ifdef CONFIG_FSL_DPAA_HOOKS + if (dpaa_eth_hooks.rx_error && + dpaa_eth_hooks.rx_error(net_dev, fd, fqid) == DPAA_ETH_STOLEN) + /* it's up to the hook to perform resource cleanup */ + return; +#endif + percpu_priv->stats.rx_errors++; + + if (fd->status & FM_PORT_FRM_ERR_DMA) + percpu_priv->rx_errors.dme++; + if (fd->status & FM_PORT_FRM_ERR_PHYSICAL) + percpu_priv->rx_errors.fpe++; + if (fd->status & FM_PORT_FRM_ERR_SIZE) + percpu_priv->rx_errors.fse++; + if (fd->status & FM_PORT_FRM_ERR_PRS_HDR_ERR) + percpu_priv->rx_errors.phe++; + if (fd->status & FM_FD_STAT_L4CV) + dpa_csum_validation(priv, percpu_priv, fd); + + dpa_fd_release(net_dev, fd); +} + +static void _dpa_tx_error(struct net_device *net_dev, + const struct dpa_priv_s *priv, + struct dpa_percpu_priv_s *percpu_priv, + const struct qm_fd *fd, + u32 fqid) +{ + struct sk_buff *skb; + + if (netif_msg_hw(priv) && net_ratelimit()) + netdev_warn(net_dev, "FD status = 0x%08x\n", + fd->status & FM_FD_STAT_TX_ERRORS); +#ifdef CONFIG_FSL_DPAA_HOOKS + if (dpaa_eth_hooks.tx_error && + dpaa_eth_hooks.tx_error(net_dev, fd, fqid) == DPAA_ETH_STOLEN) + /* now the hook must ensure proper cleanup */ + return; +#endif + percpu_priv->stats.tx_errors++; + + /* If we intended the buffers from this frame to go into the bpools + * when the FMan transmit was done, we need to put it in manually. + */ + if (fd->bpid != 0xff) { + dpa_fd_release(net_dev, fd); + return; + } + + skb = _dpa_cleanup_tx_fd(priv, fd); + if (!priv->ecdev) + dev_kfree_skb(skb); +} + +static void __hot _dpa_tx_conf(struct net_device *net_dev, + const struct dpa_priv_s *priv, + struct dpa_percpu_priv_s *percpu_priv, + const struct qm_fd *fd, + u32 fqid) +{ + struct sk_buff *skb; + + /* do we need the timestamp for the error frames? */ + + if (unlikely(fd->status & FM_FD_STAT_TX_ERRORS) != 0) { + if (netif_msg_hw(priv) && net_ratelimit()) + netdev_warn(net_dev, "FD status = 0x%08x\n", + fd->status & FM_FD_STAT_TX_ERRORS); + + percpu_priv->stats.tx_errors++; + } + + /* hopefully we need not get the timestamp before the hook */ +#ifdef CONFIG_FSL_DPAA_HOOKS + if (dpaa_eth_hooks.tx_confirm && + dpaa_eth_hooks.tx_confirm(net_dev, fd, fqid) == DPAA_ETH_STOLEN) + /* it's the hook that must now perform cleanup */ + return; +#endif + /* This might not perfectly reflect the reality, if the core dequeuing + * the Tx confirmation is different from the one that did the enqueue, + * but at least it'll show up in the total count. + */ + percpu_priv->tx_confirm++; + + skb = _dpa_cleanup_tx_fd(priv, fd); + if (!priv->ecdev) + dev_kfree_skb(skb); +} + +static enum qman_cb_dqrr_result +priv_rx_error_dqrr(struct qman_portal *portal, + struct qman_fq *fq, + const struct qm_dqrr_entry *dq) +{ + struct net_device *net_dev; + struct dpa_priv_s *priv; + struct dpa_percpu_priv_s *percpu_priv; + int *count_ptr; + + net_dev = ((struct dpa_fq *)fq)->net_dev; + priv = netdev_priv(net_dev); + + percpu_priv = raw_cpu_ptr(priv->percpu_priv); + count_ptr = raw_cpu_ptr(priv->percpu_count); + + if (unlikely(dpaa_eth_refill_bpools(priv->dpa_bp, count_ptr))) + /* Unable to refill the buffer pool due to insufficient + * system memory. Just release the frame back into the pool, + * otherwise we'll soon end up with an empty buffer pool. + */ + dpa_fd_release(net_dev, &dq->fd); + else + _dpa_rx_error(net_dev, priv, percpu_priv, &dq->fd, fq->fqid); + + return qman_cb_dqrr_consume; +} + +static enum qman_cb_dqrr_result __hot +priv_rx_default_dqrr(struct qman_portal *portal, + struct qman_fq *fq, + const struct qm_dqrr_entry *dq) +{ + struct net_device *net_dev; + struct dpa_priv_s *priv; + struct dpa_percpu_priv_s *percpu_priv; + int *count_ptr; + struct dpa_bp *dpa_bp; + + net_dev = ((struct dpa_fq *)fq)->net_dev; + priv = netdev_priv(net_dev); + dpa_bp = priv->dpa_bp; + + /* IRQ handler, non-migratable; safe to use raw_cpu_ptr here */ + percpu_priv = raw_cpu_ptr(priv->percpu_priv); + count_ptr = raw_cpu_ptr(priv->percpu_count); + + /* Vale of plenty: make sure we didn't run out of buffers */ + + if (unlikely(dpaa_eth_refill_bpools(dpa_bp, count_ptr))) + /* Unable to refill the buffer pool due to insufficient + * system memory. Just release the frame back into the pool, + * otherwise we'll soon end up with an empty buffer pool. + */ + dpa_fd_release(net_dev, &dq->fd); + else + _dpa_rx(net_dev, portal, priv, percpu_priv, &dq->fd, fq->fqid, + count_ptr); + + return qman_cb_dqrr_consume; +} + +static enum qman_cb_dqrr_result +priv_tx_conf_error_dqrr(struct qman_portal *portal, + struct qman_fq *fq, + const struct qm_dqrr_entry *dq) +{ + struct net_device *net_dev; + struct dpa_priv_s *priv; + struct dpa_percpu_priv_s *percpu_priv; + + net_dev = ((struct dpa_fq *)fq)->net_dev; + priv = netdev_priv(net_dev); + + percpu_priv = raw_cpu_ptr(priv->percpu_priv); + + _dpa_tx_error(net_dev, priv, percpu_priv, &dq->fd, fq->fqid); + + return qman_cb_dqrr_consume; +} + +static enum qman_cb_dqrr_result __hot +priv_tx_conf_default_dqrr(struct qman_portal *portal, + struct qman_fq *fq, + const struct qm_dqrr_entry *dq) +{ + struct net_device *net_dev; + struct dpa_priv_s *priv; + struct dpa_percpu_priv_s *percpu_priv; + + net_dev = ((struct dpa_fq *)fq)->net_dev; + priv = netdev_priv(net_dev); + + /* Non-migratable context, safe to use raw_cpu_ptr */ + percpu_priv = raw_cpu_ptr(priv->percpu_priv); + + _dpa_tx_conf(net_dev, priv, percpu_priv, &dq->fd, fq->fqid); + + return qman_cb_dqrr_consume; +} + +static void priv_ern(struct qman_portal *portal, + struct qman_fq *fq, + const struct qm_mr_entry *msg) +{ + struct net_device *net_dev; + const struct dpa_priv_s *priv; + struct sk_buff *skb; + struct dpa_percpu_priv_s *percpu_priv; + struct qm_fd fd = msg->ern.fd; + + net_dev = ((struct dpa_fq *)fq)->net_dev; + priv = netdev_priv(net_dev); + /* Non-migratable context, safe to use raw_cpu_ptr */ + percpu_priv = raw_cpu_ptr(priv->percpu_priv); + + percpu_priv->stats.tx_dropped++; + percpu_priv->stats.tx_fifo_errors++; + count_ern(percpu_priv, msg); + + /* If we intended this buffer to go into the pool + * when the FM was done, we need to put it in + * manually. + */ + if (msg->ern.fd.bpid != 0xff) { + dpa_fd_release(net_dev, &fd); + return; + } + + skb = _dpa_cleanup_tx_fd(priv, &fd); + dev_kfree_skb_any(skb); +} + +static const struct dpa_fq_cbs_t private_fq_cbs = { + .rx_defq = { .cb = { .dqrr = priv_rx_default_dqrr } }, + .tx_defq = { .cb = { .dqrr = priv_tx_conf_default_dqrr } }, + .rx_errq = { .cb = { .dqrr = priv_rx_error_dqrr } }, + .tx_errq = { .cb = { .dqrr = priv_tx_conf_error_dqrr } }, + .egress_ern = { .cb = { .ern = priv_ern } } +}; + +static int __cold dpa_eth_priv_start(struct net_device *net_dev) +{ + int err; + struct dpa_priv_s *priv; + + priv = netdev_priv(net_dev); + + err = dpa_start(net_dev); + + return err; +} + +static int __cold dpa_eth_priv_stop(struct net_device *net_dev) +{ + int _errno; + struct dpa_priv_s *priv; + + _errno = dpa_stop(net_dev); + /* Allow NAPI to consume any frame still in the Rx/TxConfirm + * ingress queues. This is to avoid a race between the current + * context and ksoftirqd which could leave NAPI disabled while + * in fact there's still Rx traffic to be processed. + */ + usleep_range(5000, 10000); + + priv = netdev_priv(net_dev); + + return _errno; +} + +static const struct net_device_ops dpa_private_ops = { + .ndo_open = dpa_eth_priv_start, + .ndo_start_xmit = dpa_tx, + .ndo_stop = dpa_eth_priv_stop, + .ndo_tx_timeout = dpa_timeout, + .ndo_get_stats64 = dpa_get_stats64, + .ndo_set_mac_address = dpa_set_mac_address, + .ndo_validate_addr = eth_validate_addr, +#ifdef CONFIG_FMAN_PFC + .ndo_select_queue = dpa_select_queue, +#endif + .ndo_set_rx_mode = dpa_set_rx_mode, + .ndo_init = dpa_ndo_init, + .ndo_set_features = dpa_set_features, + .ndo_fix_features = dpa_fix_features, + .ndo_do_ioctl = dpa_ioctl, +}; + +typedef int (*ec_dpaa_receive_cb)(void *pecdev, const void *data, size_t size); +typedef int (*ec_dpaa_link_cb)(void *pecdev, uint8_t link); +typedef int (*ec_dpaa_close_cb)(void *pecdev); + +static ec_dpaa_receive_cb ec_dpaa_recv_func; +static ec_dpaa_close_cb ec_dpaa_close_func; +static ec_dpaa_link_cb ec_dpaa_link_func; + +int ec_dpaa_receive_data(void *pecdev, const void *data, size_t size) +{ + int ret = 0; + + if (ec_dpaa_recv_func) + ret = ec_dpaa_recv_func(pecdev, data, size); + + return ret; +} + +int ec_dpaa_set_func_cb(ec_dpaa_receive_cb recv, ec_dpaa_link_cb link, ec_dpaa_close_cb close) +{ + ec_dpaa_recv_func = recv; + ec_dpaa_link_func = link; + ec_dpaa_close_func = close; + + return 0; +} +EXPORT_SYMBOL(ec_dpaa_set_func_cb); + +struct module *ec_dpaa_get_module(void) +{ + struct module *m = THIS_MODULE; + return m; +} +EXPORT_SYMBOL(ec_dpaa_get_module); + +#define MAX_EC_DPAA_NETDEV_CNT (16) +static int ec_dpaa_netdev_cnt; +static struct net_device *ec_dpaa_netdev[MAX_EC_DPAA_NETDEV_CNT]; + +struct net_device *ec_dpaa_get_netdev(int idx) +{ + if (idx >= MAX_EC_DPAA_NETDEV_CNT) + return NULL; + + if (idx >= ec_dpaa_netdev_cnt) + return NULL; + + return ec_dpaa_netdev[idx]; +} +EXPORT_SYMBOL(ec_dpaa_get_netdev); + +int ec_dpaa_set_ecdev(int idx, void *ecdev) +{ + struct net_device *net_dev = NULL; + struct dpa_priv_s *priv = NULL; + + net_dev = ec_dpaa_get_netdev(idx); + if (net_dev) { + priv = netdev_priv(net_dev); + priv->ecdev = ecdev; + } + + return 0; +} +EXPORT_SYMBOL(ec_dpaa_set_ecdev); + +void *ec_dpaa_get_ecdev(int idx) +{ + struct net_device *net_dev = NULL; + struct dpa_priv_s *priv = NULL; + void *ecdev = NULL; + + net_dev = ec_dpaa_get_netdev(idx); + if (net_dev) { + priv = netdev_priv(net_dev); + ecdev = priv->ecdev; + } + + return ecdev; +} +EXPORT_SYMBOL(ec_dpaa_get_ecdev); + +void ec_dpaa_poll(struct net_device *net_dev) +{ + struct dpa_priv_s *priv = netdev_priv(net_dev); + u8 link = net_dev->phydev->state; + + qman_p_poll_dqrr(priv->p, DPA_NAPI_WEIGHT); + + if (ec_dpaa_link_func) + ec_dpaa_link_func(priv->ecdev, link); +} +EXPORT_SYMBOL(ec_dpaa_poll); + +int dpa_unregister_ethercat(struct net_device *net_dev) +{ + struct dpa_priv_s *priv = netdev_priv(net_dev); + + if (priv->ecdev) { + if (ec_dpaa_close_func) + ec_dpaa_close_func(priv->ecdev); + return 0; + } + + return -1; +} +EXPORT_SYMBOL(dpa_unregister_ethercat); + +static int dpa_ethercat_netdev_init(struct net_device *net_dev, + const u8 *mac_addr, + uint16_t tx_timeout) +{ + struct dpa_priv_s *priv = netdev_priv(net_dev); + + net_dev->priv_flags |= IFF_LIVE_ADDR_CHANGE; + + net_dev->features |= net_dev->hw_features; + net_dev->vlan_features = net_dev->features; + + memcpy(net_dev->perm_addr, mac_addr, net_dev->addr_len); + memcpy(net_dev->dev_addr, mac_addr, net_dev->addr_len); + + net_dev->ethtool_ops = &dpa_ethtool_ops; + + net_dev->needed_headroom = priv->tx_headroom; + net_dev->watchdog_timeo = msecs_to_jiffies(tx_timeout); + + if (ec_dpaa_netdev_cnt < MAX_EC_DPAA_NETDEV_CNT) + ec_dpaa_netdev[ec_dpaa_netdev_cnt++] = net_dev; + + return 0; +} + +static int dpa_private_netdev_init(struct net_device *net_dev) +{ + int i; + struct dpa_priv_s *priv = netdev_priv(net_dev); + struct dpa_percpu_priv_s *percpu_priv; + const u8 *mac_addr; + + /* Although we access another CPU's private data here + * we do it at initialization so it is safe + */ + for_each_possible_cpu(i) { + percpu_priv = per_cpu_ptr(priv->percpu_priv, i); + percpu_priv->net_dev = net_dev; + } + + net_dev->netdev_ops = &dpa_private_ops; + mac_addr = priv->mac_dev->addr; + + net_dev->mem_start = priv->mac_dev->res->start; + net_dev->mem_end = priv->mac_dev->res->end; + + /* Configure the maximum MTU according to the FMan's MAXFRM */ + net_dev->min_mtu = ETH_MIN_MTU; + net_dev->max_mtu = dpa_get_max_mtu(); + + net_dev->hw_features |= (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | + NETIF_F_LLTX); + + /* Advertise S/G and HIGHDMA support for private interfaces */ + net_dev->hw_features |= NETIF_F_SG | NETIF_F_HIGHDMA; + /* Recent kernels enable GSO automatically, if + * we declare NETIF_F_SG. For conformity, we'll + * still declare GSO explicitly. + */ + net_dev->features |= NETIF_F_GSO; + + /* Advertise GRO support */ + net_dev->features |= NETIF_F_GRO; + + return dpa_ethercat_netdev_init(net_dev, mac_addr, tx_timeout); +} + +static struct dpa_bp * __cold +dpa_priv_bp_probe(struct device *dev) +{ + struct dpa_bp *dpa_bp; + + dpa_bp = devm_kzalloc(dev, sizeof(*dpa_bp), GFP_KERNEL); + if (unlikely(!dpa_bp)) { + dev_err(dev, "devm_kzalloc() failed\n"); + return ERR_PTR(-ENOMEM); + } + + dpa_bp->target_count = CONFIG_FSL_DPAA_ETH_MAX_BUF_COUNT; + + dpa_bp->free_buf_cb = _dpa_bp_free_pf; + + return dpa_bp; +} + +/* Place all ingress FQs (Rx Default, Rx Error, PCD FQs) in a dedicated CGR. + * We won't be sending congestion notifications to FMan; for now, we just use + * this CGR to generate enqueue rejections to FMan in order to drop the frames + * before they reach our ingress queues and eat up memory. + */ +static int dpaa_eth_priv_ingress_cgr_init(struct dpa_priv_s *priv) +{ + struct qm_mcc_initcgr initcgr; + u32 cs_th; + int err; + + err = qman_alloc_cgrid(&priv->ingress_cgr.cgrid); + if (err < 0) { + pr_err("Error %d allocating CGR ID\n", err); + goto out_error; + } + + /* Enable CS TD, but disable Congestion State Change Notifications. */ + memset(&initcgr, 0, sizeof(initcgr)); + initcgr.we_mask = QM_CGR_WE_CS_THRES; + initcgr.cgr.cscn_en = QM_CGR_EN; + cs_th = CONFIG_FSL_DPAA_INGRESS_CS_THRESHOLD; + qm_cgr_cs_thres_set64(&initcgr.cgr.cs_thres, cs_th, 1); + + initcgr.we_mask |= QM_CGR_WE_CSTD_EN; + initcgr.cgr.cstd_en = QM_CGR_EN; + + /* This is actually a hack, because this CGR will be associated with + * our affine SWP. However, we'll place our ingress FQs in it. + */ + err = qman_create_cgr(&priv->ingress_cgr, QMAN_CGR_FLAG_USE_INIT, + &initcgr); + if (err < 0) { + pr_err("Error %d creating ingress CGR with ID %d\n", err, + priv->ingress_cgr.cgrid); + qman_release_cgrid(priv->ingress_cgr.cgrid); + goto out_error; + } + pr_debug("Created ingress CGR %d for netdev with hwaddr %pM\n", + priv->ingress_cgr.cgrid, priv->mac_dev->addr); + + /* struct qman_cgr allows special cgrid values (i.e. outside the 0..255 + * range), but we have no common initialization path between the + * different variants of the DPAA Eth driver, so we do it here rather + * than modifying every other variant than "private Eth". + */ + priv->use_ingress_cgr = true; + +out_error: + return err; +} + +static void dpa_priv_bp_seed(struct net_device *net_dev) +{ + struct dpa_priv_s *priv = netdev_priv(net_dev); + struct dpa_bp *dpa_bp = priv->dpa_bp; + int i; + + /* Give each CPU an allotment of buffers */ + for_each_possible_cpu(i) { + /* Although we access another CPU's counters here + * we do it at boot time so it is safe + */ + int *count_ptr = per_cpu_ptr(priv->percpu_count, i); + + dpaa_eth_refill_bpools(dpa_bp, count_ptr); + } +} + +static const struct of_device_id dpa_match[]; + +#ifdef CONFIG_FSL_DPAA_DBG_LOOP +static int dpa_new_loop_id(void) +{ + static int if_id; + + return if_id++; +} +#endif + +extern int dpa_bp_create(struct net_device *net_dev, struct dpa_bp *dpa_bp, + size_t count); + +static inline void dpa_setup_ingress(const struct dpa_priv_s *priv, + struct dpa_fq *fq, + const struct qman_fq *template) +{ + fq->fq_base = *template; + fq->net_dev = priv->net_dev; + + fq->flags = QMAN_FQ_FLAG_NO_ENQUEUE; + fq->channel = priv->channel; +} + +static inline void dpa_setup_egress(const struct dpa_priv_s *priv, + struct dpa_fq *fq, + struct fm_port *port, + const struct qman_fq *template) +{ + fq->fq_base = *template; + fq->net_dev = priv->net_dev; + + if (port) { + fq->flags = QMAN_FQ_FLAG_TO_DCPORTAL; + fq->channel = (u16)fm_get_tx_port_channel(port); + } else { + fq->flags = QMAN_FQ_FLAG_NO_MODIFY; + } +} + +static void dpa_fq_setup_ethercat(struct dpa_priv_s *priv, const struct dpa_fq_cbs_t *fq_cbs, + struct fm_port *tx_port) +{ + struct dpa_fq *fq; + u16 portals[NR_CPUS]; + int cpu, portal_cnt = 0, num_portals = 0; + u32 pcd_fqid, pcd_fqid_hi_prio; + const cpumask_t *affine_cpus = qman_affine_cpus(); + int egress_cnt = 0, conf_cnt = 0; + + /* Prepare for PCD FQs init */ + for_each_cpu(cpu, affine_cpus) + portals[num_portals++] = priv->ethercat_channel; + + if (num_portals == 0) + dev_err(priv->net_dev->dev.parent, + "No Qman software (affine) channels found"); + + pcd_fqid = (priv->mac_dev) ? + DPAA_ETH_PCD_FQ_BASE(priv->mac_dev->res->start) : 0; + pcd_fqid_hi_prio = (priv->mac_dev) ? + DPAA_ETH_PCD_FQ_HI_PRIO_BASE(priv->mac_dev->res->start) : 0; + + /* Initialize each FQ in the list */ + list_for_each_entry(fq, &priv->dpa_fq_list, list) { + switch (fq->fq_type) { + case FQ_TYPE_RX_DEFAULT: + WARN_ON(!priv->mac_dev); + dpa_setup_ingress(priv, fq, &fq_cbs->rx_defq); + break; + case FQ_TYPE_RX_ERROR: + WARN_ON(!priv->mac_dev); + dpa_setup_ingress(priv, fq, &fq_cbs->rx_errq); + break; + case FQ_TYPE_RX_PCD: + /* For MACless we can't have dynamic Rx queues */ + WARN_ON(!priv->mac_dev && !fq->fqid); + dpa_setup_ingress(priv, fq, &fq_cbs->rx_defq); + if (!fq->fqid) + fq->fqid = pcd_fqid++; + fq->channel = portals[portal_cnt]; + portal_cnt = (portal_cnt + 1) % num_portals; + break; + case FQ_TYPE_RX_PCD_HI_PRIO: + /* For MACless we can't have dynamic Hi Pri Rx queues */ + WARN_ON(!priv->mac_dev && !fq->fqid); + dpa_setup_ingress(priv, fq, &fq_cbs->rx_defq); + if (!fq->fqid) + fq->fqid = pcd_fqid_hi_prio++; + fq->channel = portals[portal_cnt]; + portal_cnt = (portal_cnt + 1) % num_portals; + break; + case FQ_TYPE_TX: + dpa_setup_egress(priv, fq, tx_port, + &fq_cbs->egress_ern); + /* If we have more Tx queues than the number of cores, + * just ignore the extra ones. + */ + if (egress_cnt < DPAA_ETH_TX_QUEUES) + priv->egress_fqs[egress_cnt++] = &fq->fq_base; + break; + case FQ_TYPE_TX_CONFIRM: + WARN_ON(!priv->mac_dev); + dpa_setup_ingress(priv, fq, &fq_cbs->tx_defq); + break; + case FQ_TYPE_TX_CONF_MQ: + WARN_ON(!priv->mac_dev); + dpa_setup_ingress(priv, fq, &fq_cbs->tx_defq); + priv->conf_fqs[conf_cnt++] = &fq->fq_base; + break; + case FQ_TYPE_TX_ERROR: + WARN_ON(!priv->mac_dev); + dpa_setup_ingress(priv, fq, &fq_cbs->tx_errq); + break; + default: + dev_warn(priv->net_dev->dev.parent, + "Unknown FQ type detected!\n"); + break; + } + } + + /* The number of Tx queues may be smaller than the number of cores, if + * the Tx queue range is specified in the device tree instead of being + * dynamically allocated. + * Make sure all CPUs receive a corresponding Tx queue. + */ + while (egress_cnt < DPAA_ETH_TX_QUEUES) { + list_for_each_entry(fq, &priv->dpa_fq_list, list) { + if (fq->fq_type != FQ_TYPE_TX) + continue; + priv->egress_fqs[egress_cnt++] = &fq->fq_base; + if (egress_cnt == DPAA_ETH_TX_QUEUES) + break; + } + } +} + +static int dpaa_ethercat_probe(struct platform_device *_of_dev) +{ + int err = 0, i; + struct device *dev; + struct device_node *dpa_node; + struct dpa_bp *dpa_bp; + size_t count = 1; + struct net_device *net_dev = NULL; + struct dpa_priv_s *priv = NULL; + struct dpa_percpu_priv_s *percpu_priv; + struct fm_port_fqs port_fqs; + struct dpa_buffer_layout_s *buf_layout = NULL; + struct mac_device *mac_dev; + u32 last_cpu = 0; + + dev = &_of_dev->dev; + + dpa_node = dev->of_node; + + if (!of_device_is_available(dpa_node)) + return -ENODEV; + + /* Get the buffer pools assigned to this interface; + * run only once the default pool probing code + */ + + err = of_property_read_u32(dpa_node, "fsl,bpid", &dpa_priv_bpid); + if (err) { + dev_err(dev, "Cannot find buffer pool ID in the device tree\n"); + return -EINVAL; + } + + last_cpu = qman_get_affine_last_cpu(); + err = of_property_read_u32(dpa_node, "fsl,cpuid", &dpa_priv_cpuid); + if (err) { + dev_warn(dev, "Cannot find cpuid in the device tree, using the last core\n"); + dpa_priv_cpuid = last_cpu; + } else if (dpa_priv_cpuid > last_cpu) { + dev_warn(dev, "Invalid cpuid in the device tree, using the last core\n"); + dpa_priv_cpuid = last_cpu; + } + + dpa_bp = (dpa_bpid2pool(dpa_priv_bpid)) ? : + dpa_priv_bp_probe(dev); + if (IS_ERR(dpa_bp)) + return PTR_ERR(dpa_bp); + if (dpa_bp->bpid == 0) + dpa_bp->bpid = dpa_priv_bpid; + + /* Allocate this early, so we can store relevant information in + * the private area (needed by 1588 code in dpa_mac_probe) + */ + net_dev = alloc_etherdev_mq(sizeof(*priv), DPAA_ETH_TX_QUEUES); + if (!net_dev) { + dev_err(dev, "alloc_etherdev_mq() failed\n"); + goto alloc_etherdev_mq_failed; + } + + /* Do this here, so we can be verbose early */ + SET_NETDEV_DEV(net_dev, dev); + dev_set_drvdata(dev, net_dev); + + priv = netdev_priv(net_dev); + priv->net_dev = net_dev; + strcpy(priv->if_type, "private"); + + priv->msg_enable = netif_msg_init(debug, -1); + +#ifdef CONFIG_FSL_DPAA_DBG_LOOP + priv->loop_id = dpa_new_loop_id(); + priv->loop_to = -1; /* disabled by default */ + dpa_loop_netdevs[priv->loop_id] = net_dev; +#endif + + mac_dev = dpa_mac_probe(_of_dev); + if (IS_ERR(mac_dev) || !mac_dev) { + err = PTR_ERR(mac_dev); + goto mac_probe_failed; + } + + /* We have physical ports, so we need to establish + * the buffer layout. + */ + buf_layout = devm_kzalloc(dev, 2 * sizeof(*buf_layout), + GFP_KERNEL); + if (!buf_layout) { + dev_err(dev, "devm_kzalloc() failed\n"); + goto alloc_failed; + } + dpa_set_buffers_layout(mac_dev, buf_layout); + + /* For private ports, need to compute the size of the default + * buffer pool, based on FMan port buffer layout;also update + * the maximum buffer size for private ports if necessary + */ + dpa_bp->size = dpa_bp_size(&buf_layout[RX]); + +#ifdef CONFIG_FSL_DPAA_ETH_JUMBO_FRAME + /* We only want to use jumbo frame optimization if we actually have + * L2 MAX FRM set for jumbo frames as well. + */ + if (fm_get_max_frm() < 9600) + dev_warn(dev, + "Invalid configuration: if jumbo frames support is on, FSL_FM_MAX_FRAME_SIZE should be set to 9600\n"); +#endif + + INIT_LIST_HEAD(&priv->dpa_fq_list); + + memset(&port_fqs, 0, sizeof(port_fqs)); + + err = dpa_fq_probe_mac(dev, &priv->dpa_fq_list, &port_fqs, true, RX); + if (!err) + err = dpa_fq_probe_mac(dev, &priv->dpa_fq_list, + &port_fqs, true, TX); + + if (err < 0) + goto fq_probe_failed; + + /* bp init */ + + err = dpa_bp_create(net_dev, dpa_bp, count); + + if (err < 0) + goto bp_create_failed; + + priv->mac_dev = mac_dev; + priv->ethercat_channel = (u16)qman_affine_channel_ethercat(dpa_priv_cpuid); + priv->channel = priv->ethercat_channel; + priv->p = qman_get_affine_portal_ethercat(dpa_priv_cpuid); + + dpa_fq_setup_ethercat(priv, &private_fq_cbs, priv->mac_dev->port_dev[TX]); + + /* Create a congestion group for this netdev, with + * dynamically-allocated CGR ID. + * Must be executed after probing the MAC, but before + * assigning the egress FQs to the CGRs. + */ + err = dpaa_eth_cgr_init(priv); + if (err < 0) { + dev_err(dev, "Error initializing CGR\n"); + goto tx_cgr_init_failed; + } + err = dpaa_eth_priv_ingress_cgr_init(priv); + if (err < 0) { + dev_err(dev, "Error initializing ingress CGR\n"); + goto rx_cgr_init_failed; + } + + /* Add the FQs to the interface, and make them active */ + err = dpa_fqs_init(dev, &priv->dpa_fq_list, false); + if (err < 0) + goto fq_alloc_failed; + + priv->buf_layout = buf_layout; + priv->tx_headroom = dpa_get_headroom(&priv->buf_layout[TX]); + priv->rx_headroom = dpa_get_headroom(&priv->buf_layout[RX]); + + /* All real interfaces need their ports initialized */ + dpaa_eth_init_ports(mac_dev, dpa_bp, count, &port_fqs, + buf_layout, dev); + +#ifdef CONFIG_FMAN_PFC + for (i = 0; i < CONFIG_FMAN_PFC_COS_COUNT; i++) { + err = fm_port_set_pfc_priorities_mapping_to_qman_wq(mac_dev->port_dev[TX], i, i); + if (unlikely(err != 0)) { + dev_err(dev, "Error maping PFC %u to WQ %u\n", i, i); + goto pfc_mapping_failed; + } + } +#endif + + priv->percpu_priv = devm_alloc_percpu(dev, *priv->percpu_priv); + + if (!priv->percpu_priv) { + dev_err(dev, "devm_alloc_percpu() failed\n"); + err = -ENOMEM; + goto alloc_percpu_failed; + } + + for_each_possible_cpu(i) { + percpu_priv = per_cpu_ptr(priv->percpu_priv, i); + memset(percpu_priv, 0, sizeof(*percpu_priv)); + } + + priv->percpu_count = devm_alloc_percpu(dev, *priv->percpu_count); + if (!priv->percpu_count) { + dev_err(dev, "devm_alloc_percpu() failed\n"); + err = -ENOMEM; + goto alloc_percpu_failed; + } + + for_each_possible_cpu(i) { + int *percpu_count = per_cpu_ptr(priv->percpu_count, i); + *percpu_count = 0; + } + + err = dpa_private_netdev_init(net_dev); + + dpa_priv_bp_seed(net_dev); + + if (err < 0) + goto netdev_init_failed; + + pr_info("Ethercat port:%02hx:%02hx:%02hx:%02hx:%02hx:%02hx bpid:%d cpu:%d\n", + mac_dev->addr[0], mac_dev->addr[1], mac_dev->addr[2], + mac_dev->addr[3], mac_dev->addr[4], mac_dev->addr[5], + dpa_priv_bpid, dpa_priv_cpuid); + +#ifdef CONFIG_PM + device_set_wakeup_capable(dev, true); +#endif + + pr_info("fsl_dpa: Probed interface %s\n", net_dev->name); + + return 0; + +netdev_init_failed: + +alloc_percpu_failed: +#ifdef CONFIG_FMAN_PFC +pfc_mapping_failed: +#endif + dpa_fq_free(dev, &priv->dpa_fq_list); +fq_alloc_failed: + qman_delete_cgr_safe(&priv->ingress_cgr); + qman_release_cgrid(priv->ingress_cgr.cgrid); +rx_cgr_init_failed: + qman_delete_cgr_safe(&priv->cgr_data.cgr); + qman_release_cgrid(priv->cgr_data.cgr.cgrid); +tx_cgr_init_failed: + + dpa_bp_free(priv); +bp_create_failed: +fq_probe_failed: +alloc_failed: +mac_probe_failed: + dev_set_drvdata(dev, NULL); + free_netdev(net_dev); +alloc_etherdev_mq_failed: + if (atomic_read(&dpa_bp->refs) == 0) + devm_kfree(dev, dpa_bp); + + return err; +} + +static const struct of_device_id dpa_match[] = { + { + .compatible = "fsl,dpa-ethercat" + }, + {} +}; +MODULE_DEVICE_TABLE(of, dpa_match); + +static struct platform_driver dpa_driver = { + .driver = { + .name = KBUILD_MODNAME "-ethercat", + .of_match_table = dpa_match, + .owner = THIS_MODULE, + .pm = DPAA_PM_OPS, + }, + .probe = dpaa_ethercat_probe, + .remove = dpa_remove +}; + +static int __init __cold dpa_ethercat_load(void) +{ + int _errno; + + pr_info(DPA_DESCRIPTION "\n"); + +#ifdef CONFIG_FSL_DPAA_DBG_LOOP + dpa_debugfs_module_init(); +#endif /* CONFIG_FSL_DPAA_DBG_LOOP */ + + /* initialise dpaa_eth mirror values */ + dpa_rx_extra_headroom = fm_get_rx_extra_headroom(); + dpa_max_frm = fm_get_max_frm(); + dpa_num_cpus = num_possible_cpus(); + +#ifdef CONFIG_FSL_DPAA_DBG_LOOP + memset(dpa_loop_netdevs, 0, sizeof(dpa_loop_netdevs)); +#endif + + _errno = platform_driver_register(&dpa_driver); + if (unlikely(_errno < 0)) { + pr_err(KBUILD_MODNAME + ": %s:%hu:%s(): platform_driver_register() = %d\n", + KBUILD_BASENAME ".c", __LINE__, __func__, _errno); + } + + pr_debug(KBUILD_MODNAME ": %s:%s() ->\n", + KBUILD_BASENAME ".c", __func__); + + return _errno; +} +module_init(dpa_ethercat_load); + +static void __exit __cold dpa_ethercat_unload(void) +{ + pr_debug(KBUILD_MODNAME ": -> %s:%s()\n", + KBUILD_BASENAME ".c", __func__); + + platform_driver_unregister(&dpa_driver); + +#ifdef CONFIG_FSL_DPAA_DBG_LOOP + dpa_debugfs_module_exit(); +#endif /* CONFIG_FSL_DPAA_DBG_LOOP */ + + /* Only one channel is used and needs to be relased after all + * interfaces are removed + */ + dpa_release_channel(); + + pr_debug(KBUILD_MODNAME ": %s:%s() ->\n", + KBUILD_BASENAME ".c", __func__); +} +module_exit(dpa_ethercat_unload); diff --git a/drivers/net/ethernet/fungible/funeth/funeth_txrx.h b/drivers/net/ethernet/fungible/funeth/funeth_txrx.h index 671f51135c26..53b7e95213a8 100644 --- a/drivers/net/ethernet/fungible/funeth/funeth_txrx.h +++ b/drivers/net/ethernet/fungible/funeth/funeth_txrx.h @@ -206,9 +206,9 @@ struct funeth_rxq { #define FUN_QSTAT_READ(q, seq, stats_copy) \ do { \ - seq = u64_stats_fetch_begin_irq(&(q)->syncp); \ + seq = u64_stats_fetch_begin(&(q)->syncp); \ stats_copy = (q)->stats; \ - } while (u64_stats_fetch_retry_irq(&(q)->syncp, (seq))) + } while (u64_stats_fetch_retry(&(q)->syncp, (seq))) #define FUN_INT_NAME_LEN (IFNAMSIZ + 16) diff --git a/drivers/net/ethernet/google/gve/gve_ethtool.c b/drivers/net/ethernet/google/gve/gve_ethtool.c index 033f17cb96be..0a5953089a24 100644 --- a/drivers/net/ethernet/google/gve/gve_ethtool.c +++ b/drivers/net/ethernet/google/gve/gve_ethtool.c @@ -177,14 +177,14 @@ gve_get_ethtool_stats(struct net_device *netdev, struct gve_rx_ring *rx = &priv->rx[ring]; start = - u64_stats_fetch_begin_irq(&priv->rx[ring].statss); + u64_stats_fetch_begin(&priv->rx[ring].statss); tmp_rx_pkts = rx->rpackets; tmp_rx_bytes = rx->rbytes; tmp_rx_skb_alloc_fail = rx->rx_skb_alloc_fail; tmp_rx_buf_alloc_fail = rx->rx_buf_alloc_fail; tmp_rx_desc_err_dropped_pkt = rx->rx_desc_err_dropped_pkt; - } while (u64_stats_fetch_retry_irq(&priv->rx[ring].statss, + } while (u64_stats_fetch_retry(&priv->rx[ring].statss, start)); rx_pkts += tmp_rx_pkts; rx_bytes += tmp_rx_bytes; @@ -198,10 +198,10 @@ gve_get_ethtool_stats(struct net_device *netdev, if (priv->tx) { do { start = - u64_stats_fetch_begin_irq(&priv->tx[ring].statss); + u64_stats_fetch_begin(&priv->tx[ring].statss); tmp_tx_pkts = priv->tx[ring].pkt_done; tmp_tx_bytes = priv->tx[ring].bytes_done; - } while (u64_stats_fetch_retry_irq(&priv->tx[ring].statss, + } while (u64_stats_fetch_retry(&priv->tx[ring].statss, start)); tx_pkts += tmp_tx_pkts; tx_bytes += tmp_tx_bytes; @@ -259,13 +259,13 @@ gve_get_ethtool_stats(struct net_device *netdev, data[i++] = rx->fill_cnt - rx->cnt; do { start = - u64_stats_fetch_begin_irq(&priv->rx[ring].statss); + u64_stats_fetch_begin(&priv->rx[ring].statss); tmp_rx_bytes = rx->rbytes; tmp_rx_skb_alloc_fail = rx->rx_skb_alloc_fail; tmp_rx_buf_alloc_fail = rx->rx_buf_alloc_fail; tmp_rx_desc_err_dropped_pkt = rx->rx_desc_err_dropped_pkt; - } while (u64_stats_fetch_retry_irq(&priv->rx[ring].statss, + } while (u64_stats_fetch_retry(&priv->rx[ring].statss, start)); data[i++] = tmp_rx_bytes; data[i++] = rx->rx_cont_packet_cnt; @@ -331,9 +331,9 @@ gve_get_ethtool_stats(struct net_device *netdev, } do { start = - u64_stats_fetch_begin_irq(&priv->tx[ring].statss); + u64_stats_fetch_begin(&priv->tx[ring].statss); tmp_tx_bytes = tx->bytes_done; - } while (u64_stats_fetch_retry_irq(&priv->tx[ring].statss, + } while (u64_stats_fetch_retry(&priv->tx[ring].statss, start)); data[i++] = tmp_tx_bytes; data[i++] = tx->wake_queue; diff --git a/drivers/net/ethernet/google/gve/gve_main.c b/drivers/net/ethernet/google/gve/gve_main.c index 2e5e0a887270..07b2ab8d054c 100644 --- a/drivers/net/ethernet/google/gve/gve_main.c +++ b/drivers/net/ethernet/google/gve/gve_main.c @@ -51,10 +51,10 @@ static void gve_get_stats(struct net_device *dev, struct rtnl_link_stats64 *s) for (ring = 0; ring < priv->rx_cfg.num_queues; ring++) { do { start = - u64_stats_fetch_begin_irq(&priv->rx[ring].statss); + u64_stats_fetch_begin(&priv->rx[ring].statss); packets = priv->rx[ring].rpackets; bytes = priv->rx[ring].rbytes; - } while (u64_stats_fetch_retry_irq(&priv->rx[ring].statss, + } while (u64_stats_fetch_retry(&priv->rx[ring].statss, start)); s->rx_packets += packets; s->rx_bytes += bytes; @@ -64,10 +64,10 @@ static void gve_get_stats(struct net_device *dev, struct rtnl_link_stats64 *s) for (ring = 0; ring < priv->tx_cfg.num_queues; ring++) { do { start = - u64_stats_fetch_begin_irq(&priv->tx[ring].statss); + u64_stats_fetch_begin(&priv->tx[ring].statss); packets = priv->tx[ring].pkt_done; bytes = priv->tx[ring].bytes_done; - } while (u64_stats_fetch_retry_irq(&priv->tx[ring].statss, + } while (u64_stats_fetch_retry(&priv->tx[ring].statss, start)); s->tx_packets += packets; s->tx_bytes += bytes; @@ -1260,9 +1260,9 @@ void gve_handle_report_stats(struct gve_priv *priv) } do { - start = u64_stats_fetch_begin_irq(&priv->tx[idx].statss); + start = u64_stats_fetch_begin(&priv->tx[idx].statss); tx_bytes = priv->tx[idx].bytes_done; - } while (u64_stats_fetch_retry_irq(&priv->tx[idx].statss, start)); + } while (u64_stats_fetch_retry(&priv->tx[idx].statss, start)); stats[stats_idx++] = (struct stats) { .stat_name = cpu_to_be32(TX_WAKE_CNT), .value = cpu_to_be64(priv->tx[idx].wake_queue), diff --git a/drivers/net/ethernet/hisilicon/hns3/hnae3.h b/drivers/net/ethernet/hisilicon/hns3/hnae3.h index c693bb701ba3..05c682f712ac 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hnae3.h +++ b/drivers/net/ethernet/hisilicon/hns3/hnae3.h @@ -33,6 +33,7 @@ #include #include #include +#include #define HNAE3_MOD_VERSION "1.0" diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c index 8aae179554a8..a0aa13885475 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -2496,7 +2497,7 @@ static void hns3_fetch_stats(struct rtnl_link_stats64 *stats, unsigned int start; do { - start = u64_stats_fetch_begin_irq(&ring->syncp); + start = u64_stats_fetch_begin(&ring->syncp); if (is_tx) { stats->tx_bytes += ring->stats.tx_bytes; stats->tx_packets += ring->stats.tx_pkts; @@ -2530,7 +2531,7 @@ static void hns3_fetch_stats(struct rtnl_link_stats64 *stats, stats->multicast += ring->stats.rx_multicast; stats->rx_length_errors += ring->stats.err_pkt_len; } - } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); + } while (u64_stats_fetch_retry(&ring->syncp, start)); } static void hns3_nic_get_stats64(struct net_device *netdev, diff --git a/drivers/net/ethernet/huawei/hinic/hinic_rx.c b/drivers/net/ethernet/huawei/hinic/hinic_rx.c index d649c6e323c8..ceec8be2a73b 100644 --- a/drivers/net/ethernet/huawei/hinic/hinic_rx.c +++ b/drivers/net/ethernet/huawei/hinic/hinic_rx.c @@ -74,14 +74,14 @@ void hinic_rxq_get_stats(struct hinic_rxq *rxq, struct hinic_rxq_stats *stats) unsigned int start; do { - start = u64_stats_fetch_begin_irq(&rxq_stats->syncp); + start = u64_stats_fetch_begin(&rxq_stats->syncp); stats->pkts = rxq_stats->pkts; stats->bytes = rxq_stats->bytes; stats->errors = rxq_stats->csum_errors + rxq_stats->other_errors; stats->csum_errors = rxq_stats->csum_errors; stats->other_errors = rxq_stats->other_errors; - } while (u64_stats_fetch_retry_irq(&rxq_stats->syncp, start)); + } while (u64_stats_fetch_retry(&rxq_stats->syncp, start)); } /** diff --git a/drivers/net/ethernet/huawei/hinic/hinic_tx.c b/drivers/net/ethernet/huawei/hinic/hinic_tx.c index e91476c8ff8b..ad47ac51a139 100644 --- a/drivers/net/ethernet/huawei/hinic/hinic_tx.c +++ b/drivers/net/ethernet/huawei/hinic/hinic_tx.c @@ -99,14 +99,14 @@ void hinic_txq_get_stats(struct hinic_txq *txq, struct hinic_txq_stats *stats) unsigned int start; do { - start = u64_stats_fetch_begin_irq(&txq_stats->syncp); + start = u64_stats_fetch_begin(&txq_stats->syncp); stats->pkts = txq_stats->pkts; stats->bytes = txq_stats->bytes; stats->tx_busy = txq_stats->tx_busy; stats->tx_wake = txq_stats->tx_wake; stats->tx_dropped = txq_stats->tx_dropped; stats->big_frags_pkts = txq_stats->big_frags_pkts; - } while (u64_stats_fetch_retry_irq(&txq_stats->syncp, start)); + } while (u64_stats_fetch_retry(&txq_stats->syncp, start)); } /** diff --git a/drivers/net/ethernet/intel/fm10k/fm10k_netdev.c b/drivers/net/ethernet/intel/fm10k/fm10k_netdev.c index 2cca9e84e31e..34ab5ff9823b 100644 --- a/drivers/net/ethernet/intel/fm10k/fm10k_netdev.c +++ b/drivers/net/ethernet/intel/fm10k/fm10k_netdev.c @@ -1229,10 +1229,10 @@ static void fm10k_get_stats64(struct net_device *netdev, continue; do { - start = u64_stats_fetch_begin_irq(&ring->syncp); + start = u64_stats_fetch_begin(&ring->syncp); packets = ring->stats.packets; bytes = ring->stats.bytes; - } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); + } while (u64_stats_fetch_retry(&ring->syncp, start)); stats->rx_packets += packets; stats->rx_bytes += bytes; @@ -1245,10 +1245,10 @@ static void fm10k_get_stats64(struct net_device *netdev, continue; do { - start = u64_stats_fetch_begin_irq(&ring->syncp); + start = u64_stats_fetch_begin(&ring->syncp); packets = ring->stats.packets; bytes = ring->stats.bytes; - } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); + } while (u64_stats_fetch_retry(&ring->syncp, start)); stats->tx_packets += packets; stats->tx_bytes += bytes; diff --git a/drivers/net/ethernet/intel/i40e/i40e.h b/drivers/net/ethernet/intel/i40e/i40e.h index a81f918091cc..90d29a297e19 100644 --- a/drivers/net/ethernet/intel/i40e/i40e.h +++ b/drivers/net/ethernet/intel/i40e/i40e.h @@ -33,6 +33,7 @@ #include #include #include +#include #include #include #include diff --git a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c index e632041aed5f..995b2a7c8973 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c +++ b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c @@ -154,7 +154,7 @@ __i40e_add_ethtool_stats(u64 **data, void *pointer, * @ring: the ring to copy * * Queue statistics must be copied while protected by - * u64_stats_fetch_begin_irq, so we can't directly use i40e_add_ethtool_stats. + * u64_stats_fetch_begin, so we can't directly use i40e_add_ethtool_stats. * Assumes that queue stats are defined in i40e_gstrings_queue_stats. If the * ring pointer is null, zero out the queue stat values and update the data * pointer. Otherwise safely copy the stats from the ring into the supplied @@ -172,16 +172,16 @@ i40e_add_queue_stats(u64 **data, struct i40e_ring *ring) /* To avoid invalid statistics values, ensure that we keep retrying * the copy until we get a consistent value according to - * u64_stats_fetch_retry_irq. But first, make sure our ring is + * u64_stats_fetch_retry. But first, make sure our ring is * non-null before attempting to access its syncp. */ do { - start = !ring ? 0 : u64_stats_fetch_begin_irq(&ring->syncp); + start = !ring ? 0 : u64_stats_fetch_begin(&ring->syncp); for (i = 0; i < size; i++) { i40e_add_one_ethtool_stat(&(*data)[i], ring, &stats[i]); } - } while (ring && u64_stats_fetch_retry_irq(&ring->syncp, start)); + } while (ring && u64_stats_fetch_retry(&ring->syncp, start)); /* Once we successfully copy the stats in, update the data pointer */ *data += size; diff --git a/drivers/net/ethernet/intel/i40e/i40e_main.c b/drivers/net/ethernet/intel/i40e/i40e_main.c index 08ccf0024ce1..6b3f3727c3f1 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_main.c +++ b/drivers/net/ethernet/intel/i40e/i40e_main.c @@ -419,10 +419,10 @@ static void i40e_get_netdev_stats_struct_tx(struct i40e_ring *ring, unsigned int start; do { - start = u64_stats_fetch_begin_irq(&ring->syncp); + start = u64_stats_fetch_begin(&ring->syncp); packets = ring->stats.packets; bytes = ring->stats.bytes; - } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); + } while (u64_stats_fetch_retry(&ring->syncp, start)); stats->tx_packets += packets; stats->tx_bytes += bytes; @@ -472,10 +472,10 @@ static void i40e_get_netdev_stats_struct(struct net_device *netdev, if (!ring) continue; do { - start = u64_stats_fetch_begin_irq(&ring->syncp); + start = u64_stats_fetch_begin(&ring->syncp); packets = ring->stats.packets; bytes = ring->stats.bytes; - } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); + } while (u64_stats_fetch_retry(&ring->syncp, start)); stats->rx_packets += packets; stats->rx_bytes += bytes; @@ -897,10 +897,10 @@ static void i40e_update_vsi_stats(struct i40e_vsi *vsi) continue; do { - start = u64_stats_fetch_begin_irq(&p->syncp); + start = u64_stats_fetch_begin(&p->syncp); packets = p->stats.packets; bytes = p->stats.bytes; - } while (u64_stats_fetch_retry_irq(&p->syncp, start)); + } while (u64_stats_fetch_retry(&p->syncp, start)); tx_b += bytes; tx_p += packets; tx_restart += p->tx_stats.restart_queue; @@ -915,10 +915,10 @@ static void i40e_update_vsi_stats(struct i40e_vsi *vsi) continue; do { - start = u64_stats_fetch_begin_irq(&p->syncp); + start = u64_stats_fetch_begin(&p->syncp); packets = p->stats.packets; bytes = p->stats.bytes; - } while (u64_stats_fetch_retry_irq(&p->syncp, start)); + } while (u64_stats_fetch_retry(&p->syncp, start)); rx_b += bytes; rx_p += packets; rx_buf += p->rx_stats.alloc_buff_failed; @@ -935,10 +935,10 @@ static void i40e_update_vsi_stats(struct i40e_vsi *vsi) continue; do { - start = u64_stats_fetch_begin_irq(&p->syncp); + start = u64_stats_fetch_begin(&p->syncp); packets = p->stats.packets; bytes = p->stats.bytes; - } while (u64_stats_fetch_retry_irq(&p->syncp, start)); + } while (u64_stats_fetch_retry(&p->syncp, start)); tx_b += bytes; tx_p += packets; tx_restart += p->tx_stats.restart_queue; diff --git a/drivers/net/ethernet/intel/iavf/iavf.h b/drivers/net/ethernet/intel/iavf/iavf.h index 543931c06bb1..4e36fbc9d9e8 100644 --- a/drivers/net/ethernet/intel/iavf/iavf.h +++ b/drivers/net/ethernet/intel/iavf/iavf.h @@ -30,6 +30,7 @@ #include #include #include +#include #include #include #include diff --git a/drivers/net/ethernet/intel/iavf/iavf_ethtool.c b/drivers/net/ethernet/intel/iavf/iavf_ethtool.c index fe912b1c468e..a34303ad057d 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_ethtool.c +++ b/drivers/net/ethernet/intel/iavf/iavf_ethtool.c @@ -147,7 +147,7 @@ __iavf_add_ethtool_stats(u64 **data, void *pointer, * @ring: the ring to copy * * Queue statistics must be copied while protected by - * u64_stats_fetch_begin_irq, so we can't directly use iavf_add_ethtool_stats. + * u64_stats_fetch_begin, so we can't directly use iavf_add_ethtool_stats. * Assumes that queue stats are defined in iavf_gstrings_queue_stats. If the * ring pointer is null, zero out the queue stat values and update the data * pointer. Otherwise safely copy the stats from the ring into the supplied @@ -165,14 +165,14 @@ iavf_add_queue_stats(u64 **data, struct iavf_ring *ring) /* To avoid invalid statistics values, ensure that we keep retrying * the copy until we get a consistent value according to - * u64_stats_fetch_retry_irq. But first, make sure our ring is + * u64_stats_fetch_retry. But first, make sure our ring is * non-null before attempting to access its syncp. */ do { - start = !ring ? 0 : u64_stats_fetch_begin_irq(&ring->syncp); + start = !ring ? 0 : u64_stats_fetch_begin(&ring->syncp); for (i = 0; i < size; i++) iavf_add_one_ethtool_stat(&(*data)[i], ring, &stats[i]); - } while (ring && u64_stats_fetch_retry_irq(&ring->syncp, start)); + } while (ring && u64_stats_fetch_retry(&ring->syncp, start)); /* Once we successfully copy the stats in, update the data pointer */ *data += size; diff --git a/drivers/net/ethernet/intel/ice/ice.h b/drivers/net/ethernet/intel/ice/ice.h index f2be383d97df..f7ea8b0439fd 100644 --- a/drivers/net/ethernet/intel/ice/ice.h +++ b/drivers/net/ethernet/intel/ice/ice.h @@ -40,6 +40,7 @@ #include #include #include +#include #include #include #include diff --git a/drivers/net/ethernet/intel/ice/ice_main.c b/drivers/net/ethernet/intel/ice/ice_main.c index 3f98781e74b2..a7d9cc1292ec 100644 --- a/drivers/net/ethernet/intel/ice/ice_main.c +++ b/drivers/net/ethernet/intel/ice/ice_main.c @@ -6407,10 +6407,10 @@ ice_fetch_u64_stats_per_ring(struct u64_stats_sync *syncp, unsigned int start; do { - start = u64_stats_fetch_begin_irq(syncp); + start = u64_stats_fetch_begin(syncp); *pkts = stats.pkts; *bytes = stats.bytes; - } while (u64_stats_fetch_retry_irq(syncp, start)); + } while (u64_stats_fetch_retry(syncp, start)); } /** diff --git a/drivers/net/ethernet/intel/igb/igb_ethtool.c b/drivers/net/ethernet/intel/igb/igb_ethtool.c index 96fa1c420f91..319ed601eaa1 100644 --- a/drivers/net/ethernet/intel/igb/igb_ethtool.c +++ b/drivers/net/ethernet/intel/igb/igb_ethtool.c @@ -2316,15 +2316,15 @@ static void igb_get_ethtool_stats(struct net_device *netdev, ring = adapter->tx_ring[j]; do { - start = u64_stats_fetch_begin_irq(&ring->tx_syncp); + start = u64_stats_fetch_begin(&ring->tx_syncp); data[i] = ring->tx_stats.packets; data[i+1] = ring->tx_stats.bytes; data[i+2] = ring->tx_stats.restart_queue; - } while (u64_stats_fetch_retry_irq(&ring->tx_syncp, start)); + } while (u64_stats_fetch_retry(&ring->tx_syncp, start)); do { - start = u64_stats_fetch_begin_irq(&ring->tx_syncp2); + start = u64_stats_fetch_begin(&ring->tx_syncp2); restart2 = ring->tx_stats.restart_queue2; - } while (u64_stats_fetch_retry_irq(&ring->tx_syncp2, start)); + } while (u64_stats_fetch_retry(&ring->tx_syncp2, start)); data[i+2] += restart2; i += IGB_TX_QUEUE_STATS_LEN; @@ -2332,13 +2332,13 @@ static void igb_get_ethtool_stats(struct net_device *netdev, for (j = 0; j < adapter->num_rx_queues; j++) { ring = adapter->rx_ring[j]; do { - start = u64_stats_fetch_begin_irq(&ring->rx_syncp); + start = u64_stats_fetch_begin(&ring->rx_syncp); data[i] = ring->rx_stats.packets; data[i+1] = ring->rx_stats.bytes; data[i+2] = ring->rx_stats.drops; data[i+3] = ring->rx_stats.csum_err; data[i+4] = ring->rx_stats.alloc_failed; - } while (u64_stats_fetch_retry_irq(&ring->rx_syncp, start)); + } while (u64_stats_fetch_retry(&ring->rx_syncp, start)); i += IGB_RX_QUEUE_STATS_LEN; } spin_unlock(&adapter->stats64_lock); diff --git a/drivers/net/ethernet/intel/igb/igb_main.c b/drivers/net/ethernet/intel/igb/igb_main.c index 45ce4ed16146..9824f7cfaca4 100644 --- a/drivers/net/ethernet/intel/igb/igb_main.c +++ b/drivers/net/ethernet/intel/igb/igb_main.c @@ -6660,10 +6660,10 @@ void igb_update_stats(struct igb_adapter *adapter) } do { - start = u64_stats_fetch_begin_irq(&ring->rx_syncp); + start = u64_stats_fetch_begin(&ring->rx_syncp); _bytes = ring->rx_stats.bytes; _packets = ring->rx_stats.packets; - } while (u64_stats_fetch_retry_irq(&ring->rx_syncp, start)); + } while (u64_stats_fetch_retry(&ring->rx_syncp, start)); bytes += _bytes; packets += _packets; } @@ -6676,10 +6676,10 @@ void igb_update_stats(struct igb_adapter *adapter) for (i = 0; i < adapter->num_tx_queues; i++) { struct igb_ring *ring = adapter->tx_ring[i]; do { - start = u64_stats_fetch_begin_irq(&ring->tx_syncp); + start = u64_stats_fetch_begin(&ring->tx_syncp); _bytes = ring->tx_stats.bytes; _packets = ring->tx_stats.packets; - } while (u64_stats_fetch_retry_irq(&ring->tx_syncp, start)); + } while (u64_stats_fetch_retry(&ring->tx_syncp, start)); bytes += _bytes; packets += _packets; } diff --git a/drivers/net/ethernet/intel/igc/igc_ethtool.c b/drivers/net/ethernet/intel/igc/igc_ethtool.c index 511fc3f41208..2899109e6fb5 100644 --- a/drivers/net/ethernet/intel/igc/igc_ethtool.c +++ b/drivers/net/ethernet/intel/igc/igc_ethtool.c @@ -839,15 +839,15 @@ static void igc_ethtool_get_stats(struct net_device *netdev, ring = adapter->tx_ring[j]; do { - start = u64_stats_fetch_begin_irq(&ring->tx_syncp); + start = u64_stats_fetch_begin(&ring->tx_syncp); data[i] = ring->tx_stats.packets; data[i + 1] = ring->tx_stats.bytes; data[i + 2] = ring->tx_stats.restart_queue; - } while (u64_stats_fetch_retry_irq(&ring->tx_syncp, start)); + } while (u64_stats_fetch_retry(&ring->tx_syncp, start)); do { - start = u64_stats_fetch_begin_irq(&ring->tx_syncp2); + start = u64_stats_fetch_begin(&ring->tx_syncp2); restart2 = ring->tx_stats.restart_queue2; - } while (u64_stats_fetch_retry_irq(&ring->tx_syncp2, start)); + } while (u64_stats_fetch_retry(&ring->tx_syncp2, start)); data[i + 2] += restart2; i += IGC_TX_QUEUE_STATS_LEN; @@ -855,13 +855,13 @@ static void igc_ethtool_get_stats(struct net_device *netdev, for (j = 0; j < adapter->num_rx_queues; j++) { ring = adapter->rx_ring[j]; do { - start = u64_stats_fetch_begin_irq(&ring->rx_syncp); + start = u64_stats_fetch_begin(&ring->rx_syncp); data[i] = ring->rx_stats.packets; data[i + 1] = ring->rx_stats.bytes; data[i + 2] = ring->rx_stats.drops; data[i + 3] = ring->rx_stats.csum_err; data[i + 4] = ring->rx_stats.alloc_failed; - } while (u64_stats_fetch_retry_irq(&ring->rx_syncp, start)); + } while (u64_stats_fetch_retry(&ring->rx_syncp, start)); i += IGC_RX_QUEUE_STATS_LEN; } spin_unlock(&adapter->stats64_lock); diff --git a/drivers/net/ethernet/intel/igc/igc_main.c b/drivers/net/ethernet/intel/igc/igc_main.c index 2f3947cf513b..125669a6f6bb 100644 --- a/drivers/net/ethernet/intel/igc/igc_main.c +++ b/drivers/net/ethernet/intel/igc/igc_main.c @@ -4865,10 +4865,10 @@ void igc_update_stats(struct igc_adapter *adapter) } do { - start = u64_stats_fetch_begin_irq(&ring->rx_syncp); + start = u64_stats_fetch_begin(&ring->rx_syncp); _bytes = ring->rx_stats.bytes; _packets = ring->rx_stats.packets; - } while (u64_stats_fetch_retry_irq(&ring->rx_syncp, start)); + } while (u64_stats_fetch_retry(&ring->rx_syncp, start)); bytes += _bytes; packets += _packets; } @@ -4882,10 +4882,10 @@ void igc_update_stats(struct igc_adapter *adapter) struct igc_ring *ring = adapter->tx_ring[i]; do { - start = u64_stats_fetch_begin_irq(&ring->tx_syncp); + start = u64_stats_fetch_begin(&ring->tx_syncp); _bytes = ring->tx_stats.bytes; _packets = ring->tx_stats.packets; - } while (u64_stats_fetch_retry_irq(&ring->tx_syncp, start)); + } while (u64_stats_fetch_retry(&ring->tx_syncp, start)); bytes += _bytes; packets += _packets; } diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c index 0051aa676e19..1c22ff2dba9b 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c @@ -1335,10 +1335,10 @@ static void ixgbe_get_ethtool_stats(struct net_device *netdev, } do { - start = u64_stats_fetch_begin_irq(&ring->syncp); + start = u64_stats_fetch_begin(&ring->syncp); data[i] = ring->stats.packets; data[i+1] = ring->stats.bytes; - } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); + } while (u64_stats_fetch_retry(&ring->syncp, start)); i += 2; } for (j = 0; j < IXGBE_NUM_RX_QUEUES; j++) { @@ -1351,10 +1351,10 @@ static void ixgbe_get_ethtool_stats(struct net_device *netdev, } do { - start = u64_stats_fetch_begin_irq(&ring->syncp); + start = u64_stats_fetch_begin(&ring->syncp); data[i] = ring->stats.packets; data[i+1] = ring->stats.bytes; - } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); + } while (u64_stats_fetch_retry(&ring->syncp, start)); i += 2; } diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c index 6105419ae2d5..ec8864beb1ac 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c @@ -9051,10 +9051,10 @@ static void ixgbe_get_ring_stats64(struct rtnl_link_stats64 *stats, if (ring) { do { - start = u64_stats_fetch_begin_irq(&ring->syncp); + start = u64_stats_fetch_begin(&ring->syncp); packets = ring->stats.packets; bytes = ring->stats.bytes; - } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); + } while (u64_stats_fetch_retry(&ring->syncp, start)); stats->tx_packets += packets; stats->tx_bytes += bytes; } @@ -9074,10 +9074,10 @@ static void ixgbe_get_stats64(struct net_device *netdev, if (ring) { do { - start = u64_stats_fetch_begin_irq(&ring->syncp); + start = u64_stats_fetch_begin(&ring->syncp); packets = ring->stats.packets; bytes = ring->stats.bytes; - } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); + } while (u64_stats_fetch_retry(&ring->syncp, start)); stats->rx_packets += packets; stats->rx_bytes += bytes; } diff --git a/drivers/net/ethernet/intel/ixgbevf/ethtool.c b/drivers/net/ethernet/intel/ixgbevf/ethtool.c index ccfa6b91aac6..296915414a7c 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ethtool.c +++ b/drivers/net/ethernet/intel/ixgbevf/ethtool.c @@ -458,10 +458,10 @@ static void ixgbevf_get_ethtool_stats(struct net_device *netdev, } do { - start = u64_stats_fetch_begin_irq(&ring->syncp); + start = u64_stats_fetch_begin(&ring->syncp); data[i] = ring->stats.packets; data[i + 1] = ring->stats.bytes; - } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); + } while (u64_stats_fetch_retry(&ring->syncp, start)); i += 2; } @@ -475,10 +475,10 @@ static void ixgbevf_get_ethtool_stats(struct net_device *netdev, } do { - start = u64_stats_fetch_begin_irq(&ring->syncp); + start = u64_stats_fetch_begin(&ring->syncp); data[i] = ring->stats.packets; data[i + 1] = ring->stats.bytes; - } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); + } while (u64_stats_fetch_retry(&ring->syncp, start)); i += 2; } @@ -492,10 +492,10 @@ static void ixgbevf_get_ethtool_stats(struct net_device *netdev, } do { - start = u64_stats_fetch_begin_irq(&ring->syncp); + start = u64_stats_fetch_begin(&ring->syncp); data[i] = ring->stats.packets; data[i + 1] = ring->stats.bytes; - } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); + } while (u64_stats_fetch_retry(&ring->syncp, start)); i += 2; } } diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c index e338fa572793..a9479ddf68eb 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c @@ -4350,10 +4350,10 @@ static void ixgbevf_get_tx_ring_stats(struct rtnl_link_stats64 *stats, if (ring) { do { - start = u64_stats_fetch_begin_irq(&ring->syncp); + start = u64_stats_fetch_begin(&ring->syncp); bytes = ring->stats.bytes; packets = ring->stats.packets; - } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); + } while (u64_stats_fetch_retry(&ring->syncp, start)); stats->tx_bytes += bytes; stats->tx_packets += packets; } @@ -4376,10 +4376,10 @@ static void ixgbevf_get_stats(struct net_device *netdev, for (i = 0; i < adapter->num_rx_queues; i++) { ring = adapter->rx_ring[i]; do { - start = u64_stats_fetch_begin_irq(&ring->syncp); + start = u64_stats_fetch_begin(&ring->syncp); bytes = ring->stats.bytes; packets = ring->stats.packets; - } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); + } while (u64_stats_fetch_retry(&ring->syncp, start)); stats->rx_bytes += bytes; stats->rx_packets += packets; } diff --git a/drivers/net/ethernet/marvell/mvneta.c b/drivers/net/ethernet/marvell/mvneta.c index aca5b72cfeec..b9f742518313 100644 --- a/drivers/net/ethernet/marvell/mvneta.c +++ b/drivers/net/ethernet/marvell/mvneta.c @@ -38,7 +38,7 @@ #include #include #include -#include +#include #include /* Registers */ @@ -813,14 +813,14 @@ mvneta_get_stats64(struct net_device *dev, cpu_stats = per_cpu_ptr(pp->stats, cpu); do { - start = u64_stats_fetch_begin_irq(&cpu_stats->syncp); + start = u64_stats_fetch_begin(&cpu_stats->syncp); rx_packets = cpu_stats->es.ps.rx_packets; rx_bytes = cpu_stats->es.ps.rx_bytes; rx_dropped = cpu_stats->rx_dropped; rx_errors = cpu_stats->rx_errors; tx_packets = cpu_stats->es.ps.tx_packets; tx_bytes = cpu_stats->es.ps.tx_bytes; - } while (u64_stats_fetch_retry_irq(&cpu_stats->syncp, start)); + } while (u64_stats_fetch_retry(&cpu_stats->syncp, start)); stats->rx_packets += rx_packets; stats->rx_bytes += rx_bytes; @@ -4762,7 +4762,7 @@ mvneta_ethtool_update_pcpu_stats(struct mvneta_port *pp, stats = per_cpu_ptr(pp->stats, cpu); do { - start = u64_stats_fetch_begin_irq(&stats->syncp); + start = u64_stats_fetch_begin(&stats->syncp); skb_alloc_error = stats->es.skb_alloc_error; refill_error = stats->es.refill_error; xdp_redirect = stats->es.ps.xdp_redirect; @@ -4772,7 +4772,7 @@ mvneta_ethtool_update_pcpu_stats(struct mvneta_port *pp, xdp_xmit_err = stats->es.ps.xdp_xmit_err; xdp_tx = stats->es.ps.xdp_tx; xdp_tx_err = stats->es.ps.xdp_tx_err; - } while (u64_stats_fetch_retry_irq(&stats->syncp, start)); + } while (u64_stats_fetch_retry(&stats->syncp, start)); es->skb_alloc_error += skb_alloc_error; es->refill_error += refill_error; diff --git a/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c b/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c index f936640cca4e..8c7470ab4985 100644 --- a/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c +++ b/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c @@ -2008,7 +2008,7 @@ mvpp2_get_xdp_stats(struct mvpp2_port *port, struct mvpp2_pcpu_stats *xdp_stats) cpu_stats = per_cpu_ptr(port->stats, cpu); do { - start = u64_stats_fetch_begin_irq(&cpu_stats->syncp); + start = u64_stats_fetch_begin(&cpu_stats->syncp); xdp_redirect = cpu_stats->xdp_redirect; xdp_pass = cpu_stats->xdp_pass; xdp_drop = cpu_stats->xdp_drop; @@ -2016,7 +2016,7 @@ mvpp2_get_xdp_stats(struct mvpp2_port *port, struct mvpp2_pcpu_stats *xdp_stats) xdp_xmit_err = cpu_stats->xdp_xmit_err; xdp_tx = cpu_stats->xdp_tx; xdp_tx_err = cpu_stats->xdp_tx_err; - } while (u64_stats_fetch_retry_irq(&cpu_stats->syncp, start)); + } while (u64_stats_fetch_retry(&cpu_stats->syncp, start)); xdp_stats->xdp_redirect += xdp_redirect; xdp_stats->xdp_pass += xdp_pass; @@ -5115,12 +5115,12 @@ mvpp2_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) cpu_stats = per_cpu_ptr(port->stats, cpu); do { - start = u64_stats_fetch_begin_irq(&cpu_stats->syncp); + start = u64_stats_fetch_begin(&cpu_stats->syncp); rx_packets = cpu_stats->rx_packets; rx_bytes = cpu_stats->rx_bytes; tx_packets = cpu_stats->tx_packets; tx_bytes = cpu_stats->tx_bytes; - } while (u64_stats_fetch_retry_irq(&cpu_stats->syncp, start)); + } while (u64_stats_fetch_retry(&cpu_stats->syncp, start)); stats->rx_packets += rx_packets; stats->rx_bytes += rx_bytes; diff --git a/drivers/net/ethernet/marvell/sky2.c b/drivers/net/ethernet/marvell/sky2.c index ab33ba1c3023..ff97b140886a 100644 --- a/drivers/net/ethernet/marvell/sky2.c +++ b/drivers/net/ethernet/marvell/sky2.c @@ -3894,19 +3894,19 @@ static void sky2_get_stats(struct net_device *dev, u64 _bytes, _packets; do { - start = u64_stats_fetch_begin_irq(&sky2->rx_stats.syncp); + start = u64_stats_fetch_begin(&sky2->rx_stats.syncp); _bytes = sky2->rx_stats.bytes; _packets = sky2->rx_stats.packets; - } while (u64_stats_fetch_retry_irq(&sky2->rx_stats.syncp, start)); + } while (u64_stats_fetch_retry(&sky2->rx_stats.syncp, start)); stats->rx_packets = _packets; stats->rx_bytes = _bytes; do { - start = u64_stats_fetch_begin_irq(&sky2->tx_stats.syncp); + start = u64_stats_fetch_begin(&sky2->tx_stats.syncp); _bytes = sky2->tx_stats.bytes; _packets = sky2->tx_stats.packets; - } while (u64_stats_fetch_retry_irq(&sky2->tx_stats.syncp, start)); + } while (u64_stats_fetch_retry(&sky2->tx_stats.syncp, start)); stats->tx_packets = _packets; stats->tx_bytes = _bytes; diff --git a/drivers/net/ethernet/mediatek/mtk_eth_soc.c b/drivers/net/ethernet/mediatek/mtk_eth_soc.c index 0ac5ae16308f..c13cc420c64b 100644 --- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c +++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c @@ -866,7 +866,7 @@ static void mtk_get_stats64(struct net_device *dev, } do { - start = u64_stats_fetch_begin_irq(&hw_stats->syncp); + start = u64_stats_fetch_begin(&hw_stats->syncp); storage->rx_packets = hw_stats->rx_packets; storage->tx_packets = hw_stats->tx_packets; storage->rx_bytes = hw_stats->rx_bytes; @@ -878,7 +878,7 @@ static void mtk_get_stats64(struct net_device *dev, storage->rx_crc_errors = hw_stats->rx_fcs_errors; storage->rx_errors = hw_stats->rx_checksum_errors; storage->tx_aborted_errors = hw_stats->tx_skip; - } while (u64_stats_fetch_retry_irq(&hw_stats->syncp, start)); + } while (u64_stats_fetch_retry(&hw_stats->syncp, start)); storage->tx_errors = dev->stats.tx_errors; storage->rx_dropped = dev->stats.rx_dropped; @@ -3694,13 +3694,13 @@ static void mtk_get_ethtool_stats(struct net_device *dev, do { data_dst = data; - start = u64_stats_fetch_begin_irq(&hwstats->syncp); + start = u64_stats_fetch_begin(&hwstats->syncp); for (i = 0; i < ARRAY_SIZE(mtk_ethtool_stats); i++) *data_dst++ = *(data_src + mtk_ethtool_stats[i].offset); if (mtk_page_pool_enabled(mac->hw)) mtk_ethtool_pp_stats(mac->hw, data_dst); - } while (u64_stats_fetch_retry_irq(&hwstats->syncp, start)); + } while (u64_stats_fetch_retry(&hwstats->syncp, start)); } static int mtk_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *cmd, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index 4e7daa382bc0..699313b59330 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -38,6 +38,7 @@ #include #include #include +#include #include #include "eswitch.h" #include "en.h" diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c index 67ecdb9e708f..8345499563a4 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c @@ -827,12 +827,12 @@ mlxsw_sp_port_get_sw_stats64(const struct net_device *dev, for_each_possible_cpu(i) { p = per_cpu_ptr(mlxsw_sp_port->pcpu_stats, i); do { - start = u64_stats_fetch_begin_irq(&p->syncp); + start = u64_stats_fetch_begin(&p->syncp); rx_packets = p->rx_packets; rx_bytes = p->rx_bytes; tx_packets = p->tx_packets; tx_bytes = p->tx_bytes; - } while (u64_stats_fetch_retry_irq(&p->syncp, start)); + } while (u64_stats_fetch_retry(&p->syncp, start)); stats->rx_packets += rx_packets; stats->rx_bytes += rx_bytes; diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_tc.c b/drivers/net/ethernet/microchip/lan966x/lan966x_tc.c index 651d5493ae55..ee78655a1cd5 100644 --- a/drivers/net/ethernet/microchip/lan966x/lan966x_tc.c +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_tc.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0+ #include +#include #include "lan966x_main.h" diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_tc.c b/drivers/net/ethernet/microchip/sparx5/sparx5_tc.c index dc2c3756e3a2..7c174cb4a8b7 100644 --- a/drivers/net/ethernet/microchip/sparx5/sparx5_tc.c +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_tc.c @@ -5,6 +5,7 @@ */ #include +#include #include "sparx5_tc.h" #include "sparx5_main.h" diff --git a/drivers/net/ethernet/microsoft/mana/mana_en.c b/drivers/net/ethernet/microsoft/mana/mana_en.c index 4f4204432aaa..9cca75423933 100644 --- a/drivers/net/ethernet/microsoft/mana/mana_en.c +++ b/drivers/net/ethernet/microsoft/mana/mana_en.c @@ -316,10 +316,10 @@ static void mana_get_stats64(struct net_device *ndev, rx_stats = &apc->rxqs[q]->stats; do { - start = u64_stats_fetch_begin_irq(&rx_stats->syncp); + start = u64_stats_fetch_begin(&rx_stats->syncp); packets = rx_stats->packets; bytes = rx_stats->bytes; - } while (u64_stats_fetch_retry_irq(&rx_stats->syncp, start)); + } while (u64_stats_fetch_retry(&rx_stats->syncp, start)); st->rx_packets += packets; st->rx_bytes += bytes; @@ -329,10 +329,10 @@ static void mana_get_stats64(struct net_device *ndev, tx_stats = &apc->tx_qp[q].txq.stats; do { - start = u64_stats_fetch_begin_irq(&tx_stats->syncp); + start = u64_stats_fetch_begin(&tx_stats->syncp); packets = tx_stats->packets; bytes = tx_stats->bytes; - } while (u64_stats_fetch_retry_irq(&tx_stats->syncp, start)); + } while (u64_stats_fetch_retry(&tx_stats->syncp, start)); st->tx_packets += packets; st->tx_bytes += bytes; diff --git a/drivers/net/ethernet/microsoft/mana/mana_ethtool.c b/drivers/net/ethernet/microsoft/mana/mana_ethtool.c index c530db76880f..96d55c91c969 100644 --- a/drivers/net/ethernet/microsoft/mana/mana_ethtool.c +++ b/drivers/net/ethernet/microsoft/mana/mana_ethtool.c @@ -90,13 +90,13 @@ static void mana_get_ethtool_stats(struct net_device *ndev, rx_stats = &apc->rxqs[q]->stats; do { - start = u64_stats_fetch_begin_irq(&rx_stats->syncp); + start = u64_stats_fetch_begin(&rx_stats->syncp); packets = rx_stats->packets; bytes = rx_stats->bytes; xdp_drop = rx_stats->xdp_drop; xdp_tx = rx_stats->xdp_tx; xdp_redirect = rx_stats->xdp_redirect; - } while (u64_stats_fetch_retry_irq(&rx_stats->syncp, start)); + } while (u64_stats_fetch_retry(&rx_stats->syncp, start)); data[i++] = packets; data[i++] = bytes; @@ -109,11 +109,11 @@ static void mana_get_ethtool_stats(struct net_device *ndev, tx_stats = &apc->tx_qp[q].txq.stats; do { - start = u64_stats_fetch_begin_irq(&tx_stats->syncp); + start = u64_stats_fetch_begin(&tx_stats->syncp); packets = tx_stats->packets; bytes = tx_stats->bytes; xdp_xmit = tx_stats->xdp_xmit; - } while (u64_stats_fetch_retry_irq(&tx_stats->syncp, start)); + } while (u64_stats_fetch_retry(&tx_stats->syncp, start)); data[i++] = packets; data[i++] = bytes; diff --git a/drivers/net/ethernet/mscc/ocelot.c b/drivers/net/ethernet/mscc/ocelot.c index aa069aa433f0..6897441e0eaf 100644 --- a/drivers/net/ethernet/mscc/ocelot.c +++ b/drivers/net/ethernet/mscc/ocelot.c @@ -212,6 +212,12 @@ static void ocelot_vcap_enable(struct ocelot *ocelot, int port) ocelot_write_gix(ocelot, ANA_PORT_VCAP_CFG_S1_ENA, ANA_PORT_VCAP_CFG, port); + /* Use key S1_5TUPLE_IP4 in second lookup. */ + ocelot_write_ix(ocelot, + ANA_PORT_VCAP_S1_KEY_CFG_S1_KEY_IP6_CFG(2) | + ANA_PORT_VCAP_S1_KEY_CFG_S1_KEY_IP4_CFG(2), + ANA_PORT_VCAP_S1_KEY_CFG, port, 1); + ocelot_rmw_gix(ocelot, REW_PORT_CFG_ES0_EN, REW_PORT_CFG_ES0_EN, REW_PORT_CFG, port); @@ -345,6 +351,8 @@ static void ocelot_port_manage_port_tag(struct ocelot *ocelot, int port) struct ocelot_port *ocelot_port = ocelot->ports[port]; enum ocelot_port_tag_config tag_cfg; bool uses_native_vlan = false; + u32 port_tpid = 0; + u32 tag_tpid = 0; if (ocelot_port->vlan_aware) { uses_native_vlan = ocelot_port_uses_native_vlan(ocelot, port); @@ -355,12 +363,17 @@ static void ocelot_port_manage_port_tag(struct ocelot *ocelot, int port) tag_cfg = OCELOT_PORT_TAG_DISABLED; else tag_cfg = OCELOT_PORT_TAG_TRUNK; + + if (ocelot->qinq_enable && ocelot_port->qinq_mode) + tag_tpid = REW_TAG_CFG_TAG_TPID_CFG(1); + else + tag_tpid = REW_TAG_CFG_TAG_TPID_CFG(0); } else { tag_cfg = OCELOT_PORT_TAG_DISABLED; } - ocelot_rmw_gix(ocelot, REW_TAG_CFG_TAG_CFG(tag_cfg), - REW_TAG_CFG_TAG_CFG_M, + ocelot_rmw_gix(ocelot, REW_TAG_CFG_TAG_CFG(tag_cfg) | tag_tpid, + REW_TAG_CFG_TAG_CFG_M | REW_TAG_CFG_TAG_TPID_CFG_M, REW_TAG_CFG, port); if (uses_native_vlan) { @@ -372,9 +385,16 @@ static void ocelot_port_manage_port_tag(struct ocelot *ocelot, int port) */ native_vlan = ocelot_port_find_native_vlan(ocelot, port); + if (ocelot->qinq_enable && ocelot_port->qinq_mode) + port_tpid = REW_PORT_VLAN_CFG_PORT_TPID(ETH_P_8021AD); + else + port_tpid = REW_PORT_VLAN_CFG_PORT_TPID(ETH_P_8021Q); + ocelot_rmw_gix(ocelot, - REW_PORT_VLAN_CFG_PORT_VID(native_vlan->vid), - REW_PORT_VLAN_CFG_PORT_VID_M, + REW_PORT_VLAN_CFG_PORT_VID(native_vlan->vid) | + port_tpid, + REW_PORT_VLAN_CFG_PORT_VID_M | + REW_PORT_VLAN_CFG_PORT_TPID_M, REW_PORT_VLAN_CFG, port); } } @@ -419,6 +439,10 @@ static void ocelot_port_set_pvid(struct ocelot *ocelot, int port, struct ocelot_port *ocelot_port = ocelot->ports[port]; u16 pvid = ocelot_vlan_unaware_pvid(ocelot, ocelot_port->bridge); u32 val = 0; + u32 tag_type = 0; + + if (ocelot->qinq_enable && ocelot_port->qinq_mode) + tag_type = ANA_PORT_VLAN_CFG_VLAN_TAG_TYPE; ocelot_port->pvid_vlan = pvid_vlan; @@ -426,8 +450,8 @@ static void ocelot_port_set_pvid(struct ocelot *ocelot, int port, pvid = pvid_vlan->vid; ocelot_rmw_gix(ocelot, - ANA_PORT_VLAN_CFG_VLAN_VID(pvid), - ANA_PORT_VLAN_CFG_VLAN_VID_M, + ANA_PORT_VLAN_CFG_VLAN_VID(pvid) | tag_type, + ANA_PORT_VLAN_CFG_VLAN_VID_M | tag_type, ANA_PORT_VLAN_CFG, port); /* If there's no pvid, we should drop not only untagged traffic (which @@ -585,6 +609,15 @@ int ocelot_port_vlan_filtering(struct ocelot *ocelot, int port, ANA_PORT_VLAN_CFG_VLAN_POP_CNT(1); else val = 0; + + /* if switch is enabled for QinQ, the port for LAN should set + * VLAN_CFG.VLAN_POP_CNT=0 && VLAN_CFG.VLAN_AWARE_ENA=0. + * the port for MAN should set VLAN_CFG.VLAN_POP_CNT=1 && + * VLAN_CFG.VLAN_AWARE_ENA=1. referring to 4.3.3 in VSC9959_1_00_TS.pdf + */ + if (ocelot->qinq_enable && !ocelot_port->qinq_mode) + val = 0; + ocelot_rmw_gix(ocelot, val, ANA_PORT_VLAN_CFG_VLAN_AWARE_ENA | ANA_PORT_VLAN_CFG_VLAN_POP_CNT_M, @@ -1517,7 +1550,12 @@ u32 ocelot_get_bridge_fwd_mask(struct ocelot *ocelot, int src_port) if (!ocelot_port) continue; - if (ocelot_port->stp_state == BR_STATE_FORWARDING && + /* Keep the bridge port in forward mask if the port is forward + * state or force_forward mode. + * FRER need to set the port to force_forward mode. + */ + if ((ocelot_port->stp_state == BR_STATE_FORWARDING || + ocelot_port->force_forward) && ocelot_port->bridge == bridge) mask |= BIT(port); } diff --git a/drivers/net/ethernet/mscc/ocelot_flower.c b/drivers/net/ethernet/mscc/ocelot_flower.c index ee052404eb55..ee9656291fff 100644 --- a/drivers/net/ethernet/mscc/ocelot_flower.c +++ b/drivers/net/ethernet/mscc/ocelot_flower.c @@ -468,20 +468,27 @@ static int ocelot_flower_parse_action(struct ocelot *ocelot, int port, switch (ntohs(a->vlan.proto)) { case ETH_P_8021Q: tpid = OCELOT_TAG_TPID_SEL_8021Q; + filter->action.tag_b_tpid_sel = tpid; + filter->action.push_inner_tag = OCELOT_ES0_TAG; + filter->action.tag_b_vid_sel = OCELOT_ES0_VID; + filter->action.tag_b_pcp_sel = OCELOT_ES0_PCP; + filter->action.vid_b_val = a->vlan.vid; + filter->action.pcp_b_val = a->vlan.prio; break; case ETH_P_8021AD: tpid = OCELOT_TAG_TPID_SEL_8021AD; + filter->action.tag_a_tpid_sel = tpid; + filter->action.push_outer_tag = OCELOT_ES0_TAG; + filter->action.tag_a_vid_sel = OCELOT_ES0_VID; + filter->action.tag_a_pcp_sel = OCELOT_ES0_PCP; + filter->action.vid_a_val = a->vlan.vid; + filter->action.pcp_a_val = a->vlan.prio; break; default: NL_SET_ERR_MSG_MOD(extack, "Cannot push custom TPID"); return -EOPNOTSUPP; } - filter->action.tag_a_tpid_sel = tpid; - filter->action.push_outer_tag = OCELOT_ES0_TAG; - filter->action.tag_a_vid_sel = OCELOT_ES0_VID; - filter->action.vid_a_val = a->vlan.vid; - filter->action.pcp_a_val = a->vlan.prio; filter->type = OCELOT_VCAP_FILTER_OFFLOAD; break; case FLOW_ACTION_GATE: @@ -586,6 +593,7 @@ ocelot_flower_parse_key(struct ocelot *ocelot, int port, bool ingress, BIT(FLOW_DISSECTOR_KEY_META) | BIT(FLOW_DISSECTOR_KEY_PORTS) | BIT(FLOW_DISSECTOR_KEY_VLAN) | + BIT(FLOW_DISSECTOR_KEY_CVLAN) | BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS))) { @@ -617,6 +625,18 @@ ocelot_flower_parse_key(struct ocelot *ocelot, int port, bool ingress, match_protocol = false; } + if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CVLAN)) { + struct flow_match_vlan match; + + flow_rule_match_cvlan(rule, &match); + filter->key_type = OCELOT_VCAP_KEY_ANY; + filter->cvlan.vid.value = match.key->vlan_id; + filter->cvlan.vid.mask = match.mask->vlan_id; + filter->cvlan.pcp.value[0] = match.key->vlan_priority; + filter->cvlan.pcp.mask[0] = match.mask->vlan_priority; + match_protocol = false; + } + if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) { struct flow_match_eth_addrs match; diff --git a/drivers/net/ethernet/mscc/ocelot_ptp.c b/drivers/net/ethernet/mscc/ocelot_ptp.c index cb32234a5bf1..af69d90350e9 100644 --- a/drivers/net/ethernet/mscc/ocelot_ptp.c +++ b/drivers/net/ethernet/mscc/ocelot_ptp.c @@ -532,6 +532,7 @@ int ocelot_hwstamp_set(struct ocelot *ocelot, int port, struct ifreq *ifr) switch (cfg.tx_type) { case HWTSTAMP_TX_ON: ocelot_port->ptp_cmd = IFH_REW_OP_TWO_STEP_PTP; + skb_queue_purge(&ocelot_port->tx_skbs); break; case HWTSTAMP_TX_ONESTEP_SYNC: /* IFH_REW_OP_ONE_STEP_PTP updates the correctional field, we diff --git a/drivers/net/ethernet/mscc/ocelot_vcap.c b/drivers/net/ethernet/mscc/ocelot_vcap.c index 73cdec5ca6a3..9947b634194a 100644 --- a/drivers/net/ethernet/mscc/ocelot_vcap.c +++ b/drivers/net/ethernet/mscc/ocelot_vcap.c @@ -666,10 +666,11 @@ static void is1_action_set(struct ocelot *ocelot, struct vcap_data *data, vcap_action_set(vcap, data, VCAP_IS1_ACT_PAG_VAL, a->pag_val); } -static void is1_entry_set(struct ocelot *ocelot, int ix, +static int is1_entry_set(struct ocelot *ocelot, int ix, struct ocelot_vcap_filter *filter) { const struct vcap_props *vcap = &ocelot->vcap[VCAP_IS1]; + struct ocelot_vcap_key_vlan *ctag = &filter->cvlan; struct ocelot_vcap_key_vlan *tag = &filter->vlan; struct vcap_data data; int row = ix / 2; @@ -689,6 +690,11 @@ static void is1_entry_set(struct ocelot *ocelot, int ix, if (filter->prio != 0) data.tg |= data.tg_value; + if (filter->lookup == 1) + type = IS1_TYPE_S1_5TUPLE_IP4; + else + type = IS1_TYPE_S1_NORMAL; + vcap_key_set(vcap, &data, VCAP_IS1_HK_LOOKUP, filter->lookup, 0x3); vcap_key_set(vcap, &data, VCAP_IS1_HK_IGR_PORT_MASK, 0, ~filter->ingress_port_mask); @@ -699,12 +705,31 @@ static void is1_entry_set(struct ocelot *ocelot, int ix, tag->vid.value, tag->vid.mask); vcap_key_set(vcap, &data, VCAP_IS1_HK_PCP, tag->pcp.value[0], tag->pcp.mask[0]); - type = IS1_TYPE_S1_NORMAL; + + if (ctag->vid.value != 0) { + if (type != IS1_TYPE_S1_5TUPLE_IP4) + return -EOPNOTSUPP; + + vcap_key_bit_set(vcap, &data, VCAP_IS1_HK_TPID, + OCELOT_VCAP_BIT_1); + vcap_key_bit_set(vcap, &data, VCAP_IS1_HK_VLAN_DBL_TAGGED, + OCELOT_VCAP_BIT_1); + vcap_key_set(vcap, &data, VCAP_IS1_HK_IP4_INNER_VID, + ctag->vid.value, ctag->vid.mask); + vcap_key_set(vcap, &data, VCAP_IS1_HK_IP4_INNER_PCP, + ctag->pcp.value[0], ctag->pcp.mask[0]); + } switch (filter->key_type) { case OCELOT_VCAP_KEY_ETYPE: { struct ocelot_vcap_key_etype *etype = &filter->key.etype; + if (type != IS1_TYPE_S1_NORMAL) + return -EOPNOTSUPP; + + if (etype->dmac.mask[0]) + return -EOPNOTSUPP; + vcap_key_bytes_set(vcap, &data, VCAP_IS1_HK_L2_SMAC, etype->smac.value, etype->smac.mask); vcap_key_bytes_set(vcap, &data, VCAP_IS1_HK_ETYPE, @@ -716,48 +741,103 @@ static void is1_entry_set(struct ocelot *ocelot, int ix, struct ocelot_vcap_udp_tcp *sport = &ipv4->sport; struct ocelot_vcap_udp_tcp *dport = &ipv4->dport; enum ocelot_vcap_bit tcp_udp = OCELOT_VCAP_BIT_0; + enum ocelot_vcap_bit fragment = ipv4->fragment; + enum ocelot_vcap_bit options = ipv4->options; + enum ocelot_vcap_bit tcp = OCELOT_VCAP_BIT_0; struct ocelot_vcap_u8 proto = ipv4->proto; struct ocelot_vcap_ipv4 sip = ipv4->sip; + struct ocelot_vcap_ipv4 dip = ipv4->dip; + struct ocelot_vcap_u32 port; u32 val, msk; - vcap_key_bit_set(vcap, &data, VCAP_IS1_HK_IP_SNAP, - OCELOT_VCAP_BIT_1); - vcap_key_bit_set(vcap, &data, VCAP_IS1_HK_IP4, - OCELOT_VCAP_BIT_1); - vcap_key_bit_set(vcap, &data, VCAP_IS1_HK_ETYPE_LEN, - OCELOT_VCAP_BIT_1); - vcap_key_bytes_set(vcap, &data, VCAP_IS1_HK_L3_IP4_SIP, - sip.value.addr, sip.mask.addr); - - val = proto.value[0]; - msk = proto.mask[0]; - - if ((val == NEXTHDR_TCP || val == NEXTHDR_UDP) && msk == 0xff) - tcp_udp = OCELOT_VCAP_BIT_1; - vcap_key_bit_set(vcap, &data, VCAP_IS1_HK_TCP_UDP, tcp_udp); - - if (tcp_udp) { - enum ocelot_vcap_bit tcp = OCELOT_VCAP_BIT_0; - - if (val == NEXTHDR_TCP) - tcp = OCELOT_VCAP_BIT_1; - - vcap_key_bit_set(vcap, &data, VCAP_IS1_HK_TCP, tcp); - vcap_key_l4_port_set(vcap, &data, VCAP_IS1_HK_L4_SPORT, - sport); - /* Overloaded field */ - vcap_key_l4_port_set(vcap, &data, VCAP_IS1_HK_ETYPE, - dport); - } else { - /* IPv4 "other" frame */ - struct ocelot_vcap_u16 etype = {0}; - - /* Overloaded field */ - etype.value[0] = proto.value[0]; - etype.mask[0] = proto.mask[0]; + if (type == IS1_TYPE_S1_5TUPLE_IP4) { + vcap_key_bit_set(vcap, &data, VCAP_IS1_HK_IP4_IP4, + OCELOT_VCAP_BIT_1); + vcap_key_bit_set(vcap, &data, + VCAP_IS1_HK_IP4_L3_FRAGMENT, + fragment); + vcap_key_bit_set(vcap, &data, + VCAP_IS1_HK_IP4_L3_OPTIONS, + options); + vcap_key_bytes_set(vcap, &data, + VCAP_IS1_HK_IP4_L3_IP4_DIP, + dip.value.addr, dip.mask.addr); + vcap_key_bytes_set(vcap, &data, + VCAP_IS1_HK_IP4_L3_IP4_SIP, + sip.value.addr, sip.mask.addr); + + val = proto.value[0]; + msk = proto.mask[0]; + if ((val == NEXTHDR_TCP || val == NEXTHDR_UDP) && + msk == 0xff) { + tcp_udp = OCELOT_VCAP_BIT_1; + tcp = (val == NEXTHDR_TCP ? + OCELOT_VCAP_BIT_1 : OCELOT_VCAP_BIT_0); + + vcap_key_bit_set(vcap, &data, + VCAP_IS1_HK_IP4_TCP_UDP, + tcp_udp); + vcap_key_bit_set(vcap, &data, + VCAP_IS1_HK_IP4_TCP, tcp); + + port.value[0] = sport->value & 0xFF; + port.value[1] = sport->value >> 8; + port.value[2] = dport->value & 0xFF; + port.value[3] = dport->value >> 8; + port.mask[0] = sport->mask & 0xFF; + port.mask[1] = sport->mask >> 8; + port.mask[2] = dport->mask & 0xFF; + port.mask[3] = dport->mask >> 8; + vcap_key_bytes_set(vcap, &data, + VCAP_IS1_HK_IP4_IP_PAYLOAD_S1_5TUPLE, + port.value, port.mask); + } - vcap_key_bytes_set(vcap, &data, VCAP_IS1_HK_ETYPE, - etype.value, etype.mask); + } else if (type == IS1_TYPE_S1_NORMAL) { + vcap_key_bit_set(vcap, &data, VCAP_IS1_HK_IP_SNAP, + OCELOT_VCAP_BIT_1); + vcap_key_bit_set(vcap, &data, VCAP_IS1_HK_IP4, + OCELOT_VCAP_BIT_1); + vcap_key_bit_set(vcap, &data, VCAP_IS1_HK_ETYPE_LEN, + OCELOT_VCAP_BIT_1); + vcap_key_bytes_set(vcap, &data, VCAP_IS1_HK_L3_IP4_SIP, + sip.value.addr, sip.mask.addr); + + if (dip.mask.addr[0]) + return -EOPNOTSUPP; + + val = proto.value[0]; + msk = proto.mask[0]; + + if ((val == NEXTHDR_TCP || val == NEXTHDR_UDP) && + msk == 0xff) { + tcp_udp = OCELOT_VCAP_BIT_1; + tcp = (val == NEXTHDR_TCP ? + OCELOT_VCAP_BIT_1 : OCELOT_VCAP_BIT_0); + + vcap_key_bit_set(vcap, &data, + VCAP_IS1_HK_TCP_UDP, tcp_udp); + vcap_key_bit_set(vcap, &data, VCAP_IS1_HK_TCP, + tcp); + vcap_key_l4_port_set(vcap, &data, + VCAP_IS1_HK_L4_SPORT, + sport); + /* Overloaded field */ + vcap_key_l4_port_set(vcap, &data, + VCAP_IS1_HK_ETYPE, + dport); + } else if (msk == 0xff) { + /* IPv4 "other" frame */ + struct ocelot_vcap_u16 etype = {0}; + + /* Overloaded field */ + etype.value[0] = proto.value[0]; + etype.mask[0] = proto.mask[0]; + + vcap_key_bytes_set(vcap, &data, + VCAP_IS1_HK_ETYPE, + etype.value, etype.mask); + } } break; } @@ -775,6 +855,8 @@ static void is1_entry_set(struct ocelot *ocelot, int ix, vcap_entry2cache(ocelot, vcap, &data); vcap_action2cache(ocelot, vcap, &data); vcap_row_cmd(ocelot, vcap, row, VCAP_CMD_WRITE, VCAP_SEL_ALL); + + return 0; } static void es0_action_set(struct ocelot *ocelot, struct vcap_data *data, @@ -872,15 +954,17 @@ static void vcap_entry_get(struct ocelot *ocelot, int ix, filter->stats.pkts = cnt; } -static void vcap_entry_set(struct ocelot *ocelot, int ix, - struct ocelot_vcap_filter *filter) +static int vcap_entry_set(struct ocelot *ocelot, int ix, + struct ocelot_vcap_filter *filter) { if (filter->block_id == VCAP_IS1) return is1_entry_set(ocelot, ix, filter); if (filter->block_id == VCAP_IS2) - return is2_entry_set(ocelot, ix, filter); + is2_entry_set(ocelot, ix, filter); if (filter->block_id == VCAP_ES0) - return es0_entry_set(ocelot, ix, filter); + es0_entry_set(ocelot, ix, filter); + + return 0; } struct vcap_policer_entry { @@ -1214,8 +1298,7 @@ int ocelot_vcap_filter_add(struct ocelot *ocelot, } /* Now insert the new filter */ - vcap_entry_set(ocelot, index, filter); - return 0; + return vcap_entry_set(ocelot, index, filter); } EXPORT_SYMBOL(ocelot_vcap_filter_add); diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_common.c b/drivers/net/ethernet/netronome/nfp/nfp_net_common.c index 27f4786ace4f..a5ca5c4a7896 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_net_common.c +++ b/drivers/net/ethernet/netronome/nfp/nfp_net_common.c @@ -1631,21 +1631,21 @@ static void nfp_net_stat64(struct net_device *netdev, unsigned int start; do { - start = u64_stats_fetch_begin_irq(&r_vec->rx_sync); + start = u64_stats_fetch_begin(&r_vec->rx_sync); data[0] = r_vec->rx_pkts; data[1] = r_vec->rx_bytes; data[2] = r_vec->rx_drops; - } while (u64_stats_fetch_retry_irq(&r_vec->rx_sync, start)); + } while (u64_stats_fetch_retry(&r_vec->rx_sync, start)); stats->rx_packets += data[0]; stats->rx_bytes += data[1]; stats->rx_dropped += data[2]; do { - start = u64_stats_fetch_begin_irq(&r_vec->tx_sync); + start = u64_stats_fetch_begin(&r_vec->tx_sync); data[0] = r_vec->tx_pkts; data[1] = r_vec->tx_bytes; data[2] = r_vec->tx_errors; - } while (u64_stats_fetch_retry_irq(&r_vec->tx_sync, start)); + } while (u64_stats_fetch_retry(&r_vec->tx_sync, start)); stats->tx_packets += data[0]; stats->tx_bytes += data[1]; stats->tx_errors += data[2]; diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c b/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c index af376b900067..cc97b3d00414 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c +++ b/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c @@ -881,7 +881,7 @@ static u64 *nfp_vnic_get_sw_stats(struct net_device *netdev, u64 *data) unsigned int start; do { - start = u64_stats_fetch_begin_irq(&nn->r_vecs[i].rx_sync); + start = u64_stats_fetch_begin(&nn->r_vecs[i].rx_sync); data[0] = nn->r_vecs[i].rx_pkts; tmp[0] = nn->r_vecs[i].hw_csum_rx_ok; tmp[1] = nn->r_vecs[i].hw_csum_rx_inner_ok; @@ -889,10 +889,10 @@ static u64 *nfp_vnic_get_sw_stats(struct net_device *netdev, u64 *data) tmp[3] = nn->r_vecs[i].hw_csum_rx_error; tmp[4] = nn->r_vecs[i].rx_replace_buf_alloc_fail; tmp[5] = nn->r_vecs[i].hw_tls_rx; - } while (u64_stats_fetch_retry_irq(&nn->r_vecs[i].rx_sync, start)); + } while (u64_stats_fetch_retry(&nn->r_vecs[i].rx_sync, start)); do { - start = u64_stats_fetch_begin_irq(&nn->r_vecs[i].tx_sync); + start = u64_stats_fetch_begin(&nn->r_vecs[i].tx_sync); data[1] = nn->r_vecs[i].tx_pkts; data[2] = nn->r_vecs[i].tx_busy; tmp[6] = nn->r_vecs[i].hw_csum_tx; @@ -902,7 +902,7 @@ static u64 *nfp_vnic_get_sw_stats(struct net_device *netdev, u64 *data) tmp[10] = nn->r_vecs[i].hw_tls_tx; tmp[11] = nn->r_vecs[i].tls_tx_fallback; tmp[12] = nn->r_vecs[i].tls_tx_no_fallback; - } while (u64_stats_fetch_retry_irq(&nn->r_vecs[i].tx_sync, start)); + } while (u64_stats_fetch_retry(&nn->r_vecs[i].tx_sync, start)); data += NN_RVEC_PER_Q_STATS; diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_repr.c b/drivers/net/ethernet/netronome/nfp/nfp_net_repr.c index 8b77582bdfa0..a6b6ca1fd55e 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_net_repr.c +++ b/drivers/net/ethernet/netronome/nfp/nfp_net_repr.c @@ -134,13 +134,13 @@ nfp_repr_get_host_stats64(const struct net_device *netdev, repr_stats = per_cpu_ptr(repr->stats, i); do { - start = u64_stats_fetch_begin_irq(&repr_stats->syncp); + start = u64_stats_fetch_begin(&repr_stats->syncp); tbytes = repr_stats->tx_bytes; tpkts = repr_stats->tx_packets; tdrops = repr_stats->tx_drops; rbytes = repr_stats->rx_bytes; rpkts = repr_stats->rx_packets; - } while (u64_stats_fetch_retry_irq(&repr_stats->syncp, start)); + } while (u64_stats_fetch_retry(&repr_stats->syncp, start)); stats->tx_bytes += tbytes; stats->tx_packets += tpkts; diff --git a/drivers/net/ethernet/nvidia/forcedeth.c b/drivers/net/ethernet/nvidia/forcedeth.c index 486cbc8ab224..7a549b834e97 100644 --- a/drivers/net/ethernet/nvidia/forcedeth.c +++ b/drivers/net/ethernet/nvidia/forcedeth.c @@ -1734,12 +1734,12 @@ static void nv_get_stats(int cpu, struct fe_priv *np, u64 tx_packets, tx_bytes, tx_dropped; do { - syncp_start = u64_stats_fetch_begin_irq(&np->swstats_rx_syncp); + syncp_start = u64_stats_fetch_begin(&np->swstats_rx_syncp); rx_packets = src->stat_rx_packets; rx_bytes = src->stat_rx_bytes; rx_dropped = src->stat_rx_dropped; rx_missed_errors = src->stat_rx_missed_errors; - } while (u64_stats_fetch_retry_irq(&np->swstats_rx_syncp, syncp_start)); + } while (u64_stats_fetch_retry(&np->swstats_rx_syncp, syncp_start)); storage->rx_packets += rx_packets; storage->rx_bytes += rx_bytes; @@ -1747,11 +1747,11 @@ static void nv_get_stats(int cpu, struct fe_priv *np, storage->rx_missed_errors += rx_missed_errors; do { - syncp_start = u64_stats_fetch_begin_irq(&np->swstats_tx_syncp); + syncp_start = u64_stats_fetch_begin(&np->swstats_tx_syncp); tx_packets = src->stat_tx_packets; tx_bytes = src->stat_tx_bytes; tx_dropped = src->stat_tx_dropped; - } while (u64_stats_fetch_retry_irq(&np->swstats_tx_syncp, syncp_start)); + } while (u64_stats_fetch_retry(&np->swstats_tx_syncp, syncp_start)); storage->tx_packets += tx_packets; storage->tx_bytes += tx_bytes; diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_vnd.c b/drivers/net/ethernet/qualcomm/rmnet/rmnet_vnd.c index 1b2119b1d48a..3f5e6572d20e 100644 --- a/drivers/net/ethernet/qualcomm/rmnet/rmnet_vnd.c +++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_vnd.c @@ -135,9 +135,9 @@ static void rmnet_get_stats64(struct net_device *dev, pcpu_ptr = per_cpu_ptr(priv->pcpu_stats, cpu); do { - start = u64_stats_fetch_begin_irq(&pcpu_ptr->syncp); + start = u64_stats_fetch_begin(&pcpu_ptr->syncp); snapshot = pcpu_ptr->stats; /* struct assignment */ - } while (u64_stats_fetch_retry_irq(&pcpu_ptr->syncp, start)); + } while (u64_stats_fetch_retry(&pcpu_ptr->syncp, start)); total_stats.rx_pkts += snapshot.rx_pkts; total_stats.rx_bytes += snapshot.rx_bytes; diff --git a/drivers/net/ethernet/realtek/8139too.c b/drivers/net/ethernet/realtek/8139too.c index 469e2e229c6e..9ce0e8a64ba8 100644 --- a/drivers/net/ethernet/realtek/8139too.c +++ b/drivers/net/ethernet/realtek/8139too.c @@ -2532,16 +2532,16 @@ rtl8139_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) netdev_stats_to_stats64(stats, &dev->stats); do { - start = u64_stats_fetch_begin_irq(&tp->rx_stats.syncp); + start = u64_stats_fetch_begin(&tp->rx_stats.syncp); stats->rx_packets = tp->rx_stats.packets; stats->rx_bytes = tp->rx_stats.bytes; - } while (u64_stats_fetch_retry_irq(&tp->rx_stats.syncp, start)); + } while (u64_stats_fetch_retry(&tp->rx_stats.syncp, start)); do { - start = u64_stats_fetch_begin_irq(&tp->tx_stats.syncp); + start = u64_stats_fetch_begin(&tp->tx_stats.syncp); stats->tx_packets = tp->tx_stats.packets; stats->tx_bytes = tp->tx_stats.bytes; - } while (u64_stats_fetch_retry_irq(&tp->tx_stats.syncp, start)); + } while (u64_stats_fetch_retry(&tp->tx_stats.syncp, start)); } /* Set or clear the multicast filter for this adaptor. diff --git a/drivers/net/ethernet/socionext/sni_ave.c b/drivers/net/ethernet/socionext/sni_ave.c index d2c6a5dfdc0e..b7e24ae92525 100644 --- a/drivers/net/ethernet/socionext/sni_ave.c +++ b/drivers/net/ethernet/socionext/sni_ave.c @@ -1508,16 +1508,16 @@ static void ave_get_stats64(struct net_device *ndev, unsigned int start; do { - start = u64_stats_fetch_begin_irq(&priv->stats_rx.syncp); + start = u64_stats_fetch_begin(&priv->stats_rx.syncp); stats->rx_packets = priv->stats_rx.packets; stats->rx_bytes = priv->stats_rx.bytes; - } while (u64_stats_fetch_retry_irq(&priv->stats_rx.syncp, start)); + } while (u64_stats_fetch_retry(&priv->stats_rx.syncp, start)); do { - start = u64_stats_fetch_begin_irq(&priv->stats_tx.syncp); + start = u64_stats_fetch_begin(&priv->stats_tx.syncp); stats->tx_packets = priv->stats_tx.packets; stats->tx_bytes = priv->stats_tx.bytes; - } while (u64_stats_fetch_retry_irq(&priv->stats_tx.syncp, start)); + } while (u64_stats_fetch_retry(&priv->stats_tx.syncp, start)); stats->rx_errors = priv->stats_rx.errors; stats->tx_errors = priv->stats_tx.errors; diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c b/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c index 84276eb681d7..7d6d6e3b1df7 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c @@ -84,22 +84,38 @@ static void dwmac4_rx_queue_enable(struct mac_device_info *hw, } static void dwmac4_rx_queue_priority(struct mac_device_info *hw, - u32 prio, u32 queue) + u32 prio_mask, u32 queue) { void __iomem *ioaddr = hw->pcsr; - u32 base_register; - u32 value; + u32 clear_mask = 0; + u32 ctrl2, ctrl3; + int i; - base_register = (queue < 4) ? GMAC_RXQ_CTRL2 : GMAC_RXQ_CTRL3; - if (queue >= 4) - queue -= 4; + ctrl2 = readl(ioaddr + GMAC_RXQ_CTRL2); + ctrl3 = readl(ioaddr + GMAC_RXQ_CTRL3); - value = readl(ioaddr + base_register); + for (i = 0; i < 4; i++) + clear_mask |= ((prio_mask << GMAC_RXQCTRL_PSRQX_SHIFT(i)) & + GMAC_RXQCTRL_PSRQX_MASK(i)); + + ctrl2 &= ~clear_mask; + ctrl3 &= ~clear_mask; - value &= ~GMAC_RXQCTRL_PSRQX_MASK(queue); - value |= (prio << GMAC_RXQCTRL_PSRQX_SHIFT(queue)) & + if (queue < 4) { + ctrl2 |= (prio_mask << GMAC_RXQCTRL_PSRQX_SHIFT(queue)) & GMAC_RXQCTRL_PSRQX_MASK(queue); - writel(value, ioaddr + base_register); + + writel(ctrl2, ioaddr + GMAC_RXQ_CTRL2); + writel(ctrl3, ioaddr + GMAC_RXQ_CTRL3); + } else { + queue -= 4; + + ctrl3 |= (prio_mask << GMAC_RXQCTRL_PSRQX_SHIFT(queue)) & + GMAC_RXQCTRL_PSRQX_MASK(queue); + + writel(ctrl3, ioaddr + GMAC_RXQ_CTRL3); + writel(ctrl2, ioaddr + GMAC_RXQ_CTRL2); + } } static void dwmac4_tx_queue_priority(struct mac_device_info *hw, @@ -1209,6 +1225,7 @@ const struct stmmac_ops dwmac410_ops = { .est_configure = dwmac5_est_configure, .est_irq_status = dwmac5_est_irq_status, .fpe_configure = dwmac5_fpe_configure, + .fpe_configure_get = dwmac5_fpe_configure_get, .fpe_send_mpacket = dwmac5_fpe_send_mpacket, .fpe_irq_status = dwmac5_fpe_irq_status, .add_hw_vlan_rx_fltr = dwmac4_add_hw_vlan_rx_fltr, @@ -1261,6 +1278,7 @@ const struct stmmac_ops dwmac510_ops = { .est_configure = dwmac5_est_configure, .est_irq_status = dwmac5_est_irq_status, .fpe_configure = dwmac5_fpe_configure, + .fpe_configure_get = dwmac5_fpe_configure_get, .fpe_send_mpacket = dwmac5_fpe_send_mpacket, .fpe_irq_status = dwmac5_fpe_irq_status, .add_hw_vlan_rx_fltr = dwmac4_add_hw_vlan_rx_fltr, diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac5.c b/drivers/net/ethernet/stmicro/stmmac/dwmac5.c index e95d35f1e5a0..8633c0fa4c76 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac5.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac5.c @@ -711,10 +711,15 @@ void dwmac5_est_irq_status(void __iomem *ioaddr, struct net_device *dev, } void dwmac5_fpe_configure(void __iomem *ioaddr, u32 num_txq, u32 num_rxq, - bool enable) + bool enable, struct stmmac_fpe *fpe) { u32 value; + if (fpe) { + value = fpe->p_queues << MTL_FPECTRL_PEC_SHIFT | fpe->fragsize; + writel(value, ioaddr + MTL_FPE_CTRL_STS); + } + if (!enable) { value = readl(ioaddr + MAC_FPE_CTRL_STS); @@ -734,6 +739,18 @@ void dwmac5_fpe_configure(void __iomem *ioaddr, u32 num_txq, u32 num_rxq, writel(value, ioaddr + MAC_FPE_CTRL_STS); } +void dwmac5_fpe_configure_get(void __iomem *ioaddr, struct stmmac_fpe *fpe) +{ + u32 value; + + value = readl(ioaddr + MAC_FPE_CTRL_STS); + fpe->enable = value & EFPE; + + value = readl(ioaddr + MTL_FPE_CTRL_STS); + fpe->p_queues = (value >> MTL_FPECTRL_PEC_SHIFT) & GENMASK(7, 0); + fpe->fragsize = value & GENMASK(1, 0); +} + int dwmac5_fpe_irq_status(void __iomem *ioaddr, struct net_device *dev) { u32 value; diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac5.h b/drivers/net/ethernet/stmicro/stmmac/dwmac5.h index 53c138d0ff48..ead1f7d4dc19 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac5.h +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac5.h @@ -20,6 +20,9 @@ #define SVER BIT(1) #define EFPE BIT(0) +#define MTL_FPE_CTRL_STS 0x00000c90 +#define MTL_FPECTRL_PEC_SHIFT 8 + #define MAC_PPS_CONTROL 0x00000b70 #define PPS_MAXIDX(x) ((((x) + 1) * 8) - 1) #define PPS_MINIDX(x) ((x) * 8) @@ -154,7 +157,8 @@ int dwmac5_est_configure(void __iomem *ioaddr, struct stmmac_est *cfg, void dwmac5_est_irq_status(void __iomem *ioaddr, struct net_device *dev, struct stmmac_extra_stats *x, u32 txqcnt); void dwmac5_fpe_configure(void __iomem *ioaddr, u32 num_txq, u32 num_rxq, - bool enable); + bool enable, struct stmmac_fpe *fpe); +void dwmac5_fpe_configure_get(void __iomem *ioaddr, struct stmmac_fpe *fpe); void dwmac5_fpe_send_mpacket(void __iomem *ioaddr, enum stmmac_mpacket_type type); int dwmac5_fpe_irq_status(void __iomem *ioaddr, struct net_device *dev); diff --git a/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_core.c b/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_core.c index c6c4d7948fe5..a004b2e5d2e3 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_core.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_core.c @@ -1430,7 +1430,8 @@ static int dwxgmac3_est_configure(void __iomem *ioaddr, struct stmmac_est *cfg, } static void dwxgmac3_fpe_configure(void __iomem *ioaddr, u32 num_txq, - u32 num_rxq, bool enable) + u32 num_rxq, bool enable, + struct stmmac_fpe *fpe) { u32 value; diff --git a/drivers/net/ethernet/stmicro/stmmac/hwif.h b/drivers/net/ethernet/stmicro/stmmac/hwif.h index 156071275125..2cea77133e7f 100644 --- a/drivers/net/ethernet/stmicro/stmmac/hwif.h +++ b/drivers/net/ethernet/stmicro/stmmac/hwif.h @@ -279,6 +279,12 @@ struct stmmac_pps_cfg; struct stmmac_rss; struct stmmac_est; +struct stmmac_fpe { + u8 enable; + u8 p_queues; + u8 fragsize; +}; + /* Helpers to program the MAC core */ struct stmmac_ops { /* MAC core initialization */ @@ -391,7 +397,8 @@ struct stmmac_ops { void (*est_irq_status)(void __iomem *ioaddr, struct net_device *dev, struct stmmac_extra_stats *x, u32 txqcnt); void (*fpe_configure)(void __iomem *ioaddr, u32 num_txq, u32 num_rxq, - bool enable); + bool enable, struct stmmac_fpe *fpe); + void (*fpe_configure_get)(void __iomem *ioaddr, struct stmmac_fpe *fpe); void (*fpe_send_mpacket)(void __iomem *ioaddr, enum stmmac_mpacket_type type); int (*fpe_irq_status)(void __iomem *ioaddr, struct net_device *dev); @@ -495,6 +502,8 @@ struct stmmac_ops { stmmac_do_void_callback(__priv, mac, est_irq_status, __args) #define stmmac_fpe_configure(__priv, __args...) \ stmmac_do_void_callback(__priv, mac, fpe_configure, __args) +#define stmmac_fpe_configure_get(__priv, __args...) \ + stmmac_do_void_callback(__priv, mac, fpe_configure_get, __args) #define stmmac_fpe_send_mpacket(__priv, __args...) \ stmmac_do_void_callback(__priv, mac, fpe_send_mpacket, __args) #define stmmac_fpe_irq_status(__priv, __args...) \ diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac.h b/drivers/net/ethernet/stmicro/stmmac/stmmac.h index 782825822278..d07b173791e4 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac.h +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac.h @@ -325,6 +325,7 @@ struct stmmac_priv { /* XDP BPF Program */ unsigned long *af_xdp_zc_qps; struct bpf_prog *xdp_prog; + bool fp_enabled_admin; }; enum stmmac_state { diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c index 35c8dd92d369..79988e61be69 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c @@ -19,6 +19,7 @@ #include "stmmac.h" #include "dwmac_dma.h" #include "dwxgmac2.h" +#include "common.h" #define REG_SPACE_SIZE 0x1060 #define GMAC4_REG_SPACE_SIZE 0x116C @@ -1149,6 +1150,105 @@ static int stmmac_set_tunable(struct net_device *dev, return ret; } +static int stmmac_reset_preempt(struct net_device *dev, bool enable) +{ + struct stmmac_priv *priv = netdev_priv(dev); + struct stmmac_fpe_cfg *fpe_cfg = priv->plat->fpe_cfg; + bool *hs_enable = &fpe_cfg->hs_enable; + + if (enable) { + if (priv->fp_enabled_admin) { + priv->plat->fpe_cfg->enable = 1; + if (!*hs_enable) + stmmac_fpe_configure(priv, priv->ioaddr, priv->plat->tx_queues_to_use, + priv->plat->rx_queues_to_use, true, NULL); + } + } else { + priv->plat->fpe_cfg->enable = 0; + stmmac_fpe_configure(priv, priv->ioaddr, priv->plat->tx_queues_to_use, + priv->plat->rx_queues_to_use, false, NULL); + } + + return 0; +} + +static int stmmac_set_preempt(struct net_device *dev, struct ethtool_fp *fpcmd) +{ + struct stmmac_priv *priv = netdev_priv(dev); + struct stmmac_fpe fpe; + + if (!priv->dma_cap.fpesel) + return -EOPNOTSUPP; + + if (!fpcmd->disabled && + (fpcmd->min_frag_size < 60 || fpcmd->min_frag_size > 252)) + return -EINVAL; + + fpe.p_queues = fpcmd->preemptible_queues_mask; + fpe.fragsize = DIV_ROUND_UP((fpcmd->min_frag_size + 4), 64) - 1; + + if (priv->plat->fpe_cfg->lo_fpe_state == FPE_STATE_ON) + fpe.enable = 1; + else + fpe.enable = fpcmd->fp_enabled ? 0 : 1; + + /* To support preemption MAC should have more than 1 TX queue with at + * least 1 Queue designated as Express Queue. Queue 0 is always used as + * preemption queue when preemption MAC is enabled. + */ + if (fpe.p_queues >= (GENMASK(priv->plat->tx_queues_to_use - 1, 0) - 1)) { + netdev_err(priv->dev, + "Preemptable queue mask 0x%x not supported.", + fpe.p_queues); + return -EINVAL; + } + + if (fpcmd->disabled) { + fpe.enable = 0; + priv->plat->fpe_cfg->enable = 0; + priv->fp_enabled_admin = 0; + } else { + priv->plat->fpe_cfg->enable = 1; + priv->fp_enabled_admin = 1; + } + stmmac_fpe_configure(priv, priv->ioaddr, priv->plat->tx_queues_to_use, + priv->plat->rx_queues_to_use, fpe.enable, &fpe); + + stmmac_fpe_handshake(priv, fpcmd->fp_enabled); + + return 0; +} + +static int stmmac_get_preempt(struct net_device *dev, struct ethtool_fp *fpcmd) +{ + struct stmmac_priv *priv = netdev_priv(dev); + struct stmmac_fpe fpe; + int ret; + + ret = stmmac_fpe_configure_get(priv, priv->ioaddr, &fpe); + if (ret) { + fpcmd->fp_supported = 0; + fpcmd->supported_queues_mask = 0; + fpcmd->preemptible_queues_mask = 0; + fpcmd->min_frag_size = 0; + + return 0; + } + + fpcmd->fp_supported = 1; + fpcmd->fp_status = priv->plat->fpe_cfg->enable; + fpcmd->fp_active = fpe.enable; + fpcmd->supported_queues_mask = GENMASK(priv->plat->tx_queues_to_use - 1, + 0); + fpcmd->preemptible_queues_mask = fpe.p_queues; + /* Queue 0 is always preemption when preemption is active. */ + if (fpcmd->fp_active) + fpcmd->preemptible_queues_mask |= 1; + fpcmd->min_frag_size = (fpe.fragsize + 1) * 64 - 4; + + return 0; +} + static const struct ethtool_ops stmmac_ethtool_ops = { .supported_coalesce_params = ETHTOOL_COALESCE_USECS | ETHTOOL_COALESCE_MAX_FRAMES, @@ -1188,6 +1288,9 @@ static const struct ethtool_ops stmmac_ethtool_ops = { .set_tunable = stmmac_set_tunable, .get_link_ksettings = stmmac_ethtool_get_link_ksettings, .set_link_ksettings = stmmac_ethtool_set_link_ksettings, + .set_preempt = stmmac_set_preempt, + .get_preempt = stmmac_get_preempt, + .reset_preempt = stmmac_reset_preempt, }; void stmmac_set_ethtool_ops(struct net_device *netdev) diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c index 72c0b874acee..63a0738ebdf2 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c @@ -956,11 +956,17 @@ static void stmmac_fpe_link_state_handle(struct stmmac_priv *priv, bool is_up) enum stmmac_fpe_state *lp_state = &fpe_cfg->lp_fpe_state; bool *hs_enable = &fpe_cfg->hs_enable; - if (is_up && *hs_enable) { - stmmac_fpe_send_mpacket(priv, priv->ioaddr, MPACKET_VERIFY); + if (is_up) { + if (*hs_enable) + stmmac_fpe_send_mpacket(priv, priv->ioaddr, MPACKET_VERIFY); } else { *lo_state = FPE_STATE_OFF; *lp_state = FPE_STATE_OFF; + priv->plat->fpe_cfg->enable = false; + stmmac_fpe_configure(priv, priv->ioaddr, + priv->plat->tx_queues_to_use, + priv->plat->rx_queues_to_use, + false, NULL); } } @@ -2433,12 +2439,14 @@ static bool stmmac_xdp_xmit_zc(struct stmmac_priv *priv, u32 queue, u32 budget) /* We are sharing with slow path and stop XSK TX desc submission when * available TX ring is less than threshold. */ - if (unlikely(stmmac_tx_avail(priv, queue) < STMMAC_TX_XSK_AVAIL) || - !netif_carrier_ok(priv->dev)) { + if (unlikely(stmmac_tx_avail(priv, queue) < STMMAC_TX_XSK_AVAIL)) { work_done = false; break; } + if (!netif_carrier_ok(priv->dev)) + break; + if (!xsk_tx_peek_desc(pool, &xdp_desc)) break; @@ -6775,7 +6783,9 @@ static const struct net_device_ops stmmac_netdev_ops = { .ndo_fix_features = stmmac_fix_features, .ndo_set_features = stmmac_set_features, .ndo_set_rx_mode = stmmac_set_rx_mode, +#ifndef CONFIG_NET_SCH_MULTIQ .ndo_tx_timeout = stmmac_tx_timeout, +#endif .ndo_eth_ioctl = stmmac_ioctl, .ndo_setup_tc = stmmac_setup_tc, .ndo_select_queue = stmmac_select_queue, @@ -6928,6 +6938,7 @@ static void stmmac_napi_add(struct net_device *dev) { struct stmmac_priv *priv = netdev_priv(dev); u32 queue, maxq; + char name[NAPINAMSIZ]; maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use); @@ -6939,16 +6950,22 @@ static void stmmac_napi_add(struct net_device *dev) spin_lock_init(&ch->lock); if (queue < priv->plat->rx_queues_to_use) { - netif_napi_add(dev, &ch->rx_napi, stmmac_napi_poll_rx); + snprintf(name, NAPINAMSIZ, "rx-%d", queue); + netif_napi_add_named(dev, &ch->rx_napi, stmmac_napi_poll_rx, + NAPI_POLL_WEIGHT, name); } if (queue < priv->plat->tx_queues_to_use) { - netif_napi_add_tx(dev, &ch->tx_napi, - stmmac_napi_poll_tx); + snprintf(name, NAPINAMSIZ, "tx-%d", queue); + netif_napi_add_tx_named(dev, &ch->tx_napi, + stmmac_napi_poll_tx, + NAPI_POLL_WEIGHT, name); } if (queue < priv->plat->rx_queues_to_use && queue < priv->plat->tx_queues_to_use) { - netif_napi_add(dev, &ch->rxtx_napi, - stmmac_napi_poll_rxtx); + snprintf(name, NAPINAMSIZ, "zc-%d", queue); + netif_napi_add_named(dev, &ch->rxtx_napi, + stmmac_napi_poll_rxtx, + NAPI_POLL_WEIGHT, name); } } } @@ -7025,7 +7042,6 @@ static void stmmac_fpe_lp_task(struct work_struct *work) enum stmmac_fpe_state *lo_state = &fpe_cfg->lo_fpe_state; enum stmmac_fpe_state *lp_state = &fpe_cfg->lp_fpe_state; bool *hs_enable = &fpe_cfg->hs_enable; - bool *enable = &fpe_cfg->enable; int retries = 20; while (retries-- > 0) { @@ -7038,7 +7054,7 @@ static void stmmac_fpe_lp_task(struct work_struct *work) stmmac_fpe_configure(priv, priv->ioaddr, priv->plat->tx_queues_to_use, priv->plat->rx_queues_to_use, - *enable); + true, NULL); netdev_info(priv->dev, "configured FPE\n"); @@ -7481,7 +7497,7 @@ int stmmac_suspend(struct device *dev) /* Disable FPE */ stmmac_fpe_configure(priv, priv->ioaddr, priv->plat->tx_queues_to_use, - priv->plat->rx_queues_to_use, false); + priv->plat->rx_queues_to_use, false, NULL); stmmac_fpe_handshake(priv, false); stmmac_fpe_stop_wq(priv); diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_tc.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_tc.c index 773e415cc2de..7c1759d99952 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_tc.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_tc.c @@ -922,7 +922,7 @@ static int tc_setup_taprio(struct stmmac_priv *priv, struct plat_stmmacenet_data *plat = priv->plat; struct timespec64 time, current_time, qopt_time; ktime_t current_time_ns; - bool fpe = false; + struct stmmac_fpe fpe; int i, ret = 0; u64 ctr; @@ -988,6 +988,8 @@ static int tc_setup_taprio(struct stmmac_priv *priv, priv->plat->est->enable = qopt->enable; mutex_unlock(&priv->plat->est->lock); + stmmac_fpe_configure_get(priv, priv->ioaddr, &fpe); + for (i = 0; i < size; i++) { s64 delta_ns = qopt->entries[i].interval; u32 gates = qopt->entries[i].gate_mask; @@ -999,16 +1001,18 @@ static int tc_setup_taprio(struct stmmac_priv *priv, switch (qopt->entries[i].command) { case TC_TAPRIO_CMD_SET_GATES: - if (fpe) + if (fpe.enable) return -EINVAL; break; case TC_TAPRIO_CMD_SET_AND_HOLD: + if (!fpe.enable) + return -EINVAL; gates |= BIT(0); - fpe = true; break; case TC_TAPRIO_CMD_SET_AND_RELEASE: + if (!fpe.enable) + return -EINVAL; gates &= ~BIT(0); - fpe = true; break; default: return -EOPNOTSUPP; @@ -1035,16 +1039,6 @@ static int tc_setup_taprio(struct stmmac_priv *priv, priv->plat->est->ctr[0] = do_div(ctr, NSEC_PER_SEC); priv->plat->est->ctr[1] = (u32)ctr; - if (fpe && !priv->dma_cap.fpesel) { - mutex_unlock(&priv->plat->est->lock); - return -EOPNOTSUPP; - } - - /* Actual FPE register configuration will be done after FPE handshake - * is success. - */ - priv->plat->fpe_cfg->enable = fpe; - ret = stmmac_est_configure(priv, priv->ioaddr, priv->plat->est, priv->plat->clk_ptp_rate); mutex_unlock(&priv->plat->est->lock); @@ -1055,11 +1049,6 @@ static int tc_setup_taprio(struct stmmac_priv *priv, netdev_info(priv->dev, "configured EST\n"); - if (fpe) { - stmmac_fpe_handshake(priv, true); - netdev_info(priv->dev, "start FPE handshake\n"); - } - return 0; disable: @@ -1071,16 +1060,6 @@ static int tc_setup_taprio(struct stmmac_priv *priv, mutex_unlock(&priv->plat->est->lock); } - priv->plat->fpe_cfg->enable = false; - stmmac_fpe_configure(priv, priv->ioaddr, - priv->plat->tx_queues_to_use, - priv->plat->rx_queues_to_use, - false); - netdev_info(priv->dev, "disabled FPE\n"); - - stmmac_fpe_handshake(priv, false); - netdev_info(priv->dev, "stop FPE handshake\n"); - return ret; } diff --git a/drivers/net/ethernet/ti/am65-cpsw-nuss.c b/drivers/net/ethernet/ti/am65-cpsw-nuss.c index 25466cbdc16b..450c20d65d19 100644 --- a/drivers/net/ethernet/ti/am65-cpsw-nuss.c +++ b/drivers/net/ethernet/ti/am65-cpsw-nuss.c @@ -1376,12 +1376,12 @@ static void am65_cpsw_nuss_ndo_get_stats(struct net_device *dev, cpu_stats = per_cpu_ptr(ndev_priv->stats, cpu); do { - start = u64_stats_fetch_begin_irq(&cpu_stats->syncp); + start = u64_stats_fetch_begin(&cpu_stats->syncp); rx_packets = cpu_stats->rx_packets; rx_bytes = cpu_stats->rx_bytes; tx_packets = cpu_stats->tx_packets; tx_bytes = cpu_stats->tx_bytes; - } while (u64_stats_fetch_retry_irq(&cpu_stats->syncp, start)); + } while (u64_stats_fetch_retry(&cpu_stats->syncp, start)); stats->rx_packets += rx_packets; stats->rx_bytes += rx_bytes; diff --git a/drivers/net/ethernet/ti/cpsw_priv.c b/drivers/net/ethernet/ti/cpsw_priv.c index 758295c898ac..e966dd47e2db 100644 --- a/drivers/net/ethernet/ti/cpsw_priv.c +++ b/drivers/net/ethernet/ti/cpsw_priv.c @@ -20,6 +20,7 @@ #include #include #include +#include #include "cpsw.h" #include "cpts.h" diff --git a/drivers/net/ethernet/ti/netcp_core.c b/drivers/net/ethernet/ti/netcp_core.c index 9eb9eaff4dc9..1bb596a9d8a2 100644 --- a/drivers/net/ethernet/ti/netcp_core.c +++ b/drivers/net/ethernet/ti/netcp_core.c @@ -1916,16 +1916,16 @@ netcp_get_stats(struct net_device *ndev, struct rtnl_link_stats64 *stats) unsigned int start; do { - start = u64_stats_fetch_begin_irq(&p->syncp_rx); + start = u64_stats_fetch_begin(&p->syncp_rx); rxpackets = p->rx_packets; rxbytes = p->rx_bytes; - } while (u64_stats_fetch_retry_irq(&p->syncp_rx, start)); + } while (u64_stats_fetch_retry(&p->syncp_rx, start)); do { - start = u64_stats_fetch_begin_irq(&p->syncp_tx); + start = u64_stats_fetch_begin(&p->syncp_tx); txpackets = p->tx_packets; txbytes = p->tx_bytes; - } while (u64_stats_fetch_retry_irq(&p->syncp_tx, start)); + } while (u64_stats_fetch_retry(&p->syncp_tx, start)); stats->rx_packets = rxpackets; stats->rx_bytes = rxbytes; diff --git a/drivers/net/ethernet/via/via-rhine.c b/drivers/net/ethernet/via/via-rhine.c index 0fb15a17b547..d716e6fe26e1 100644 --- a/drivers/net/ethernet/via/via-rhine.c +++ b/drivers/net/ethernet/via/via-rhine.c @@ -2217,16 +2217,16 @@ rhine_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) netdev_stats_to_stats64(stats, &dev->stats); do { - start = u64_stats_fetch_begin_irq(&rp->rx_stats.syncp); + start = u64_stats_fetch_begin(&rp->rx_stats.syncp); stats->rx_packets = rp->rx_stats.packets; stats->rx_bytes = rp->rx_stats.bytes; - } while (u64_stats_fetch_retry_irq(&rp->rx_stats.syncp, start)); + } while (u64_stats_fetch_retry(&rp->rx_stats.syncp, start)); do { - start = u64_stats_fetch_begin_irq(&rp->tx_stats.syncp); + start = u64_stats_fetch_begin(&rp->tx_stats.syncp); stats->tx_packets = rp->tx_stats.packets; stats->tx_bytes = rp->tx_stats.bytes; - } while (u64_stats_fetch_retry_irq(&rp->tx_stats.syncp, start)); + } while (u64_stats_fetch_retry(&rp->tx_stats.syncp, start)); } static void rhine_set_rx_mode(struct net_device *dev) diff --git a/drivers/net/ethernet/xilinx/xilinx_axienet_main.c b/drivers/net/ethernet/xilinx/xilinx_axienet_main.c index 420a73203ca1..ffc943a12762 100644 --- a/drivers/net/ethernet/xilinx/xilinx_axienet_main.c +++ b/drivers/net/ethernet/xilinx/xilinx_axienet_main.c @@ -1305,16 +1305,16 @@ axienet_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) netdev_stats_to_stats64(stats, &dev->stats); do { - start = u64_stats_fetch_begin_irq(&lp->rx_stat_sync); + start = u64_stats_fetch_begin(&lp->rx_stat_sync); stats->rx_packets = u64_stats_read(&lp->rx_packets); stats->rx_bytes = u64_stats_read(&lp->rx_bytes); - } while (u64_stats_fetch_retry_irq(&lp->rx_stat_sync, start)); + } while (u64_stats_fetch_retry(&lp->rx_stat_sync, start)); do { - start = u64_stats_fetch_begin_irq(&lp->tx_stat_sync); + start = u64_stats_fetch_begin(&lp->tx_stat_sync); stats->tx_packets = u64_stats_read(&lp->tx_packets); stats->tx_bytes = u64_stats_read(&lp->tx_bytes); - } while (u64_stats_fetch_retry_irq(&lp->tx_stat_sync, start)); + } while (u64_stats_fetch_retry(&lp->tx_stat_sync, start)); } static const struct net_device_ops axienet_netdev_ops = { diff --git a/drivers/net/hyperv/netvsc_drv.c b/drivers/net/hyperv/netvsc_drv.c index 89eb4f179a3c..f9b219e6cd58 100644 --- a/drivers/net/hyperv/netvsc_drv.c +++ b/drivers/net/hyperv/netvsc_drv.c @@ -1264,12 +1264,12 @@ static void netvsc_get_vf_stats(struct net_device *net, unsigned int start; do { - start = u64_stats_fetch_begin_irq(&stats->syncp); + start = u64_stats_fetch_begin(&stats->syncp); rx_packets = stats->rx_packets; tx_packets = stats->tx_packets; rx_bytes = stats->rx_bytes; tx_bytes = stats->tx_bytes; - } while (u64_stats_fetch_retry_irq(&stats->syncp, start)); + } while (u64_stats_fetch_retry(&stats->syncp, start)); tot->rx_packets += rx_packets; tot->tx_packets += tx_packets; @@ -1294,12 +1294,12 @@ static void netvsc_get_pcpu_stats(struct net_device *net, unsigned int start; do { - start = u64_stats_fetch_begin_irq(&stats->syncp); + start = u64_stats_fetch_begin(&stats->syncp); this_tot->vf_rx_packets = stats->rx_packets; this_tot->vf_tx_packets = stats->tx_packets; this_tot->vf_rx_bytes = stats->rx_bytes; this_tot->vf_tx_bytes = stats->tx_bytes; - } while (u64_stats_fetch_retry_irq(&stats->syncp, start)); + } while (u64_stats_fetch_retry(&stats->syncp, start)); this_tot->rx_packets = this_tot->vf_rx_packets; this_tot->tx_packets = this_tot->vf_tx_packets; this_tot->rx_bytes = this_tot->vf_rx_bytes; @@ -1318,20 +1318,20 @@ static void netvsc_get_pcpu_stats(struct net_device *net, tx_stats = &nvchan->tx_stats; do { - start = u64_stats_fetch_begin_irq(&tx_stats->syncp); + start = u64_stats_fetch_begin(&tx_stats->syncp); packets = tx_stats->packets; bytes = tx_stats->bytes; - } while (u64_stats_fetch_retry_irq(&tx_stats->syncp, start)); + } while (u64_stats_fetch_retry(&tx_stats->syncp, start)); this_tot->tx_bytes += bytes; this_tot->tx_packets += packets; rx_stats = &nvchan->rx_stats; do { - start = u64_stats_fetch_begin_irq(&rx_stats->syncp); + start = u64_stats_fetch_begin(&rx_stats->syncp); packets = rx_stats->packets; bytes = rx_stats->bytes; - } while (u64_stats_fetch_retry_irq(&rx_stats->syncp, start)); + } while (u64_stats_fetch_retry(&rx_stats->syncp, start)); this_tot->rx_bytes += bytes; this_tot->rx_packets += packets; @@ -1370,21 +1370,21 @@ static void netvsc_get_stats64(struct net_device *net, tx_stats = &nvchan->tx_stats; do { - start = u64_stats_fetch_begin_irq(&tx_stats->syncp); + start = u64_stats_fetch_begin(&tx_stats->syncp); packets = tx_stats->packets; bytes = tx_stats->bytes; - } while (u64_stats_fetch_retry_irq(&tx_stats->syncp, start)); + } while (u64_stats_fetch_retry(&tx_stats->syncp, start)); t->tx_bytes += bytes; t->tx_packets += packets; rx_stats = &nvchan->rx_stats; do { - start = u64_stats_fetch_begin_irq(&rx_stats->syncp); + start = u64_stats_fetch_begin(&rx_stats->syncp); packets = rx_stats->packets; bytes = rx_stats->bytes; multicast = rx_stats->multicast + rx_stats->broadcast; - } while (u64_stats_fetch_retry_irq(&rx_stats->syncp, start)); + } while (u64_stats_fetch_retry(&rx_stats->syncp, start)); t->rx_bytes += bytes; t->rx_packets += packets; @@ -1527,24 +1527,24 @@ static void netvsc_get_ethtool_stats(struct net_device *dev, tx_stats = &nvdev->chan_table[j].tx_stats; do { - start = u64_stats_fetch_begin_irq(&tx_stats->syncp); + start = u64_stats_fetch_begin(&tx_stats->syncp); packets = tx_stats->packets; bytes = tx_stats->bytes; xdp_xmit = tx_stats->xdp_xmit; - } while (u64_stats_fetch_retry_irq(&tx_stats->syncp, start)); + } while (u64_stats_fetch_retry(&tx_stats->syncp, start)); data[i++] = packets; data[i++] = bytes; data[i++] = xdp_xmit; rx_stats = &nvdev->chan_table[j].rx_stats; do { - start = u64_stats_fetch_begin_irq(&rx_stats->syncp); + start = u64_stats_fetch_begin(&rx_stats->syncp); packets = rx_stats->packets; bytes = rx_stats->bytes; xdp_drop = rx_stats->xdp_drop; xdp_redirect = rx_stats->xdp_redirect; xdp_tx = rx_stats->xdp_tx; - } while (u64_stats_fetch_retry_irq(&rx_stats->syncp, start)); + } while (u64_stats_fetch_retry(&rx_stats->syncp, start)); data[i++] = packets; data[i++] = bytes; data[i++] = xdp_drop; diff --git a/drivers/net/ifb.c b/drivers/net/ifb.c index 1c64d5347b8e..78253ad57b2e 100644 --- a/drivers/net/ifb.c +++ b/drivers/net/ifb.c @@ -162,18 +162,18 @@ static void ifb_stats64(struct net_device *dev, for (i = 0; i < dev->num_tx_queues; i++,txp++) { do { - start = u64_stats_fetch_begin_irq(&txp->rx_stats.sync); + start = u64_stats_fetch_begin(&txp->rx_stats.sync); packets = txp->rx_stats.packets; bytes = txp->rx_stats.bytes; - } while (u64_stats_fetch_retry_irq(&txp->rx_stats.sync, start)); + } while (u64_stats_fetch_retry(&txp->rx_stats.sync, start)); stats->rx_packets += packets; stats->rx_bytes += bytes; do { - start = u64_stats_fetch_begin_irq(&txp->tx_stats.sync); + start = u64_stats_fetch_begin(&txp->tx_stats.sync); packets = txp->tx_stats.packets; bytes = txp->tx_stats.bytes; - } while (u64_stats_fetch_retry_irq(&txp->tx_stats.sync, start)); + } while (u64_stats_fetch_retry(&txp->tx_stats.sync, start)); stats->tx_packets += packets; stats->tx_bytes += bytes; } @@ -245,12 +245,12 @@ static void ifb_fill_stats_data(u64 **data, int j; do { - start = u64_stats_fetch_begin_irq(&q_stats->sync); + start = u64_stats_fetch_begin(&q_stats->sync); for (j = 0; j < IFB_Q_STATS_LEN; j++) { offset = ifb_q_stats_desc[j].offset; (*data)[j] = *(u64 *)(stats_base + offset); } - } while (u64_stats_fetch_retry_irq(&q_stats->sync, start)); + } while (u64_stats_fetch_retry(&q_stats->sync, start)); *data += IFB_Q_STATS_LEN; } diff --git a/drivers/net/ipvlan/ipvlan_main.c b/drivers/net/ipvlan/ipvlan_main.c index cd16bc8bf154..1b55928e89b8 100644 --- a/drivers/net/ipvlan/ipvlan_main.c +++ b/drivers/net/ipvlan/ipvlan_main.c @@ -301,13 +301,13 @@ static void ipvlan_get_stats64(struct net_device *dev, for_each_possible_cpu(idx) { pcptr = per_cpu_ptr(ipvlan->pcpu_stats, idx); do { - strt= u64_stats_fetch_begin_irq(&pcptr->syncp); + strt = u64_stats_fetch_begin(&pcptr->syncp); rx_pkts = u64_stats_read(&pcptr->rx_pkts); rx_bytes = u64_stats_read(&pcptr->rx_bytes); rx_mcast = u64_stats_read(&pcptr->rx_mcast); tx_pkts = u64_stats_read(&pcptr->tx_pkts); tx_bytes = u64_stats_read(&pcptr->tx_bytes); - } while (u64_stats_fetch_retry_irq(&pcptr->syncp, + } while (u64_stats_fetch_retry(&pcptr->syncp, strt)); s->rx_packets += rx_pkts; diff --git a/drivers/net/loopback.c b/drivers/net/loopback.c index 2e9742952c4e..f6d53e63ef4e 100644 --- a/drivers/net/loopback.c +++ b/drivers/net/loopback.c @@ -106,10 +106,10 @@ void dev_lstats_read(struct net_device *dev, u64 *packets, u64 *bytes) lb_stats = per_cpu_ptr(dev->lstats, i); do { - start = u64_stats_fetch_begin_irq(&lb_stats->syncp); + start = u64_stats_fetch_begin(&lb_stats->syncp); tpackets = u64_stats_read(&lb_stats->packets); tbytes = u64_stats_read(&lb_stats->bytes); - } while (u64_stats_fetch_retry_irq(&lb_stats->syncp, start)); + } while (u64_stats_fetch_retry(&lb_stats->syncp, start)); *bytes += tbytes; *packets += tpackets; } diff --git a/drivers/net/macsec.c b/drivers/net/macsec.c index 578f470e9fad..2de422ab3e6a 100644 --- a/drivers/net/macsec.c +++ b/drivers/net/macsec.c @@ -2800,9 +2800,9 @@ static void get_rx_sc_stats(struct net_device *dev, stats = per_cpu_ptr(rx_sc->stats, cpu); do { - start = u64_stats_fetch_begin_irq(&stats->syncp); + start = u64_stats_fetch_begin(&stats->syncp); memcpy(&tmp, &stats->stats, sizeof(tmp)); - } while (u64_stats_fetch_retry_irq(&stats->syncp, start)); + } while (u64_stats_fetch_retry(&stats->syncp, start)); sum->InOctetsValidated += tmp.InOctetsValidated; sum->InOctetsDecrypted += tmp.InOctetsDecrypted; @@ -2881,9 +2881,9 @@ static void get_tx_sc_stats(struct net_device *dev, stats = per_cpu_ptr(macsec_priv(dev)->secy.tx_sc.stats, cpu); do { - start = u64_stats_fetch_begin_irq(&stats->syncp); + start = u64_stats_fetch_begin(&stats->syncp); memcpy(&tmp, &stats->stats, sizeof(tmp)); - } while (u64_stats_fetch_retry_irq(&stats->syncp, start)); + } while (u64_stats_fetch_retry(&stats->syncp, start)); sum->OutPktsProtected += tmp.OutPktsProtected; sum->OutPktsEncrypted += tmp.OutPktsEncrypted; @@ -2937,9 +2937,9 @@ static void get_secy_stats(struct net_device *dev, struct macsec_dev_stats *sum) stats = per_cpu_ptr(macsec_priv(dev)->stats, cpu); do { - start = u64_stats_fetch_begin_irq(&stats->syncp); + start = u64_stats_fetch_begin(&stats->syncp); memcpy(&tmp, &stats->stats, sizeof(tmp)); - } while (u64_stats_fetch_retry_irq(&stats->syncp, start)); + } while (u64_stats_fetch_retry(&stats->syncp, start)); sum->OutPktsUntagged += tmp.OutPktsUntagged; sum->InPktsUntagged += tmp.InPktsUntagged; diff --git a/drivers/net/macvlan.c b/drivers/net/macvlan.c index b8cc55b2d721..99a971929c8e 100644 --- a/drivers/net/macvlan.c +++ b/drivers/net/macvlan.c @@ -948,13 +948,13 @@ static void macvlan_dev_get_stats64(struct net_device *dev, for_each_possible_cpu(i) { p = per_cpu_ptr(vlan->pcpu_stats, i); do { - start = u64_stats_fetch_begin_irq(&p->syncp); + start = u64_stats_fetch_begin(&p->syncp); rx_packets = u64_stats_read(&p->rx_packets); rx_bytes = u64_stats_read(&p->rx_bytes); rx_multicast = u64_stats_read(&p->rx_multicast); tx_packets = u64_stats_read(&p->tx_packets); tx_bytes = u64_stats_read(&p->tx_bytes); - } while (u64_stats_fetch_retry_irq(&p->syncp, start)); + } while (u64_stats_fetch_retry(&p->syncp, start)); stats->rx_packets += rx_packets; stats->rx_bytes += rx_bytes; diff --git a/drivers/net/mhi_net.c b/drivers/net/mhi_net.c index 0b9d37979133..3d322ac4f6a5 100644 --- a/drivers/net/mhi_net.c +++ b/drivers/net/mhi_net.c @@ -104,19 +104,19 @@ static void mhi_ndo_get_stats64(struct net_device *ndev, unsigned int start; do { - start = u64_stats_fetch_begin_irq(&mhi_netdev->stats.rx_syncp); + start = u64_stats_fetch_begin(&mhi_netdev->stats.rx_syncp); stats->rx_packets = u64_stats_read(&mhi_netdev->stats.rx_packets); stats->rx_bytes = u64_stats_read(&mhi_netdev->stats.rx_bytes); stats->rx_errors = u64_stats_read(&mhi_netdev->stats.rx_errors); - } while (u64_stats_fetch_retry_irq(&mhi_netdev->stats.rx_syncp, start)); + } while (u64_stats_fetch_retry(&mhi_netdev->stats.rx_syncp, start)); do { - start = u64_stats_fetch_begin_irq(&mhi_netdev->stats.tx_syncp); + start = u64_stats_fetch_begin(&mhi_netdev->stats.tx_syncp); stats->tx_packets = u64_stats_read(&mhi_netdev->stats.tx_packets); stats->tx_bytes = u64_stats_read(&mhi_netdev->stats.tx_bytes); stats->tx_errors = u64_stats_read(&mhi_netdev->stats.tx_errors); stats->tx_dropped = u64_stats_read(&mhi_netdev->stats.tx_dropped); - } while (u64_stats_fetch_retry_irq(&mhi_netdev->stats.tx_syncp, start)); + } while (u64_stats_fetch_retry(&mhi_netdev->stats.tx_syncp, start)); } static const struct net_device_ops mhi_netdev_ops = { diff --git a/drivers/net/netdevsim/netdev.c b/drivers/net/netdevsim/netdev.c index 9a1a5b203624..e470e3398abc 100644 --- a/drivers/net/netdevsim/netdev.c +++ b/drivers/net/netdevsim/netdev.c @@ -67,10 +67,10 @@ nsim_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) unsigned int start; do { - start = u64_stats_fetch_begin_irq(&ns->syncp); + start = u64_stats_fetch_begin(&ns->syncp); stats->tx_bytes = ns->tx_bytes; stats->tx_packets = ns->tx_packets; - } while (u64_stats_fetch_retry_irq(&ns->syncp, start)); + } while (u64_stats_fetch_retry(&ns->syncp, start)); } static int diff --git a/drivers/net/phy/phy.c b/drivers/net/phy/phy.c index 95f07cf7b37c..1c72733f6824 100644 --- a/drivers/net/phy/phy.c +++ b/drivers/net/phy/phy.c @@ -36,7 +36,8 @@ #include #include -#define PHY_STATE_TIME HZ +#define PHY_STATE_TIME HZ +#define PHY_STATE_TIME_MS 100 #define PHY_STATE_STR(_state) \ case PHY_##_state: \ @@ -1281,7 +1282,7 @@ void phy_state_machine(struct work_struct *work) */ mutex_lock(&phydev->lock); if (phy_polling_mode(phydev) && phy_is_started(phydev)) - phy_queue_state_machine(phydev, PHY_STATE_TIME); + phy_queue_state_machine(phydev, (PHY_STATE_TIME_MS * HZ) / 1000); mutex_unlock(&phydev->lock); } diff --git a/drivers/net/team/team.c b/drivers/net/team/team.c index 921ca59822b0..382756c3fb83 100644 --- a/drivers/net/team/team.c +++ b/drivers/net/team/team.c @@ -1866,13 +1866,13 @@ team_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) for_each_possible_cpu(i) { p = per_cpu_ptr(team->pcpu_stats, i); do { - start = u64_stats_fetch_begin_irq(&p->syncp); + start = u64_stats_fetch_begin(&p->syncp); rx_packets = u64_stats_read(&p->rx_packets); rx_bytes = u64_stats_read(&p->rx_bytes); rx_multicast = u64_stats_read(&p->rx_multicast); tx_packets = u64_stats_read(&p->tx_packets); tx_bytes = u64_stats_read(&p->tx_bytes); - } while (u64_stats_fetch_retry_irq(&p->syncp, start)); + } while (u64_stats_fetch_retry(&p->syncp, start)); stats->rx_packets += rx_packets; stats->rx_bytes += rx_bytes; diff --git a/drivers/net/team/team_mode_loadbalance.c b/drivers/net/team/team_mode_loadbalance.c index b095a4b4957b..18d99fda997c 100644 --- a/drivers/net/team/team_mode_loadbalance.c +++ b/drivers/net/team/team_mode_loadbalance.c @@ -466,9 +466,9 @@ static void __lb_one_cpu_stats_add(struct lb_stats *acc_stats, struct lb_stats tmp; do { - start = u64_stats_fetch_begin_irq(syncp); + start = u64_stats_fetch_begin(syncp); tmp.tx_bytes = cpu_stats->tx_bytes; - } while (u64_stats_fetch_retry_irq(syncp, start)); + } while (u64_stats_fetch_retry(syncp, start)); acc_stats->tx_bytes += tmp.tx_bytes; } diff --git a/drivers/net/veth.c b/drivers/net/veth.c index 36c5a41f84e4..605f511a886c 100644 --- a/drivers/net/veth.c +++ b/drivers/net/veth.c @@ -182,12 +182,12 @@ static void veth_get_ethtool_stats(struct net_device *dev, size_t offset; do { - start = u64_stats_fetch_begin_irq(&rq_stats->syncp); + start = u64_stats_fetch_begin(&rq_stats->syncp); for (j = 0; j < VETH_RQ_STATS_LEN; j++) { offset = veth_rq_stats_desc[j].offset; data[idx + j] = *(u64 *)(stats_base + offset); } - } while (u64_stats_fetch_retry_irq(&rq_stats->syncp, start)); + } while (u64_stats_fetch_retry(&rq_stats->syncp, start)); idx += VETH_RQ_STATS_LEN; } @@ -203,12 +203,12 @@ static void veth_get_ethtool_stats(struct net_device *dev, tx_idx += (i % dev->real_num_tx_queues) * VETH_TQ_STATS_LEN; do { - start = u64_stats_fetch_begin_irq(&rq_stats->syncp); + start = u64_stats_fetch_begin(&rq_stats->syncp); for (j = 0; j < VETH_TQ_STATS_LEN; j++) { offset = veth_tq_stats_desc[j].offset; data[tx_idx + j] += *(u64 *)(base + offset); } - } while (u64_stats_fetch_retry_irq(&rq_stats->syncp, start)); + } while (u64_stats_fetch_retry(&rq_stats->syncp, start)); } } @@ -381,13 +381,13 @@ static void veth_stats_rx(struct veth_stats *result, struct net_device *dev) unsigned int start; do { - start = u64_stats_fetch_begin_irq(&stats->syncp); + start = u64_stats_fetch_begin(&stats->syncp); peer_tq_xdp_xmit_err = stats->vs.peer_tq_xdp_xmit_err; xdp_tx_err = stats->vs.xdp_tx_err; packets = stats->vs.xdp_packets; bytes = stats->vs.xdp_bytes; drops = stats->vs.rx_drops; - } while (u64_stats_fetch_retry_irq(&stats->syncp, start)); + } while (u64_stats_fetch_retry(&stats->syncp, start)); result->peer_tq_xdp_xmit_err += peer_tq_xdp_xmit_err; result->xdp_tx_err += xdp_tx_err; result->xdp_packets += packets; diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c index 21d3461fb5d1..666622ae4b9d 100644 --- a/drivers/net/virtio_net.c +++ b/drivers/net/virtio_net.c @@ -2107,18 +2107,18 @@ static void virtnet_stats(struct net_device *dev, struct send_queue *sq = &vi->sq[i]; do { - start = u64_stats_fetch_begin_irq(&sq->stats.syncp); + start = u64_stats_fetch_begin(&sq->stats.syncp); tpackets = sq->stats.packets; tbytes = sq->stats.bytes; terrors = sq->stats.tx_timeouts; - } while (u64_stats_fetch_retry_irq(&sq->stats.syncp, start)); + } while (u64_stats_fetch_retry(&sq->stats.syncp, start)); do { - start = u64_stats_fetch_begin_irq(&rq->stats.syncp); + start = u64_stats_fetch_begin(&rq->stats.syncp); rpackets = rq->stats.packets; rbytes = rq->stats.bytes; rdrops = rq->stats.drops; - } while (u64_stats_fetch_retry_irq(&rq->stats.syncp, start)); + } while (u64_stats_fetch_retry(&rq->stats.syncp, start)); tot->rx_packets += rpackets; tot->tx_packets += tpackets; @@ -2726,12 +2726,12 @@ static void virtnet_get_ethtool_stats(struct net_device *dev, stats_base = (u8 *)&rq->stats; do { - start = u64_stats_fetch_begin_irq(&rq->stats.syncp); + start = u64_stats_fetch_begin(&rq->stats.syncp); for (j = 0; j < VIRTNET_RQ_STATS_LEN; j++) { offset = virtnet_rq_stats_desc[j].offset; data[idx + j] = *(u64 *)(stats_base + offset); } - } while (u64_stats_fetch_retry_irq(&rq->stats.syncp, start)); + } while (u64_stats_fetch_retry(&rq->stats.syncp, start)); idx += VIRTNET_RQ_STATS_LEN; } @@ -2740,12 +2740,12 @@ static void virtnet_get_ethtool_stats(struct net_device *dev, stats_base = (u8 *)&sq->stats; do { - start = u64_stats_fetch_begin_irq(&sq->stats.syncp); + start = u64_stats_fetch_begin(&sq->stats.syncp); for (j = 0; j < VIRTNET_SQ_STATS_LEN; j++) { offset = virtnet_sq_stats_desc[j].offset; data[idx + j] = *(u64 *)(stats_base + offset); } - } while (u64_stats_fetch_retry_irq(&sq->stats.syncp, start)); + } while (u64_stats_fetch_retry(&sq->stats.syncp, start)); idx += VIRTNET_SQ_STATS_LEN; } } diff --git a/drivers/net/vrf.c b/drivers/net/vrf.c index f6dcec66f0a4..bdb3a76a352e 100644 --- a/drivers/net/vrf.c +++ b/drivers/net/vrf.c @@ -159,13 +159,13 @@ static void vrf_get_stats64(struct net_device *dev, dstats = per_cpu_ptr(dev->dstats, i); do { - start = u64_stats_fetch_begin_irq(&dstats->syncp); + start = u64_stats_fetch_begin(&dstats->syncp); tbytes = dstats->tx_bytes; tpkts = dstats->tx_pkts; tdrops = dstats->tx_drps; rbytes = dstats->rx_bytes; rpkts = dstats->rx_pkts; - } while (u64_stats_fetch_retry_irq(&dstats->syncp, start)); + } while (u64_stats_fetch_retry(&dstats->syncp, start)); stats->tx_bytes += tbytes; stats->tx_packets += tpkts; stats->tx_dropped += tdrops; diff --git a/drivers/net/vxlan/vxlan_vnifilter.c b/drivers/net/vxlan/vxlan_vnifilter.c index c5cf55030158..c3ff30ab782e 100644 --- a/drivers/net/vxlan/vxlan_vnifilter.c +++ b/drivers/net/vxlan/vxlan_vnifilter.c @@ -129,9 +129,9 @@ static void vxlan_vnifilter_stats_get(const struct vxlan_vni_node *vninode, pstats = per_cpu_ptr(vninode->stats, i); do { - start = u64_stats_fetch_begin_irq(&pstats->syncp); + start = u64_stats_fetch_begin(&pstats->syncp); memcpy(&temp, &pstats->stats, sizeof(temp)); - } while (u64_stats_fetch_retry_irq(&pstats->syncp, start)); + } while (u64_stats_fetch_retry(&pstats->syncp, start)); dest->rx_packets += temp.rx_packets; dest->rx_bytes += temp.rx_bytes; diff --git a/drivers/net/wwan/mhi_wwan_mbim.c b/drivers/net/wwan/mhi_wwan_mbim.c index ef70bb7c88ad..3f72ae943b29 100644 --- a/drivers/net/wwan/mhi_wwan_mbim.c +++ b/drivers/net/wwan/mhi_wwan_mbim.c @@ -456,19 +456,19 @@ static void mhi_mbim_ndo_get_stats64(struct net_device *ndev, unsigned int start; do { - start = u64_stats_fetch_begin_irq(&link->rx_syncp); + start = u64_stats_fetch_begin(&link->rx_syncp); stats->rx_packets = u64_stats_read(&link->rx_packets); stats->rx_bytes = u64_stats_read(&link->rx_bytes); stats->rx_errors = u64_stats_read(&link->rx_errors); - } while (u64_stats_fetch_retry_irq(&link->rx_syncp, start)); + } while (u64_stats_fetch_retry(&link->rx_syncp, start)); do { - start = u64_stats_fetch_begin_irq(&link->tx_syncp); + start = u64_stats_fetch_begin(&link->tx_syncp); stats->tx_packets = u64_stats_read(&link->tx_packets); stats->tx_bytes = u64_stats_read(&link->tx_bytes); stats->tx_errors = u64_stats_read(&link->tx_errors); stats->tx_dropped = u64_stats_read(&link->tx_dropped); - } while (u64_stats_fetch_retry_irq(&link->tx_syncp, start)); + } while (u64_stats_fetch_retry(&link->tx_syncp, start)); } static void mhi_mbim_ul_callback(struct mhi_device *mhi_dev, diff --git a/drivers/net/xen-netfront.c b/drivers/net/xen-netfront.c index dc404e05970c..14aec417fa06 100644 --- a/drivers/net/xen-netfront.c +++ b/drivers/net/xen-netfront.c @@ -1392,16 +1392,16 @@ static void xennet_get_stats64(struct net_device *dev, unsigned int start; do { - start = u64_stats_fetch_begin_irq(&tx_stats->syncp); + start = u64_stats_fetch_begin(&tx_stats->syncp); tx_packets = tx_stats->packets; tx_bytes = tx_stats->bytes; - } while (u64_stats_fetch_retry_irq(&tx_stats->syncp, start)); + } while (u64_stats_fetch_retry(&tx_stats->syncp, start)); do { - start = u64_stats_fetch_begin_irq(&rx_stats->syncp); + start = u64_stats_fetch_begin(&rx_stats->syncp); rx_packets = rx_stats->packets; rx_bytes = rx_stats->bytes; - } while (u64_stats_fetch_retry_irq(&rx_stats->syncp, start)); + } while (u64_stats_fetch_retry(&rx_stats->syncp, start)); tot->rx_packets += rx_packets; tot->tx_packets += tx_packets; diff --git a/drivers/nfc/Kconfig b/drivers/nfc/Kconfig index 288c6f1c6979..64d3846b93be 100644 --- a/drivers/nfc/Kconfig +++ b/drivers/nfc/Kconfig @@ -60,6 +60,7 @@ config NFC_VIRTUAL_NCI If unsure, say N. +source "drivers/nfc/pn5xx/Kconfig" source "drivers/nfc/fdp/Kconfig" source "drivers/nfc/pn544/Kconfig" source "drivers/nfc/pn533/Kconfig" diff --git a/drivers/nfc/Makefile b/drivers/nfc/Makefile index 7b1bfde1d971..62d495c57a15 100644 --- a/drivers/nfc/Makefile +++ b/drivers/nfc/Makefile @@ -3,6 +3,7 @@ # Makefile for nfc devices # +obj-$(CONFIG_NFC_NXP_PN5XX) += pn5xx/ obj-$(CONFIG_NFC_FDP) += fdp/ obj-$(CONFIG_NFC_PN544) += pn544/ obj-$(CONFIG_NFC_MICROREAD) += microread/ diff --git a/drivers/nfc/pn5xx/Kconfig b/drivers/nfc/pn5xx/Kconfig new file mode 100644 index 000000000000..10db8c403866 --- /dev/null +++ b/drivers/nfc/pn5xx/Kconfig @@ -0,0 +1,12 @@ +config NFC_NXP_PN5XX + tristate "NXP PN5XX based driver" + depends on I2C + select CRC_CCITT + help + NXP PN5XX driver based on I2C. + This is a driver to provides I2C access to PN5xx and PN7120 NFC + Controller devices + + To compile this driver as a module, choose m here. The module will + be called pn5xx_i2c. + Say N if unsure. diff --git a/drivers/nfc/pn5xx/Makefile b/drivers/nfc/pn5xx/Makefile new file mode 100644 index 000000000000..b31bbb51423a --- /dev/null +++ b/drivers/nfc/pn5xx/Makefile @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Makefile for PN5xx NFC driver +# + +obj-$(CONFIG_NFC_NXP_PN5XX) += pn5xx_i2c.o diff --git a/drivers/nfc/pn5xx/README.md b/drivers/nfc/pn5xx/README.md new file mode 100644 index 000000000000..c0c8d5c73757 --- /dev/null +++ b/drivers/nfc/pn5xx/README.md @@ -0,0 +1,2 @@ +# nxp-pn5xx +NXP's NFC Open Source Kernel mode driver diff --git a/drivers/nfc/pn5xx/pn5xx_i2c.c b/drivers/nfc/pn5xx/pn5xx_i2c.c new file mode 100644 index 000000000000..3511f1441cf6 --- /dev/null +++ b/drivers/nfc/pn5xx/pn5xx_i2c.c @@ -0,0 +1,672 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright (C) 2010 Trusted Logic S.A. + * Copyright 2015,2019-2023 NXP + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "pn5xx_i2c.h" +#include +#include +#include +#include + +#include + +#define MAX_BUFFER_SIZE 512 + +#define MODE_OFF 0 +#define MODE_RUN 1 +#define MODE_FW 2 + +/* pn7120, pn548, pn547 and pn544 are supported */ +#define CHIP "pn544" +#define DRIVER_CARD "PN54x NFC" +#define DRIVER_DESC "NFC driver for PN54x Family" + +#ifndef CONFIG_OF +#define CONFIG_OF +#endif + +struct pn54x_dev { + wait_queue_head_t read_wq; + struct mutex read_mutex; + struct i2c_client *client; + struct miscdevice pn54x_device; + int ven_gpio; + int firm_gpio; + int irq_gpio; + int clkreq_gpio; + struct regulator *pvdd_reg; + struct regulator *vbat_reg; + struct regulator *pmuvcc_reg; + struct regulator *sevdd_reg; + bool irq_enabled; + spinlock_t irq_enabled_lock; +}; + +static struct task_struct *test_task; +/* Interrupt control and handler */ +static void pn54x_disable_irq(struct pn54x_dev *pn54x_dev) +{ + unsigned long flags; + + spin_lock_irqsave(&pn54x_dev->irq_enabled_lock, flags); + if (pn54x_dev->irq_enabled) { + disable_irq_nosync(pn54x_dev->client->irq); + pn54x_dev->irq_enabled = false; + } + spin_unlock_irqrestore(&pn54x_dev->irq_enabled_lock, flags); +} + +static irqreturn_t pn54x_dev_irq_handler(int irq, void *dev_id) +{ + struct pn54x_dev *pn54x_dev = dev_id; + + pn54x_disable_irq(pn54x_dev); + + /* Wake up waiting readers */ + wake_up(&pn54x_dev->read_wq); + + return IRQ_HANDLED; +} + +static struct pn54x_dev *gpn54x_dev = NULL; +static int pn54x_dev_loop_handler(void *data) +{ + struct pn54x_dev *pn54x_dev = (struct pn54x_dev *)gpn54x_dev; + + printk("In pn54x_dev_loop_handler...%x \n", data); + while (1) { + /* Wake up waiting readers */ + wake_up(&pn54x_dev->read_wq); + msleep(200); + } + + return 0; +} + +/* private functions */ +static int pn544_enable(struct pn54x_dev *dev, int mode) +{ + int r; + + /* + * turn on the regulators + * -- if the regulators were specified, they're required + */ + if (dev->pvdd_reg != NULL) { + r = regulator_enable(dev->pvdd_reg); + if (r < 0) { + pr_err("%s: not able to enable pvdd\n", __func__); + return r; + } + } + if (dev->vbat_reg != NULL) { + r = regulator_enable(dev->vbat_reg); + if (r < 0) { + pr_err("%s: not able to enable vbat\n", __func__); + goto enable_exit0; + } + } + if (dev->pmuvcc_reg != NULL) { + r = regulator_enable(dev->pmuvcc_reg); + if (r < 0) { + pr_err("%s: not able to enable pmuvcc\n", __func__); + goto enable_exit1; + } + } + if (dev->sevdd_reg != NULL) { + r = regulator_enable(dev->sevdd_reg); + if (r < 0) { + pr_err("%s: not able to enable sevdd\n", __func__); + goto enable_exit2; + } + } + + if (MODE_RUN == mode) { + printk("%s power on\n", __func__); + if (gpio_is_valid(dev->firm_gpio)) + gpio_set_value_cansleep(dev->firm_gpio, 0); + msleep(100); + } else if (MODE_FW == mode) { + /* power on with firmware download (requires hw reset) */ + pr_info("%s power on with firmware\n", __func__); + msleep(20); + if (gpio_is_valid(dev->firm_gpio)) { + gpio_set_value(dev->firm_gpio, 1); + } else { + pr_err("%s Unused Firm GPIO %d\n", __func__, mode); + return GPIO_UNUSED; + } + msleep(20); + msleep(100); + msleep(20); + } else { + pr_err("%s bad arg %d\n", __func__, mode); + return -EINVAL; + } + + return 0; + +enable_exit2: + if (dev->pmuvcc_reg) + regulator_disable(dev->pmuvcc_reg); +enable_exit1: + if (dev->vbat_reg) + regulator_disable(dev->vbat_reg); +enable_exit0: + if (dev->pvdd_reg) + regulator_disable(dev->pvdd_reg); + + return r; +} + +static void pn544_disable(struct pn54x_dev *dev) +{ + /* power off */ + printk("%s power off\n", __func__); + if (gpio_is_valid(dev->firm_gpio)) + gpio_set_value_cansleep(dev->firm_gpio, 0); + msleep(100); + + if (dev->sevdd_reg) + regulator_disable(dev->sevdd_reg); + if (dev->pmuvcc_reg) + regulator_disable(dev->pmuvcc_reg); + if (dev->vbat_reg) + regulator_disable(dev->vbat_reg); + if (dev->pvdd_reg) + regulator_disable(dev->pvdd_reg); +} + +/* driver functions */ +static ssize_t pn54x_dev_read(struct file *filp, char __user *buf, + size_t count, loff_t *offset) +{ + struct pn54x_dev *pn54x_dev = filp->private_data; + char tmp[MAX_BUFFER_SIZE]; + int ret; + + if (count > MAX_BUFFER_SIZE) + count = MAX_BUFFER_SIZE; + + mutex_lock(&pn54x_dev->read_mutex); + + /* Read data */ + ret = i2c_master_recv(pn54x_dev->client, tmp, count); + + mutex_unlock(&pn54x_dev->read_mutex); + + /* + * pn54x seems to be slow in handling I2C read requests + * so add 1ms delay after recv operation + */ + udelay(1000); + + if (ret < 0) + return ret; + + if (ret > count) { + pr_err("%s: received too many bytes from i2c (%d)\n", + __func__, ret); + return -EIO; + } + if (copy_to_user(buf, tmp, ret)) { + pr_err("%s : failed to copy to user space\n", __func__); + return -EFAULT; + } + return ret; + +fail: + mutex_unlock(&pn54x_dev->read_mutex); + return ret; +} + +static ssize_t pn54x_dev_write(struct file *filp, const char __user *buf, + size_t count, loff_t *offset) +{ + struct pn54x_dev *pn54x_dev; + char tmp[MAX_BUFFER_SIZE]; + int ret; + + pn54x_dev = filp->private_data; + + if (count > MAX_BUFFER_SIZE) + count = MAX_BUFFER_SIZE; + + if (copy_from_user(tmp, buf, count)) { + pr_err("%s : failed to copy from user space\n", __func__); + return -EFAULT; + } + + pr_debug("%s : writing %zu bytes.\n", __func__, count); + /* Write data */ + ret = i2c_master_send(pn54x_dev->client, tmp, count); + if (ret != count) { + pr_err("%s : i2c_master_send returned %d\n", __func__, ret); + pr_err("I2C addr is 0x%02X, name is %s\n", + pn54x_dev->client->addr, pn54x_dev->client->name); + ret = -EIO; + } + + /* + * pn54x seems to be slow in handling I2C write requests + * so add 1ms delay after I2C send oparation + */ + udelay(1000); + + return ret; +} + +static int pn54x_dev_open(struct inode *inode, struct file *filp) +{ + struct pn54x_dev *pn54x_dev = container_of(filp->private_data, + struct pn54x_dev, + pn54x_device); + + filp->private_data = pn54x_dev; + + pr_info("%s : %d,%d\n", __func__, imajor(inode), iminor(inode)); + + return 0; +} + +static int pn54x_dev_release(struct inode *inode, struct file *filp) +{ + pr_info("%s : closing %d,%d\n", __func__, imajor(inode), iminor(inode)); + + return 0; +} + +static long pn54x_dev_ioctl(struct file *filp, unsigned int cmd, + unsigned long arg) +{ + struct pn54x_dev *pn54x_dev = filp->private_data; + + pr_info("%s, cmd=%d, arg=%lu\n", __func__, cmd, arg); + switch (cmd) { + case PN544_SET_PWR: + if (arg == 2) { + /* power on w/FW */ + if (pn544_enable(pn54x_dev, arg) == GPIO_UNUSED) + return GPIO_UNUSED; + } else if (arg == 1) { + /* power on */ + pn544_enable(pn54x_dev, arg); + } else if (arg == 0) { + /* power off */ + pn544_disable(pn54x_dev); + } else { + pr_err("%s bad SET_PWR arg %lu\n", __func__, arg); + return -EINVAL; + } + break; + case PN54X_CLK_REQ: + if (arg == 1) { + if (gpio_is_valid(pn54x_dev->clkreq_gpio)) { + gpio_set_value(pn54x_dev->clkreq_gpio, 1); + } else { + pr_err("%s Unused Clkreq GPIO %lu\n", + __func__, arg); + return GPIO_UNUSED; + } + } else if (arg == 0) { + if (gpio_is_valid(pn54x_dev->clkreq_gpio)) { + gpio_set_value(pn54x_dev->clkreq_gpio, 0); + } else { + pr_err("%s Unused Clkreq GPIO %lu\n", + __func__, arg); + return GPIO_UNUSED; + } + } else { + pr_err("%s bad CLK_REQ arg %lu\n", __func__, arg); + return -EINVAL; + } + break; + default: + pr_err("%s bad ioctl %u\n", __func__, cmd); + return -EINVAL; + } + + return 0; +} + +static const struct file_operations pn54x_dev_fops = { + .owner = THIS_MODULE, + .llseek = no_llseek, + .read = pn54x_dev_read, + .write = pn54x_dev_write, + .open = pn54x_dev_open, + .release = pn54x_dev_release, + .unlocked_ioctl = pn54x_dev_ioctl, +}; + +/* Handlers for alternative sources of platform_data */ +#ifdef CONFIG_OF +/* Translate OpenFirmware node properties into platform_data */ +static int pn54x_get_pdata(struct device *dev, + struct pn544_i2c_platform_data *pdata) +{ + struct device_node *node; + u32 flags; + int val; + + /* make sure there is actually a device tree node */ + node = dev->of_node; + if (!node) + return -ENODEV; + + memset(pdata, 0, sizeof(*pdata)); + + /* + * read the dev tree data + * firm pin - controls firmware download - OPTIONAL + */ + val = of_get_named_gpio_flags(node, "firmware-gpios", 0, &flags); + if (val >= 0) { + pdata->firm_gpio = val; + } else { + pdata->firm_gpio = GPIO_UNUSED; + dev_warn(dev, "FIRM GPIO error getting from OF node\n"); + } + + /* clkreq pin - controls the clock to the PN547 - OPTIONAL */ + val = of_get_named_gpio_flags(node, "nxp,pn54x-clkreq", 0, &flags); + if (val >= 0) { + pdata->clkreq_gpio = val; + } else { + pdata->clkreq_gpio = GPIO_UNUSED; + dev_warn(dev, + "CLKREQ GPIO error getting from OF node\n"); + } + + /* + * handle the regulator lines - these are optional + * PVdd - pad Vdd (544, 547) + * Vbat - Battery (544, 547) + * PMUVcc - UICC Power (544, 547) + * SEVdd - SE Power (544) + * + * Will attempt to load a matching Regulator Resource for each + * If no resource is provided, then the input will not be controlled + * Example: if only PVdd is provided, it is the only one that will be + * turned on/off. + */ + pdata->pvdd_reg = regulator_get(dev, "nxp,pn54x-pvdd"); + if (IS_ERR(pdata->pvdd_reg)) { + pr_err("%s: could not get nxp,pn54x-pvdd, rc=%ld\n", + __func__, PTR_ERR(pdata->pvdd_reg)); + pdata->pvdd_reg = NULL; + } + + pdata->vbat_reg = regulator_get(dev, "nxp,pn54x-vbat"); + if (IS_ERR(pdata->vbat_reg)) { + pr_err("%s: could not get nxp,pn54x-vbat, rc=%ld\n", + __func__, PTR_ERR(pdata->vbat_reg)); + pdata->vbat_reg = NULL; + } + + pdata->pmuvcc_reg = regulator_get(dev, "nxp,pn54x-pmuvcc"); + if (IS_ERR(pdata->pmuvcc_reg)) { + pr_err("%s: could not get nxp,pn54x-pmuvcc, rc=%ld\n", + __func__, PTR_ERR(pdata->pmuvcc_reg)); + pdata->pmuvcc_reg = NULL; + } + + pdata->sevdd_reg = regulator_get(dev, "nxp,pn54x-sevdd"); + if (IS_ERR(pdata->sevdd_reg)) { + pr_err("%s: could not get nxp,pn54x-sevdd, rc=%ld\n", + __func__, PTR_ERR(pdata->sevdd_reg)); + pdata->sevdd_reg = NULL; + } + + return 0; +} +#else +static int pn54x_get_pdata(struct device *dev, + struct pn544_i2c_platform_data *pdata) +{ + pdata = dev->platform_data; + return 0; +} +#endif + +/* pn54x_probe */ +#ifdef KERNEL_3_4_AND_OLDER +static int __devinit pn54x_probe(struct i2c_client *client, + const struct i2c_device_id *id) +#else +static int pn54x_probe(struct i2c_client *client, + const struct i2c_device_id *id) +#endif +{ + int ret; + /* gpio values, from board file or DT */ + struct pn544_i2c_platform_data *pdata; + struct pn544_i2c_platform_data tmp_pdata; + /* internal device specific data */ + struct pn54x_dev *pn54x_dev; + + pr_info("%s\n", __func__); + + /* + * ---- retrieve the platform data ---- + * If the dev.platform_data is NULL, then + * attempt to read from the device tree + */ + if (!client->dev.platform_data) { + ret = pn54x_get_pdata(&(client->dev), &tmp_pdata); + if (ret) + return ret; + + pdata = &tmp_pdata; + } else { + pdata = client->dev.platform_data; + } + + if (pdata == NULL) { + pr_err("%s : nfc probe fail\n", __func__); + return -ENODEV; + } + + /* validate the the adapter has basic I2C functionality */ + if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C)) { + pr_err("%s : need I2C_FUNC_I2C\n", __func__); + return -ENODEV; + } + + if (gpio_is_valid(pdata->firm_gpio)) { + pr_info("%s: request firm_gpio %d\n", + __func__, pdata->firm_gpio); + ret = gpio_request(pdata->firm_gpio, "nfc_firm"); + if (ret) { + pr_err("%s :not able to get GPIO firm_gpio\n", + __func__); + goto err_firm; + } + } + + if (gpio_is_valid(pdata->clkreq_gpio)) { + pr_info("%s: request clkreq_gpio %d\n", + __func__, pdata->clkreq_gpio); + ret = gpio_request(pdata->clkreq_gpio, "nfc_clkreq"); + if (ret) { + pr_err("%s :not able to get GPIO clkreq_gpio\n", + __func__); + goto err_clkreq; + } + } + + /* allocate the pn54x driver information structure */ + pn54x_dev = kzalloc(sizeof(*pn54x_dev), GFP_KERNEL); + if (pn54x_dev == NULL) { + dev_err(&client->dev, + "failed to allocate memory for module data\n"); + ret = -ENOMEM; + goto err_exit; + } + + /* store the platform data in the driver info struct */ + pn54x_dev->firm_gpio = pdata->firm_gpio; + pn54x_dev->clkreq_gpio = pdata->clkreq_gpio; + pn54x_dev->pvdd_reg = pdata->pvdd_reg; + pn54x_dev->vbat_reg = pdata->vbat_reg; + pn54x_dev->pmuvcc_reg = pdata->pmuvcc_reg; + pn54x_dev->sevdd_reg = pdata->sevdd_reg; + + pn54x_dev->client = client; + + if (gpio_is_valid(pn54x_dev->firm_gpio)) { + ret = gpio_direction_output(pn54x_dev->firm_gpio, 0); + if (ret < 0) { + pr_err("%s : not able to set firm_gpio as output\n", + __func__); + goto err_exit; + } + } + + if (gpio_is_valid(pn54x_dev->clkreq_gpio)) { + ret = gpio_direction_output(pn54x_dev->clkreq_gpio, 0); + if (ret < 0) { + pr_err("%s : not able to set clkreq_gpio as output\n", + __func__); + goto err_exit; + } + } + + /* init mutex and queues */ + init_waitqueue_head(&pn54x_dev->read_wq); + mutex_init(&pn54x_dev->read_mutex); + spin_lock_init(&pn54x_dev->irq_enabled_lock); + + /* register as a misc device - character based with one entry point */ + pn54x_dev->pn54x_device.minor = MISC_DYNAMIC_MINOR; + pn54x_dev->pn54x_device.name = CHIP; + pn54x_dev->pn54x_device.fops = &pn54x_dev_fops; + ret = misc_register(&pn54x_dev->pn54x_device); + if (ret) { + pr_err("%s : misc_register failed\n", __FILE__); + goto err_misc_register; + } + + /* + * request irq. the irq is set whenever the chip has data available + * for reading. it is cleared when all data has been read. + */ + i2c_set_clientdata(client, pn54x_dev); + + return 0; + +err_request_irq_failed: + misc_deregister(&pn54x_dev->pn54x_device); +err_misc_register: +err_exit: + if (gpio_is_valid(pdata->clkreq_gpio)) + gpio_free(pdata->clkreq_gpio); +err_clkreq: + if (gpio_is_valid(pdata->firm_gpio)) + gpio_free(pdata->firm_gpio); +err_firm: +err_ven: + return ret; +} + +#ifdef KERNEL_3_4_AND_OLDER +static int __devexit pn54x_remove(struct i2c_client *client) +#else +static void pn54x_remove(struct i2c_client *client) +#endif +{ + struct pn54x_dev *pn54x_dev; + + pr_info("%s\n", __func__); + + pn54x_dev = i2c_get_clientdata(client); + free_irq(client->irq, pn54x_dev); + misc_deregister(&pn54x_dev->pn54x_device); + mutex_destroy(&pn54x_dev->read_mutex); + if (gpio_is_valid(pn54x_dev->firm_gpio)) + gpio_free(pn54x_dev->firm_gpio); + if (gpio_is_valid(pn54x_dev->clkreq_gpio)) + gpio_free(pn54x_dev->clkreq_gpio); + regulator_put(pn54x_dev->pvdd_reg); + regulator_put(pn54x_dev->vbat_reg); + regulator_put(pn54x_dev->pmuvcc_reg); + regulator_put(pn54x_dev->sevdd_reg); + + kfree(pn54x_dev); + +#ifdef KERNEL_3_4_AND_OLDER + return 0; +#else + return; +#endif +} + +#ifdef CONFIG_OF +static struct of_device_id pn54x_dt_match[] = { + { .compatible = "nxp,pn547", }, + { .compatible = "nxp,pn544", }, + {}, +}; +MODULE_DEVICE_TABLE(of, pn54x_dt_match); +#endif + +static const struct i2c_device_id pn54x_id[] = { + { "pn547", 0 }, + { }, +}; +MODULE_DEVICE_TABLE(i2c, pn54x_id); + +static struct i2c_driver pn54x_driver = { + .id_table = pn54x_id, + .probe = pn54x_probe, +#ifdef KERNEL_3_4_AND_OLDER + .remove = __devexit_p(pn54x_remove), +#else + .remove = pn54x_remove, +#endif + .driver = { + .owner = THIS_MODULE, + .name = "pn544", + .of_match_table = pn54x_dt_match, + }, +}; + +/* module load/unload record keeping */ +static int __init pn54x_dev_init(void) +{ + pr_info("%s\n", __func__); + return i2c_add_driver(&pn54x_driver); +} + +static void __exit pn54x_dev_exit(void) +{ + pr_info("%s\n", __func__); + i2c_del_driver(&pn54x_driver); +} + +module_init(pn54x_dev_init); +module_exit(pn54x_dev_exit); + +MODULE_AUTHOR("Sylvain Fonteneau"); +MODULE_DESCRIPTION(DRIVER_DESC); +MODULE_LICENSE("GPL"); diff --git a/drivers/nfc/pn5xx/pn5xx_i2c.h b/drivers/nfc/pn5xx/pn5xx_i2c.h new file mode 100644 index 000000000000..4b53655007e0 --- /dev/null +++ b/drivers/nfc/pn5xx/pn5xx_i2c.h @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright (C) 2010 Trusted Logic S.A. + * Copyright 2015,2019-2023 NXP + * + */ + +#define PN544_MAGIC 0xE9 + +/* + * PN544 power control via ioctl + * PN544_SET_PWR(0): power off + * PN544_SET_PWR(1): power on + * PN544_SET_PWR(2): reset and power on with firmware download enabled + */ + +#define PWR_OFF 0 +#define PWR_ON 1 +#define PWR_FW 2 + +#define CLK_OFF 0 +#define CLK_ON 1 + +#define GPIO_UNUSED -1 + +#define PN544_SET_PWR _IOW(PN544_MAGIC, 0x01, unsigned int) +#define PN54X_CLK_REQ _IOW(PN544_MAGIC, 0x02, unsigned int) + +struct pn544_i2c_platform_data { + unsigned int irq_gpio; + unsigned int ven_gpio; + unsigned int firm_gpio; + unsigned int clkreq_gpio; + struct regulator *pvdd_reg; + struct regulator *vbat_reg; + struct regulator *pmuvcc_reg; + struct regulator *sevdd_reg; +}; diff --git a/drivers/nfc/pn5xx/sample_devicetree.txt b/drivers/nfc/pn5xx/sample_devicetree.txt new file mode 100644 index 000000000000..0e81959052fc --- /dev/null +++ b/drivers/nfc/pn5xx/sample_devicetree.txt @@ -0,0 +1,17 @@ +Example: + +&i2c{ + + status = "okay"; + + pn547: pn547@29 { + + compatible = "nxp,pn547"; + + reg = <0x29>; + clock-frequency = <400000>; + + interrupt-gpios = <&gpio2 17 0>; + enable-gpios = <&gpio4 21 0>; + }; +}; diff --git a/drivers/rpmsg/Kconfig b/drivers/rpmsg/Kconfig index afba55506632..968ddd1fac18 100644 --- a/drivers/rpmsg/Kconfig +++ b/drivers/rpmsg/Kconfig @@ -81,10 +81,17 @@ config RPMSG_VIRTIO select RPMSG_NS select VIRTIO +config RPMSG_PERF + tristate "RPMSG performance test driver" + depends on RPMSG + default m + help + Say Y here to enable the RPMSG performance test driver. + config HAVE_IMX_RPMSG bool "IMX RPMSG driver on the AMP SOCs" default y - depends on IMX_MBOX + depends on IMX_MBOX || GENERIC_SOFTWARE_MAILBOX select RPMSG_VIRTIO help Say y here to enable support for the iMX Rpmsg Driver providing @@ -118,4 +125,11 @@ config IMX_RPMSG_TTY executed by the command "echo > ", thus remote M core would receive the string. +config RPMSG_8M_BUF + bool "RPMSG 8M bytes buffer support" + default n + depends on HAVE_IMX_RPMSG + help + Say y here to enable support for 8M bytes Vring buffer for rpmsg. + endmenu diff --git a/drivers/rpmsg/Makefile b/drivers/rpmsg/Makefile index 7c4ee52e92c4..f64fa4d0ada2 100644 --- a/drivers/rpmsg/Makefile +++ b/drivers/rpmsg/Makefile @@ -1,6 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_RPMSG) += rpmsg_core.o obj-$(CONFIG_RPMSG_CHAR) += rpmsg_char.o +obj-$(CONFIG_RPMSG_PERF) += rpmsg_perf.o obj-$(CONFIG_RPMSG_CTRL) += rpmsg_ctrl.o obj-$(CONFIG_RPMSG_NS) += rpmsg_ns.o obj-$(CONFIG_RPMSG_MTK_SCP) += mtk_rpmsg.o diff --git a/drivers/rpmsg/imx_rpmsg.c b/drivers/rpmsg/imx_rpmsg.c index 8d4ecd7360fd..b4277d5cfe37 100644 --- a/drivers/rpmsg/imx_rpmsg.c +++ b/drivers/rpmsg/imx_rpmsg.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* - * Copyright 2019 NXP + * Copyright 2019,2023 NXP */ #include @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -163,8 +164,12 @@ static struct virtqueue *rp_find_vq(struct virtio_device *vdev, return ERR_PTR(-ENOMEM); /* ioremap'ing normal memory, so we cast away sparse's complaints */ - rpvq->addr = (__force void *) ioremap(virdev->vring[index], - RPMSG_RING_SIZE); + if (of_dma_is_coherent(dev->of_node)) + rpvq->addr = (__force void *)ioremap_cache(virdev->vring[index], + RPMSG_RING_SIZE); + else + rpvq->addr = (__force void *)ioremap(virdev->vring[index], + RPMSG_RING_SIZE); if (!rpvq->addr) { err = -ENOMEM; goto free_rpvq; @@ -466,6 +471,13 @@ static int imx_rpmsg_rxdb_channel_init(struct imx_rpmsg_vproc *rpdev) return ret; } +static int imx_rpmsg_rxdb_channel_deinit(struct imx_rpmsg_vproc *rpdev) +{ + mbox_free_channel(rpdev->rxdb_ch); + + return 0; +} + static void imx_rpmsg_rx_callback(struct mbox_client *c, void *msg) { int buf_space; @@ -531,6 +543,14 @@ static int imx_rpmsg_xtr_channel_init(struct imx_rpmsg_vproc *rpdev) return ret; } +static int imx_rpmsg_xtr_channel_deinit(struct imx_rpmsg_vproc *rpdev) +{ + mbox_free_channel(rpdev->tx_ch); + mbox_free_channel(rpdev->rx_ch); + + return 0; +} + static int imx_rpmsg_probe(struct platform_device *pdev) { int j, ret = 0; @@ -653,6 +673,36 @@ static int imx_rpmsg_probe(struct platform_device *pdev) return ret; } +static int imx_rpmsg_remove(struct platform_device *pdev) +{ + struct imx_rpmsg_vproc *rpdev = platform_get_drvdata(pdev); + struct device *dev = &pdev->dev; + int i; + +#ifdef CONFIG_IMX_SCU + if (rpdev->variant == IMX8QXP || rpdev->variant == IMX8QM) { + imx_scu_irq_unregister_notifier(&rpdev->proc_nb); + imx_scu_irq_group_enable(IMX_SC_IRQ_GROUP_REBOOTED, + BIT(rpdev->mub_partition), false); + } +#endif + imx_rpmsg_rxdb_channel_deinit(rpdev); + + for (i = 0; i < rpdev->vdev_nums; i++) { + unregister_virtio_device(&rpdev->ivdev[i]->vdev); + kfree(rpdev->ivdev[i]); + } + + if (rpdev->flags & SPECIFIC_DMA_POOL) + of_reserved_mem_device_release(dev); + + imx_rpmsg_xtr_channel_deinit(rpdev); + + cancel_delayed_work_sync(&rpdev->rpmsg_work); + + return 0; +} + static struct platform_driver imx_rpmsg_driver = { .driver = { .owner = THIS_MODULE, @@ -660,6 +710,7 @@ static struct platform_driver imx_rpmsg_driver = { .of_match_table = imx_rpmsg_dt_ids, }, .probe = imx_rpmsg_probe, + .remove = imx_rpmsg_remove, }; static int __init imx_rpmsg_init(void) diff --git a/drivers/rpmsg/imx_rpmsg_tty.c b/drivers/rpmsg/imx_rpmsg_tty.c index 6114c686c388..ab3e64cc1a06 100644 --- a/drivers/rpmsg/imx_rpmsg_tty.c +++ b/drivers/rpmsg/imx_rpmsg_tty.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* - * Copyright 2019 NXP + * Copyright 2019,2023 NXP */ #include @@ -13,8 +13,24 @@ #include #include +#ifdef CONFIG_RPMSG_8M_BUF +struct rpmsg_hdr { + __rpmsg32 src; + __rpmsg32 dst; + __rpmsg32 reserved; + __rpmsg16 len; + __rpmsg16 flags; + u8 data[]; +} __packed; + +#define MAX_RPMSG_BUF_SIZE (512 * 2) +/* this needs to be less then (RPMSG_BUF_SIZE - sizeof(struct rpmsg_hdr)) */ +#define RPMSG_MAX_SIZE (MAX_RPMSG_BUF_SIZE - sizeof(struct rpmsg_hdr)) +#else /* this needs to be less then (RPMSG_BUF_SIZE - sizeof(struct rpmsg_hdr)) */ #define RPMSG_MAX_SIZE 256 +#endif + #define MSG "hello world!" /* diff --git a/drivers/rpmsg/rpmsg_perf.c b/drivers/rpmsg/rpmsg_perf.c new file mode 100644 index 000000000000..f58b15b2ea8d --- /dev/null +++ b/drivers/rpmsg/rpmsg_perf.c @@ -0,0 +1,545 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2023 NXP + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "rpmsg_internal.h" + +#define RPMSG_PERF_AS_SENDER_IOCTL _IO(0xb5, 0x5) +#define RPMSG_PERF_AS_RECEIVER_IOCTL _IO(0xb5, 0x6) +#define RPMSG_PERF_AS_RECEIVER_END_ACK_IOCTL _IO(0xb5, 0x7) +#define RPMSG_PERF_GET_RUNNING_STA_IOCTL _IO(0xb5, 0x8) + +#define RPMSG_DEV_MAX (MINORMASK + 1) + +static dev_t rpmsg_major; + +static DEFINE_IDA(rpmsg_perf_minor_ida); + +#define dev_to_eptdev(dev) container_of(dev, struct rpmsg_eptdev, dev) +#define cdev_to_eptdev(i_cdev) container_of(i_cdev, struct rpmsg_eptdev, cdev) + +struct packet_header { + uint32_t preamble; + bool no_copy; + uint32_t packet_size; + uint32_t packet_cnt; + uint32_t test_time; /* unit: second */ +}; + +enum { + RPMSG_PERF_PREAMBLE_SENDER_START = 0xBECAACEA, + RPMSG_PERF_PREAMBLE_SENDER_END = 0xBECAACEB, + RPMSG_PERF_PREAMBLE_SENDER_END_ACK = 0xBECAACEC, + RPMSG_PERF_PREAMBLE_RECEIVER_START = 0xBECAACED, + RPMSG_PERF_PREAMBLE_RECEIVER_END = 0xBECAACEE, + RPMSG_PERF_PREAMBLE_RECEIVER_END_ACK = 0xBECAACEF, +}; + +enum dev_state { + RPMSG_DEV_IDLE, + RPMSG_DEV_SENDING, + RPMSG_DEV_RECEIVING, +}; + +struct test_statistic { + uint32_t recv_packet_cnt; + uint32_t send_packet_cnt; + uint32_t packet_size; + uint32_t test_time; +}; + +/** + * struct rpmsg_eptdev - endpoint device context + * @dev: endpoint device + * @cdev: cdev for the endpoint device + * @rpdev: underlaying rpmsg device + * @chinfo: info used to open the endpoint + * @ept_lock: synchronization of @ept modifications + * @ept: rpmsg endpoint reference, when open + * @default_ept: set to channel default endpoint if the default endpoint should be re-used + * on device open to prevent endpoint address update. + */ +struct rpmsg_eptdev { + struct device dev; + struct cdev cdev; + + struct rpmsg_device *rpdev; + struct rpmsg_channel_info chinfo; + + struct mutex ept_lock; + struct rpmsg_endpoint *ept; + struct rpmsg_endpoint *default_ept; + + enum dev_state state; + struct test_statistic statistic; + + struct packet_header param; +}; + +static int rpmsg_perf_ept_cb(struct rpmsg_device *rpdev, void *buf, int len, + void *priv, u32 addr) +{ + struct packet_header *hdr = (struct packet_header *)buf; + struct rpmsg_eptdev *eptdev = priv; + struct test_statistic *statistic = &eptdev->statistic; + uint32_t rate; + + switch (hdr->preamble) { + case RPMSG_PERF_PREAMBLE_SENDER_END_ACK: + if (eptdev->state == RPMSG_DEV_SENDING) { + eptdev->state = RPMSG_DEV_IDLE; + rate = statistic->send_packet_cnt / + statistic->test_time / 1000; + pr_info("packet size: %u, sent packets: %u, time: %u s, rate: %u kpps\n", + statistic->packet_size, + statistic->send_packet_cnt, + statistic->test_time, rate); + statistic->packet_size = 0; + statistic->send_packet_cnt = 0; + } + break; + default: + break; + } + statistic->recv_packet_cnt++; + + return 0; +} + +static int rpmsg_perf_eptdev_open(struct inode *inode, struct file *filp) +{ + struct rpmsg_eptdev *eptdev = cdev_to_eptdev(inode->i_cdev); + struct rpmsg_endpoint *ept; + struct rpmsg_device *rpdev = eptdev->rpdev; + struct device *dev = &eptdev->dev; + + mutex_lock(&eptdev->ept_lock); + if (eptdev->ept) { + mutex_unlock(&eptdev->ept_lock); + return -EBUSY; + } + + get_device(dev); + + /* + * If the default_ept is set, the rpmsg device default endpoint is used. + * Else a new endpoint is created on open that will be destroyed on release. + */ + if (eptdev->default_ept) + ept = eptdev->default_ept; + else + ept = rpmsg_create_ept(rpdev, rpmsg_perf_ept_cb, eptdev, eptdev->chinfo); + + if (!ept) { + dev_err(dev, "failed to open %s\n", eptdev->chinfo.name); + put_device(dev); + mutex_unlock(&eptdev->ept_lock); + return -EINVAL; + } + + eptdev->ept = ept; + filp->private_data = eptdev; + mutex_unlock(&eptdev->ept_lock); + + return 0; +} + +static int rpmsg_perf_eptdev_release(struct inode *inode, struct file *filp) +{ + struct rpmsg_eptdev *eptdev = cdev_to_eptdev(inode->i_cdev); + struct device *dev = &eptdev->dev; + + /* Close the endpoint, if it's not already destroyed by the parent */ + mutex_lock(&eptdev->ept_lock); + if (eptdev->ept) { + if (!eptdev->default_ept) + rpmsg_destroy_ept(eptdev->ept); + eptdev->ept = NULL; + } + mutex_unlock(&eptdev->ept_lock); + + put_device(dev); + + return 0; +} + +static int rpmsg_perf_sender_thread(void *p) +{ + struct rpmsg_eptdev *eptdev = (struct rpmsg_eptdev *)p; + struct packet_header hdr = eptdev->param; + unsigned long timeout; + uint8_t *data; + uint32_t packet_len; + int ret; + + hdr.preamble = RPMSG_PERF_PREAMBLE_SENDER_START; + packet_len = hdr.packet_size; + eptdev->statistic.send_packet_cnt = 0; + + ret = rpmsg_sendto(eptdev->ept, &hdr, sizeof(hdr), eptdev->chinfo.dst); + if (ret) { + pr_err("failed to send packet header\n"); + return ret; + } + + /* Prepare data packets */ + data = kmalloc(packet_len, GFP_KERNEL); + if (data == NULL) { + pr_err("allocate data buffer failure\n"); + return -ENOMEM; + } + memset(data, 0, packet_len); + + udelay(100); + timeout = jiffies + msecs_to_jiffies(hdr.test_time * 1000); + + do { + do { + ret = rpmsg_trysendto(eptdev->ept, data, packet_len, + eptdev->chinfo.dst); + } while (ret != 0); + eptdev->statistic.send_packet_cnt++; + } while (time_before(jiffies, timeout)); + + memset(&hdr, 0, sizeof(hdr)); + hdr.preamble = RPMSG_PERF_PREAMBLE_SENDER_END; + + ret = rpmsg_sendto(eptdev->ept, &hdr, sizeof(hdr), eptdev->chinfo.dst); + if (ret) { + pr_err("failed to send RPMSG_PERF_PREAMBLE_SENDER_END packet\n"); + } + + kfree(data); + + return ret; +} + +static long rpmsg_perf_eptdev_ioctl(struct file *fp, unsigned int cmd, + unsigned long arg) +{ + struct rpmsg_eptdev *eptdev = fp->private_data; + struct test_statistic *statistic = &eptdev->statistic; + struct task_struct *sender_thread_h; + struct packet_header hdr = {0}; + uint32_t packet_cnt; + uint32_t rate; + uint32_t status; + long ret; + + switch (cmd) { + case RPMSG_PERF_GET_RUNNING_STA_IOCTL: + status = eptdev->state; + if (copy_to_user((char __user *)arg, &status, sizeof(status))) { + pr_err("copy_to_user() failed\n"); + return -EFAULT; + } + break; + case RPMSG_PERF_AS_SENDER_IOCTL: + if (eptdev->state != RPMSG_DEV_IDLE) + return -EINVAL; + + eptdev->state = RPMSG_DEV_SENDING; + ret = copy_from_user((void *)&hdr, (const void __user *)arg, + sizeof(struct packet_header)); + if (ret) { + pr_err("copy_from_user() failed\n"); + return -EFAULT; + } + + eptdev->param.no_copy = hdr.no_copy; + eptdev->param.packet_size = hdr.packet_size; + eptdev->param.test_time = hdr.test_time; + statistic->packet_size = hdr.packet_size; + statistic->test_time = hdr.test_time; + + sender_thread_h = kthread_run(rpmsg_perf_sender_thread, + eptdev, "rpmsg_sender"); + if (IS_ERR(sender_thread_h)) { + pr_err("failed to create sender thread\n"); + return -EFAULT; + } + break; + case RPMSG_PERF_AS_RECEIVER_IOCTL: + if (eptdev->state != RPMSG_DEV_IDLE) + return -EINVAL; + + eptdev->state = RPMSG_DEV_RECEIVING; + ret = copy_from_user((void *)&hdr, (const void __user *)arg, + sizeof(struct packet_header)); + if (ret) { + pr_err("copy_from_user() failed\n"); + return -EFAULT; + } + hdr.preamble = RPMSG_PERF_PREAMBLE_RECEIVER_START; + statistic->test_time = hdr.test_time; + statistic->packet_size = hdr.packet_size; + statistic->recv_packet_cnt = 0; + + ret = rpmsg_sendto(eptdev->ept, &hdr, sizeof(hdr), + eptdev->chinfo.dst); + if (ret) + pr_err("failed to send RPMSG_PERF_PREAMBLE_RECEIVER_START packet\n"); + break; + case RPMSG_PERF_AS_RECEIVER_END_ACK_IOCTL: + if (eptdev->state != RPMSG_DEV_RECEIVING) + return -EINVAL; + + packet_cnt = statistic->recv_packet_cnt - 1; + rate = packet_cnt / statistic->test_time / 1000; + pr_info("packet size: %u, received packets: %u, time: %u s, rate: %u kpps\n", + statistic->packet_size, packet_cnt, + statistic->test_time, rate); + + hdr.preamble = RPMSG_PERF_PREAMBLE_RECEIVER_END_ACK; + hdr.packet_cnt = packet_cnt; + ret = rpmsg_sendto(eptdev->ept, &hdr, sizeof(hdr), + eptdev->chinfo.dst); + if (ret) { + pr_err("failed to send RPMSG_PERF_PREAMBLE_RECEIVER_END_ACK packet\n"); + return ret; + } + eptdev->state = RPMSG_DEV_IDLE; + statistic->test_time = 0; + break; + default: + ret = -EINVAL; + break; + } + + return ret; +} + +static const struct file_operations rpmsg_perf_eptdev_fops = { + .owner = THIS_MODULE, + .open = rpmsg_perf_eptdev_open, + .release = rpmsg_perf_eptdev_release, + .unlocked_ioctl = rpmsg_perf_eptdev_ioctl, + .compat_ioctl = compat_ptr_ioctl, +}; + +static ssize_t name_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct rpmsg_eptdev *eptdev = dev_get_drvdata(dev); + + return sprintf(buf, "%s\n", eptdev->chinfo.name); +} +static DEVICE_ATTR_RO(name); + +static ssize_t src_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct rpmsg_eptdev *eptdev = dev_get_drvdata(dev); + + return sprintf(buf, "%d\n", eptdev->chinfo.src); +} +static DEVICE_ATTR_RO(src); + +static ssize_t dst_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct rpmsg_eptdev *eptdev = dev_get_drvdata(dev); + + return sprintf(buf, "%d\n", eptdev->chinfo.dst); +} +static DEVICE_ATTR_RO(dst); + +static struct attribute *rpmsg_perf_eptdev_attrs[] = { + &dev_attr_name.attr, + &dev_attr_src.attr, + &dev_attr_dst.attr, + NULL +}; +ATTRIBUTE_GROUPS(rpmsg_perf_eptdev); + +static void rpmsg_eptdev_release_device(struct device *dev) +{ + struct rpmsg_eptdev *eptdev = dev_to_eptdev(dev); + + ida_simple_remove(&rpmsg_perf_minor_ida, MINOR(eptdev->dev.devt)); + kfree(eptdev); +} + +static struct rpmsg_eptdev *rpmsg_perf_eptdev_alloc(struct rpmsg_device *rpdev, + struct device *parent) +{ + struct rpmsg_eptdev *eptdev; + struct device *dev; + + eptdev = kzalloc(sizeof(*eptdev), GFP_KERNEL); + if (!eptdev) + return ERR_PTR(-ENOMEM); + + eptdev->state = RPMSG_DEV_IDLE; + + dev = &eptdev->dev; + eptdev->rpdev = rpdev; + + mutex_init(&eptdev->ept_lock); + + device_initialize(dev); + dev->class = rpmsg_class; + dev->parent = parent; + dev->groups = rpmsg_perf_eptdev_groups; + dev_set_drvdata(dev, eptdev); + + cdev_init(&eptdev->cdev, &rpmsg_perf_eptdev_fops); + eptdev->cdev.owner = THIS_MODULE; + + return eptdev; +} + +static int rpmsg_perf_eptdev_add(struct rpmsg_eptdev *eptdev, + struct rpmsg_channel_info chinfo) +{ + struct device *dev = &eptdev->dev; + int dst = eptdev->rpdev->dst; + int ret; + + eptdev->chinfo = chinfo; + + ret = ida_simple_get(&rpmsg_perf_minor_ida, 0, + RPMSG_DEV_MAX, GFP_KERNEL); + if (ret < 0) + goto free_eptdev; + dev->devt = MKDEV(MAJOR(rpmsg_major), ret); + + dev->id = dst; + dev_set_name(dev, "rpmsg-perf%d", dst); + + ret = cdev_device_add(&eptdev->cdev, &eptdev->dev); + if (ret) + goto free_minor_ida; + + /* We can now rely on the release function for cleanup */ + dev->release = rpmsg_eptdev_release_device; + + return ret; + +free_minor_ida: + ida_simple_remove(&rpmsg_perf_minor_ida, MINOR(dev->devt)); +free_eptdev: + put_device(dev); + kfree(eptdev); + + return ret; +} + +static int rpmsg_perf_probe(struct rpmsg_device *rpdev) +{ + struct rpmsg_channel_info chinfo; + struct rpmsg_eptdev *eptdev; + struct device *dev = &rpdev->dev; + + memcpy(chinfo.name, rpdev->id.name, RPMSG_NAME_SIZE); + chinfo.src = rpdev->src; + chinfo.dst = rpdev->dst; + + eptdev = rpmsg_perf_eptdev_alloc(rpdev, dev); + if (IS_ERR(eptdev)) + return PTR_ERR(eptdev); + + /* Set the default_ept to the rpmsg device endpoint */ + eptdev->default_ept = rpdev->ept; + + /* + * The rpmsg_perf_ept_cb uses *priv parameter to get its + * rpmsg_eptdev context. Stored it in default_ept *priv field. + */ + eptdev->default_ept->priv = eptdev; + + return rpmsg_perf_eptdev_add(eptdev, chinfo); +} + +static int rpmsg_perf_eptdev_chrdev_destroy(struct device *dev, void *data) +{ + struct rpmsg_eptdev *eptdev = dev_to_eptdev(dev); + + mutex_lock(&eptdev->ept_lock); + if (eptdev->ept) { + /* The default endpoint is released by the rpmsg core */ + if (!eptdev->default_ept) + rpmsg_destroy_ept(eptdev->ept); + eptdev->ept = NULL; + } + mutex_unlock(&eptdev->ept_lock); + + cdev_device_del(&eptdev->cdev, &eptdev->dev); + put_device(&eptdev->dev); + + return 0; +} + +static void rpmsg_perf_remove(struct rpmsg_device *rpdev) +{ + int ret; + + ret = device_for_each_child(&rpdev->dev, NULL, + rpmsg_perf_eptdev_chrdev_destroy); + if (ret) + dev_warn(&rpdev->dev, "failed to destroy endpoints: %d\n", ret); +} + +static struct rpmsg_device_id rpmsg_perf_id_table[] = { + { .name = "rpmsg-perf" }, + { }, +}; + +static struct rpmsg_driver rpmsg_perf_driver = { + .probe = rpmsg_perf_probe, + .remove = rpmsg_perf_remove, + .callback = rpmsg_perf_ept_cb, + .id_table = rpmsg_perf_id_table, + .drv.name = "rpmsg_perf", +}; + +static int rpmsg_perf_init(void) +{ + int ret; + + ret = alloc_chrdev_region(&rpmsg_major, 0, RPMSG_DEV_MAX, "rpmsg_perf"); + if (ret < 0) { + pr_err("failed to allocate char dev region\n"); + return ret; + } + + ret = register_rpmsg_driver(&rpmsg_perf_driver); + if (ret < 0) { + pr_err("rpmsg: failed to register rpmsg raw driver\n"); + goto free_region; + } + + return 0; + +free_region: + unregister_chrdev_region(rpmsg_major, RPMSG_DEV_MAX); + + return ret; +} +postcore_initcall(rpmsg_perf_init); + +static void rpmsg_perf_exit(void) +{ + unregister_rpmsg_driver(&rpmsg_perf_driver); + unregister_chrdev_region(rpmsg_major, RPMSG_DEV_MAX); +} +module_exit(rpmsg_perf_exit); + +MODULE_ALIAS("rpmsg:rpmsg_perf"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/rpmsg/virtio_rpmsg_bus.c b/drivers/rpmsg/virtio_rpmsg_bus.c index 905ac7910c98..d9f341bfd299 100644 --- a/drivers/rpmsg/virtio_rpmsg_bus.c +++ b/drivers/rpmsg/virtio_rpmsg_bus.c @@ -127,8 +127,13 @@ struct virtio_rpmsg_channel { * can change this without changing anything in the firmware of the remote * processor. */ +#ifdef CONFIG_RPMSG_8M_BUF +#define MAX_RPMSG_NUM_BUFS (1024 * 8) +#define MAX_RPMSG_BUF_SIZE (512 * 2) +#else #define MAX_RPMSG_NUM_BUFS (512) #define MAX_RPMSG_BUF_SIZE (512) +#endif /* * Local addresses are dynamically allocated on-demand. diff --git a/drivers/rtc/Kconfig b/drivers/rtc/Kconfig index 36d99d7a183c..cd0e5bdd37f6 100644 --- a/drivers/rtc/Kconfig +++ b/drivers/rtc/Kconfig @@ -1850,6 +1850,7 @@ config RTC_DRV_IMX_SC config RTC_DRV_IMX_RPMSG tristate "NXP RPMSG RTC support" depends on OF + depends on RPMSG help If you say yes here you get support for the NXP RPMSG RTC module. diff --git a/drivers/spi/spi.c b/drivers/spi/spi.c index 151fef199c38..825412e799cd 100644 --- a/drivers/spi/spi.c +++ b/drivers/spi/spi.c @@ -127,10 +127,10 @@ do { \ unsigned int start; \ pcpu_stats = per_cpu_ptr(in, i); \ do { \ - start = u64_stats_fetch_begin_irq( \ + start = u64_stats_fetch_begin( \ &pcpu_stats->syncp); \ inc = u64_stats_read(&pcpu_stats->field); \ - } while (u64_stats_fetch_retry_irq( \ + } while (u64_stats_fetch_retry( \ &pcpu_stats->syncp, start)); \ ret += inc; \ } \ diff --git a/drivers/staging/fsl_qbman/qman_driver.c b/drivers/staging/fsl_qbman/qman_driver.c index 14671df9e4ce..135235595521 100644 --- a/drivers/staging/fsl_qbman/qman_driver.c +++ b/drivers/staging/fsl_qbman/qman_driver.c @@ -1,4 +1,5 @@ /* Copyright 2008-2012 Freescale Semiconductor, Inc. + * Copyright 2019-2023 NXP * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -722,6 +723,50 @@ static int qman_online_cpu(unsigned int cpu) #endif /* CONFIG_HOTPLUG_CPU */ +#ifdef CONFIG_FSL_DPAA_ETHERCAT +__init void qman_ethercat_portal_init(int cpu) +{ + struct qm_portal_config *pcfg; + struct qman_portal *p; + + pcfg = get_pcfg(&unused_pcfgs); + if (pcfg) { + pcfg->public_cfg.cpu = cpu; + pcfg->public_cfg.is_shared = 0; + + pcfg->iommu_domain = NULL; + portal_set_cpu(pcfg, pcfg->public_cfg.cpu); + p = qman_create_affine_portal_ethercat(pcfg, NULL, cpu); + if (p) { + pr_info("Qman portal %sinitialised, cpu %d\n", + pcfg->public_cfg.is_shared ? "(shared) " : "", + pcfg->public_cfg.cpu); + } else { + pr_crit("Qman portal failure on cpu %d\n", + pcfg->public_cfg.cpu); + } + return; + } +} + +static u32 qman_affine_last_cpu; + +u32 qman_get_affine_last_cpu(void) +{ + return qman_affine_last_cpu; +} + +__init void qman_ethercat_portal_init_on_cpu(void) +{ + int cpu = 0; + + for_each_online_cpu(cpu) { + qman_affine_last_cpu = cpu; + qman_ethercat_portal_init(cpu); + } +} +#endif + __init int qman_init(void) { struct cpumask slave_cpus; @@ -877,6 +922,11 @@ __init int qman_init(void) return ret; } #endif + +#ifdef CONFIG_FSL_DPAA_ETHERCAT + qman_ethercat_portal_init_on_cpu(); +#endif + return 0; } diff --git a/drivers/staging/fsl_qbman/qman_high.c b/drivers/staging/fsl_qbman/qman_high.c index 961177fa681e..15a38ed75a2b 100644 --- a/drivers/staging/fsl_qbman/qman_high.c +++ b/drivers/staging/fsl_qbman/qman_high.c @@ -1,4 +1,5 @@ /* Copyright 2008-2012 Freescale Semiconductor, Inc. + * Copyright 2019-2023 NXP * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -776,6 +777,29 @@ struct qman_portal *qman_create_portal( return NULL; } +#ifdef CONFIG_FSL_DPAA_ETHERCAT +static struct qman_portal ethercat_portal[NR_CPUS]; +static u16 ethercat_channel[NR_CPUS]; +static DEFINE_SPINLOCK(ethercat_mask_lock); + +struct qman_portal *qman_create_affine_portal_ethercat + (const struct qm_portal_config *config, + const struct qman_cgrs *cgrs, int cpu) +{ + struct qman_portal *res; + struct qman_portal *portal = NULL; + + portal = ðercat_portal[cpu]; + res = qman_create_portal(portal, config, cgrs); + if (res) { + spin_lock(ðercat_mask_lock); + ethercat_channel[cpu] = config->public_cfg.channel; + spin_unlock(ðercat_mask_lock); + } + return res; +} +#endif + struct qman_portal *qman_create_affine_portal( const struct qm_portal_config *config, const struct qman_cgrs *cgrs) @@ -1412,6 +1436,20 @@ void *qman_get_affine_portal(int cpu) } EXPORT_SYMBOL(qman_get_affine_portal); +#ifdef CONFIG_FSL_DPAA_ETHERCAT +u16 qman_affine_channel_ethercat(int cpu) +{ + return ethercat_channel[cpu]; +} +EXPORT_SYMBOL(qman_affine_channel_ethercat); + +void *qman_get_affine_portal_ethercat(int cpu) +{ + return ðercat_portal[cpu]; +} +EXPORT_SYMBOL(qman_get_affine_portal_ethercat); +#endif + int qman_p_poll_dqrr(struct qman_portal *p, unsigned int limit) { int ret; diff --git a/drivers/staging/fsl_qbman/qman_private.h b/drivers/staging/fsl_qbman/qman_private.h index 4ba7bd5b0bb2..47a06dc35e5a 100644 --- a/drivers/staging/fsl_qbman/qman_private.h +++ b/drivers/staging/fsl_qbman/qman_private.h @@ -1,4 +1,5 @@ /* Copyright 2008-2012 Freescale Semiconductor, Inc. + * Copyright 2019-2023 NXP * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -232,6 +233,11 @@ struct qman_portal *qman_create_portal( const struct qm_portal_config *config, const struct qman_cgrs *cgrs); +#ifdef CONFIG_FSL_DPAA_ETHERCAT +struct qman_portal *qman_create_affine_portal_ethercat + (const struct qm_portal_config *config, + const struct qman_cgrs *cgrs, int cpu); +#endif struct qman_portal *qman_create_affine_portal( const struct qm_portal_config *config, const struct qman_cgrs *cgrs); diff --git a/drivers/tty/Kconfig b/drivers/tty/Kconfig index cc30ff93e2e4..ccfbd6187e4a 100644 --- a/drivers/tty/Kconfig +++ b/drivers/tty/Kconfig @@ -382,4 +382,6 @@ config RPMSG_TTY endif # TTY + + source "drivers/tty/serdev/Kconfig" diff --git a/drivers/tty/rpmsg_tty.c b/drivers/tty/rpmsg_tty.c index 29db413bbc03..a5d44406b566 100644 --- a/drivers/tty/rpmsg_tty.c +++ b/drivers/tty/rpmsg_tty.c @@ -1,13 +1,14 @@ // SPDX-License-Identifier: GPL-2.0 /* * Copyright (C) 2021 STMicroelectronics - All Rights Reserved + * Copyright 2023 NXP * * The rpmsg tty driver implements serial communication on the RPMsg bus to makes * possible for user-space programs to send and receive rpmsg messages as a standard * tty protocol. * - * The remote processor can instantiate a new tty by requesting a "rpmsg-tty" RPMsg service. - * The "rpmsg-tty" service is directly used for data exchange. No flow control is implemented yet. + * The remote processor can instantiate a new tty by requesting a "srtm-uart-channel" or "rpmsg-tty" RPMsg service. + * The "srtm-uart-channel" or "rpmsg-tty" service is directly used for data exchange. No flow control is implemented yet. */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt @@ -17,6 +18,43 @@ #include #include #include +#include +#include + +/* + * The srtm (simplified real time message) protocol for uart: + * + * +---------------+-------------------------------+ + * | Byte Offset | Content | + * +---------------+---+---+---+---+---+---+---+---+ + * | 0 | Category | + * +---------------+---+---+---+---+---+---+---+---+ + * | 1 ~ 2 | Version | + * +---------------+---+---+---+---+---+---+---+---+ + * | 3 | Type | + * +---------------+---+---+---+---+---+---+---+---+ + * | 4 | Command | + * +---------------+---+---+---+---+---+---+---+---+ + * | 5 | Priority | + * +---------------+---+---+---+---+---+---+---+---+ + * | 6 | Reserved1 | + * +---------------+---+---+---+---+---+---+---+---+ + * | 7 | Reserved2 | + * +---------------+---+---+---+---+---+---+---+---+ + * | 8 | Reserved3 | + * +---------------+---+---+---+---+---+---+---+---+ + * | 9 | Reserved4 | + * +---------------+---+---+---+---+---+---+---+---+ + * | 10 | UART BUS ID | + * +---------------+---+---+---+---+---+---+---+---+ + * | 11 | reserved/ret code | + * +---------------+---+---+---+---+---+---+---+---+ + * | 12 ~ 13 | Flags | + * +---------------+---+---+---+---+---+---+---+---+ + * | 14 ~ 495 | Data | + * +---------------+---+---+---+---+---+---+---+---+ + * UART BUS ID: real uart instance, such as: the soc has UART0(UART BUS ID = 0), UART1(UART BUS ID = 1), UART2(UART BUS ID = 2), ... , UARTx(UART BUS ID = x) + */ #define RPMSG_TTY_NAME "ttyRPMSG" #define MAX_TTY_RPMSG 32 @@ -29,18 +67,72 @@ static struct tty_driver *rpmsg_tty_driver; struct rpmsg_tty_port { struct tty_port port; /* TTY port data */ int id; /* TTY rpmsg index */ + int bus_id; /* used for srtm uart protocol, which real uart will be used, such as, uart0, uart1, uart2, uart3... */ + int flags; /* used for srtm uart protocol */ + u16 srtm_uart_msg_data_max_sz; /* used for srtm uart protocol */ + bool use_srtm_uart_protocol; /* used for srtm uart protocol */ struct rpmsg_device *rpdev; /* rpmsg device */ }; +struct srtm_uart_msg_header { + struct imx_srtm_head common; + u8 bus_id; /* The bus_id is used when send data from acore to mcore; The bus_id is useless when acore received data that from mcore*/ + union { + u8 reserved; /* used in request packet */ + u8 retCode; /* used in response packet */ + }; + u16 flags; +} __packed __aligned[1]; + +struct srtm_uart_msg { + struct srtm_uart_msg_header header; + /* srtm uart Payload Start */ + u8 data[1]; +} __packed __aligned(1); + +struct imx_srtm_uart_data_structure +{ + bool use_srtm_uart_protocol; +}; + +const static struct imx_srtm_uart_data_structure imx_srtm_uart_data = { + .use_srtm_uart_protocol = true, +}; + +const static struct imx_srtm_uart_data_structure rpmsg_tty_data = { + .use_srtm_uart_protocol = false, +}; + static int rpmsg_tty_cb(struct rpmsg_device *rpdev, void *data, int len, void *priv, u32 src) { struct rpmsg_tty_port *cport = dev_get_drvdata(&rpdev->dev); + struct srtm_uart_msg *msg = (struct srtm_uart_msg *)data; int copied; + u8 *payload = data; + int payload_len = len; - if (!len) + if (!payload_len) return -EINVAL; - copied = tty_insert_flip_string(&cport->port, data, len); - if (copied != len) + if (cport->use_srtm_uart_protocol) { + if (payload_len < (sizeof(struct srtm_uart_msg))) + return -EINVAL; + payload_len -= (sizeof(struct srtm_uart_msg) - 1); /* 1: msg->data[0] */ + + if (msg->header.common.type != IMX_SRTM_TYPE_RESPONSE && msg->header.common.type != IMX_SRTM_TYPE_NOTIFY) { + return -EINVAL; + } + + if (payload_len > cport->srtm_uart_msg_data_max_sz) { + dev_err(&rpdev->dev, + "%s failed: data length greater than %d, len=%d\n", + __func__, cport->srtm_uart_msg_data_max_sz, payload_len); + return -EINVAL; + } + payload = &msg->data[0]; + } + + copied = tty_insert_flip_string(&cport->port, payload, payload_len); + if (copied != payload_len) dev_err_ratelimited(&rpdev->dev, "Trunc buffer: available space is %d\n", copied); tty_flip_buffer_push(&cport->port); @@ -77,27 +169,52 @@ static int rpmsg_tty_write(struct tty_struct *tty, const u8 *buf, int len) { struct rpmsg_tty_port *cport = tty->driver_data; struct rpmsg_device *rpdev; - int msg_max_size, msg_size; + int msg_max_size, msg_size = -1; + struct srtm_uart_msg *msg = NULL; int ret; + void *data = NULL; + int data_len = 0; rpdev = cport->rpdev; - msg_max_size = rpmsg_get_mtu(rpdev->ept); + msg_max_size = cport->use_srtm_uart_protocol ? (cport->srtm_uart_msg_data_max_sz) : (rpmsg_get_mtu(rpdev->ept)); if (msg_max_size < 0) return msg_max_size; msg_size = min(len, msg_max_size); - + if (cport->use_srtm_uart_protocol) { + data_len = msg_size + sizeof(struct srtm_uart_msg) - 1; /* srtm uart msg header + data len */ + msg = kmalloc(data_len, GFP_KERNEL); + if (!msg) + return -ENOMEM; + + memset(msg, 0, data_len); + msg->header.common.cate = IMX_SRTM_CATEGORY_UART; + msg->header.common.major = IMX_SRTM_VER_UART; + msg->header.common.minor = IMX_SRTM_VER_UART >> 8; + msg->header.common.type = IMX_SRTM_TYPE_NOTIFY; + msg->header.common.cmd = IMX_SRTM_UART_COMMAND_SEND; + msg->header.common.reserved[0] = IMX_SRTM_UART_PRIORITY; + msg->header.bus_id = cport->bus_id & 0xFF; + msg->header.flags = cport->flags & 0xFFFF; + memcpy(&msg->data[0], buf, msg_size); + data = (void *)msg; + } else { + data = (void *)buf; + data_len = msg_size; + } /* * Use rpmsg_trysend instead of rpmsg_send to send the message so the caller is not * hung until a rpmsg buffer is available. In such case rpmsg_trysend returns -ENOMEM. */ - ret = rpmsg_trysend(rpdev->ept, (void *)buf, msg_size); + ret = rpmsg_trysend(rpdev->ept, data, data_len); if (ret) { dev_dbg_ratelimited(&rpdev->dev, "rpmsg_send failed: %d\n", ret); - return ret; + msg_size = ret; } + kfree(msg); + return msg_size; } @@ -105,8 +222,10 @@ static unsigned int rpmsg_tty_write_room(struct tty_struct *tty) { struct rpmsg_tty_port *cport = tty->driver_data; int size; + struct rpmsg_device *rpdev; - size = rpmsg_get_mtu(cport->rpdev->ept); + rpdev = cport->rpdev; + size = cport->use_srtm_uart_protocol ? (cport->srtm_uart_msg_data_max_sz) : (rpmsg_get_mtu(rpdev->ept)); if (size < 0) return 0; @@ -172,7 +291,12 @@ static int rpmsg_tty_probe(struct rpmsg_device *rpdev) struct rpmsg_tty_port *cport; struct device *dev = &rpdev->dev; struct device *tty_dev; + struct device_node *np; + struct imx_srtm_uart_data_structure *srtm_uart_data = NULL; + struct srtm_uart_msg *msg = NULL; int ret; + int data_len = 0; + char buf[64]; cport = rpmsg_tty_alloc_cport(); if (IS_ERR(cport)) @@ -181,6 +305,23 @@ static int rpmsg_tty_probe(struct rpmsg_device *rpdev) tty_port_init(&cport->port); cport->port.ops = &rpmsg_tty_port_ops; + cport->rpdev = rpdev; + + srtm_uart_data = (struct imx_srtm_uart_data_structure *)rpdev->id.driver_data; + if (srtm_uart_data && srtm_uart_data->use_srtm_uart_protocol == true) { + cport->bus_id = 0xFF; /* mcore directly print the data that received from acore */ + cport->use_srtm_uart_protocol = true; + snprintf(buf, sizeof(buf), "uart-rpbus-%d", cport->id); + np = of_find_node_by_name(NULL, buf); + if (np && of_device_is_compatible(np, "fsl,uart-rpbus") && of_device_is_available(np)) { + of_property_read_u32(np, "bus_id", &cport->bus_id); /* mcore will use the id as uart instance, then write data to the real uart instance */ + of_property_read_u32(np, "flags", &cport->flags); + } + cport->srtm_uart_msg_data_max_sz = rpmsg_get_mtu(rpdev->ept) - (sizeof(struct srtm_uart_msg) - 1); /* 1: data[1] of struct srtm_uart_msg */ + } else { + cport->use_srtm_uart_protocol = false; + } + tty_dev = tty_port_register_device(&cport->port, rpmsg_tty_driver, cport->id, dev); if (IS_ERR(tty_dev)) { @@ -189,9 +330,28 @@ static int rpmsg_tty_probe(struct rpmsg_device *rpdev) return ret; } - cport->rpdev = rpdev; - - dev_set_drvdata(dev, cport); + dev_set_drvdata(dev, (void *)cport); + + /* Say hello to remote to acknowleage each other */ + if (cport->use_srtm_uart_protocol) { + data_len = sizeof(struct srtm_uart_msg); /* srtm uart msg header */ + msg = kmalloc(data_len, GFP_KERNEL); + if (!msg) + return -ENOMEM; + + memset(msg, 0, data_len); + msg->header.common.cate = IMX_SRTM_CATEGORY_UART; + msg->header.common.major = IMX_SRTM_VER_UART; + msg->header.common.minor = IMX_SRTM_VER_UART >> 8; + msg->header.common.type = IMX_SRTM_TYPE_NOTIFY; + msg->header.common.cmd = IMX_SRTM_UART_COMMAND_HELLO; + msg->header.common.reserved[0] = IMX_SRTM_UART_PRIORITY; + msg->header.bus_id = cport->bus_id & 0xFF; + msg->header.flags = cport->flags & 0xFFFF; + + rpmsg_send(rpdev->ept, (void *)msg, data_len); + kfree(msg); + } dev_dbg(dev, "New channel: 0x%x -> 0x%x: " RPMSG_TTY_NAME "%d\n", rpdev->src, rpdev->dst, cport->id); @@ -214,7 +374,8 @@ static void rpmsg_tty_remove(struct rpmsg_device *rpdev) } static struct rpmsg_device_id rpmsg_driver_tty_id_table[] = { - { .name = "rpmsg-tty" }, + { .name = "srtm-uart-channel", .driver_data = (kernel_ulong_t)&imx_srtm_uart_data, }, + { .name = "rpmsg-tty", .driver_data = (kernel_ulong_t)&rpmsg_tty_data, }, { }, }; MODULE_DEVICE_TABLE(rpmsg, rpmsg_driver_tty_id_table); @@ -283,5 +444,6 @@ module_init(rpmsg_tty_init); module_exit(rpmsg_tty_exit); MODULE_AUTHOR("Arnaud Pouliquen "); +MODULE_AUTHOR("Biwen Li "); MODULE_DESCRIPTION("remote processor messaging tty driver"); MODULE_LICENSE("GPL v2"); diff --git a/drivers/tty/serial/8250/8250.h b/drivers/tty/serial/8250/8250.h index eeb7b43ebe53..b17715d340c3 100644 --- a/drivers/tty/serial/8250/8250.h +++ b/drivers/tty/serial/8250/8250.h @@ -176,12 +176,49 @@ static inline void serial_dl_write(struct uart_8250_port *up, int value) up->dl_write(up, value); } +static inline int serial8250_in_IER(struct uart_8250_port *up) +{ + struct uart_port *port = &up->port; + unsigned long flags; + bool is_console; + int ier; + + is_console = uart_console(port); + + if (is_console) + printk_cpu_sync_get_irqsave(flags); + + ier = serial_in(up, UART_IER); + + if (is_console) + printk_cpu_sync_put_irqrestore(flags); + + return ier; +} + +static inline void serial8250_set_IER(struct uart_8250_port *up, int ier) +{ + struct uart_port *port = &up->port; + unsigned long flags; + bool is_console; + + is_console = uart_console(port); + + if (is_console) + printk_cpu_sync_get_irqsave(flags); + + serial_out(up, UART_IER, ier); + + if (is_console) + printk_cpu_sync_put_irqrestore(flags); +} + static inline bool serial8250_set_THRI(struct uart_8250_port *up) { if (up->ier & UART_IER_THRI) return false; up->ier |= UART_IER_THRI; - serial_out(up, UART_IER, up->ier); + serial8250_set_IER(up, up->ier); return true; } @@ -190,7 +227,7 @@ static inline bool serial8250_clear_THRI(struct uart_8250_port *up) if (!(up->ier & UART_IER_THRI)) return false; up->ier &= ~UART_IER_THRI; - serial_out(up, UART_IER, up->ier); + serial8250_set_IER(up, up->ier); return true; } diff --git a/drivers/tty/serial/8250/8250_aspeed_vuart.c b/drivers/tty/serial/8250/8250_aspeed_vuart.c index 9d2a7856784f..7cc6b527c088 100644 --- a/drivers/tty/serial/8250/8250_aspeed_vuart.c +++ b/drivers/tty/serial/8250/8250_aspeed_vuart.c @@ -278,7 +278,7 @@ static void __aspeed_vuart_set_throttle(struct uart_8250_port *up, up->ier &= ~irqs; if (!throttle) up->ier |= irqs; - serial_out(up, UART_IER, up->ier); + serial8250_set_IER(up, up->ier); } static void aspeed_vuart_set_throttle(struct uart_port *port, bool throttle) { diff --git a/drivers/tty/serial/8250/8250_bcm7271.c b/drivers/tty/serial/8250/8250_bcm7271.c index ffc7f67e27e3..8b211e668bc0 100644 --- a/drivers/tty/serial/8250/8250_bcm7271.c +++ b/drivers/tty/serial/8250/8250_bcm7271.c @@ -609,7 +609,7 @@ static int brcmuart_startup(struct uart_port *port) * will handle this. */ up->ier &= ~UART_IER_RDI; - serial_port_out(port, UART_IER, up->ier); + serial8250_set_IER(up, up->ier); priv->tx_running = false; priv->dma.rx_dma = NULL; @@ -775,10 +775,12 @@ static int brcmuart_handle_irq(struct uart_port *p) unsigned int iir = serial_port_in(p, UART_IIR); struct brcmuart_priv *priv = p->private_data; struct uart_8250_port *up = up_to_u8250p(p); + unsigned long cs_flags; unsigned int status; unsigned long flags; unsigned int ier; unsigned int mcr; + bool is_console; int handled = 0; /* @@ -789,6 +791,10 @@ static int brcmuart_handle_irq(struct uart_port *p) spin_lock_irqsave(&p->lock, flags); status = serial_port_in(p, UART_LSR); if ((status & UART_LSR_DR) == 0) { + is_console = uart_console(p); + + if (is_console) + printk_cpu_sync_get_irqsave(cs_flags); ier = serial_port_in(p, UART_IER); /* @@ -809,6 +815,9 @@ static int brcmuart_handle_irq(struct uart_port *p) serial_port_in(p, UART_RX); } + if (is_console) + printk_cpu_sync_put_irqrestore(cs_flags); + handled = 1; } spin_unlock_irqrestore(&p->lock, flags); @@ -823,8 +832,10 @@ static enum hrtimer_restart brcmuart_hrtimer_func(struct hrtimer *t) struct brcmuart_priv *priv = container_of(t, struct brcmuart_priv, hrt); struct uart_port *p = priv->up; struct uart_8250_port *up = up_to_u8250p(p); + unsigned long cs_flags; unsigned int status; unsigned long flags; + bool is_console; if (priv->shutdown) return HRTIMER_NORESTART; @@ -846,12 +857,20 @@ static enum hrtimer_restart brcmuart_hrtimer_func(struct hrtimer *t) /* re-enable receive unless upper layer has disabled it */ if ((up->ier & (UART_IER_RLSI | UART_IER_RDI)) == (UART_IER_RLSI | UART_IER_RDI)) { + is_console = uart_console(p); + + if (is_console) + printk_cpu_sync_get_irqsave(cs_flags); + status = serial_port_in(p, UART_IER); status |= (UART_IER_RLSI | UART_IER_RDI); serial_port_out(p, UART_IER, status); status = serial_port_in(p, UART_MCR); status |= UART_MCR_RTS; serial_port_out(p, UART_MCR, status); + + if (is_console) + printk_cpu_sync_put_irqrestore(cs_flags); } spin_unlock_irqrestore(&p->lock, flags); return HRTIMER_NORESTART; diff --git a/drivers/tty/serial/8250/8250_core.c b/drivers/tty/serial/8250/8250_core.c index 81a5dab1a828..536f639ff56c 100644 --- a/drivers/tty/serial/8250/8250_core.c +++ b/drivers/tty/serial/8250/8250_core.c @@ -255,8 +255,11 @@ static void serial8250_timeout(struct timer_list *t) static void serial8250_backup_timeout(struct timer_list *t) { struct uart_8250_port *up = from_timer(up, t, timer); + struct uart_port *port = &up->port; unsigned int iir, ier = 0, lsr; + unsigned long cs_flags; unsigned long flags; + bool is_console; spin_lock_irqsave(&up->port.lock, flags); @@ -265,8 +268,16 @@ static void serial8250_backup_timeout(struct timer_list *t) * based handler. */ if (up->port.irq) { + is_console = uart_console(port); + + if (is_console) + printk_cpu_sync_get_irqsave(cs_flags); + ier = serial_in(up, UART_IER); serial_out(up, UART_IER, 0); + + if (is_console) + printk_cpu_sync_put_irqrestore(cs_flags); } iir = serial_in(up, UART_IIR); @@ -289,7 +300,7 @@ static void serial8250_backup_timeout(struct timer_list *t) serial8250_tx_chars(up); if (up->port.irq) - serial_out(up, UART_IER, ier); + serial8250_set_IER(up, ier); spin_unlock_irqrestore(&up->port.lock, flags); @@ -575,6 +586,14 @@ serial8250_register_ports(struct uart_driver *drv, struct device *dev) #ifdef CONFIG_SERIAL_8250_CONSOLE +static void univ8250_console_write_atomic(struct console *co, const char *s, + unsigned int count) +{ + struct uart_8250_port *up = &serial8250_ports[co->index]; + + serial8250_console_write_atomic(up, s, count); +} + static void univ8250_console_write(struct console *co, const char *s, unsigned int count) { @@ -668,6 +687,7 @@ static int univ8250_console_match(struct console *co, char *name, int idx, static struct console univ8250_console = { .name = "ttyS", + .write_atomic = univ8250_console_write_atomic, .write = univ8250_console_write, .device = uart_console_device, .setup = univ8250_console_setup, @@ -961,7 +981,7 @@ static void serial_8250_overrun_backoff_work(struct work_struct *work) spin_lock_irqsave(&port->lock, flags); up->ier |= UART_IER_RLSI | UART_IER_RDI; up->port.read_status_mask |= UART_LSR_DR; - serial_out(up, UART_IER, up->ier); + serial8250_set_IER(up, up->ier); spin_unlock_irqrestore(&port->lock, flags); } diff --git a/drivers/tty/serial/8250/8250_exar.c b/drivers/tty/serial/8250/8250_exar.c index b406cba10b0e..246c32c75a4c 100644 --- a/drivers/tty/serial/8250/8250_exar.c +++ b/drivers/tty/serial/8250/8250_exar.c @@ -189,6 +189,8 @@ static void xr17v35x_set_divisor(struct uart_port *p, unsigned int baud, static int xr17v35x_startup(struct uart_port *port) { + struct uart_8250_port *up = up_to_u8250p(port); + /* * First enable access to IER [7:5], ISR [5:4], FCR [5:4], * MCR [7:5] and MSR [7:0] @@ -199,7 +201,7 @@ static int xr17v35x_startup(struct uart_port *port) * Make sure all interrups are masked until initialization is * complete and the FIFOs are cleared */ - serial_port_out(port, UART_IER, 0); + serial8250_set_IER(up, 0); return serial8250_do_startup(port); } diff --git a/drivers/tty/serial/8250/8250_fsl.c b/drivers/tty/serial/8250/8250_fsl.c index 8adfaa183f77..eaf148245a10 100644 --- a/drivers/tty/serial/8250/8250_fsl.c +++ b/drivers/tty/serial/8250/8250_fsl.c @@ -58,7 +58,8 @@ int fsl8250_handle_irq(struct uart_port *port) if ((orig_lsr & UART_LSR_OE) && (up->overrun_backoff_time_ms > 0)) { unsigned long delay; - up->ier = port->serial_in(port, UART_IER); + up->ier = serial8250_in_IER(up); + if (up->ier & (UART_IER_RLSI | UART_IER_RDI)) { port->ops->stop_rx(port); } else { diff --git a/drivers/tty/serial/8250/8250_ingenic.c b/drivers/tty/serial/8250/8250_ingenic.c index 2b2f5d8d24b9..2b78e6c394fb 100644 --- a/drivers/tty/serial/8250/8250_ingenic.c +++ b/drivers/tty/serial/8250/8250_ingenic.c @@ -146,6 +146,7 @@ OF_EARLYCON_DECLARE(x1000_uart, "ingenic,x1000-uart", static void ingenic_uart_serial_out(struct uart_port *p, int offset, int value) { + struct uart_8250_port *up = up_to_u8250p(p); int ier; switch (offset) { @@ -167,7 +168,7 @@ static void ingenic_uart_serial_out(struct uart_port *p, int offset, int value) * If we have enabled modem status IRQs we should enable * modem mode. */ - ier = p->serial_in(p, UART_IER); + ier = serial8250_in_IER(up); if (ier & UART_IER_MSI) value |= UART_MCR_MDCE | UART_MCR_FCM; diff --git a/drivers/tty/serial/8250/8250_mtk.c b/drivers/tty/serial/8250/8250_mtk.c index fb1d5ec0940e..3e7203909d6a 100644 --- a/drivers/tty/serial/8250/8250_mtk.c +++ b/drivers/tty/serial/8250/8250_mtk.c @@ -222,12 +222,40 @@ static void mtk8250_shutdown(struct uart_port *port) static void mtk8250_disable_intrs(struct uart_8250_port *up, int mask) { - serial_out(up, UART_IER, serial_in(up, UART_IER) & (~mask)); + struct uart_port *port = &up->port; + unsigned long flags; + bool is_console; + int ier; + + is_console = uart_console(port); + + if (is_console) + printk_cpu_sync_get_irqsave(flags); + + ier = serial_in(up, UART_IER); + serial_out(up, UART_IER, ier & (~mask)); + + if (is_console) + printk_cpu_sync_put_irqrestore(flags); } static void mtk8250_enable_intrs(struct uart_8250_port *up, int mask) { - serial_out(up, UART_IER, serial_in(up, UART_IER) | mask); + struct uart_port *port = &up->port; + unsigned long flags; + bool is_console; + int ier; + + is_console = uart_console(port); + + if (is_console) + printk_cpu_sync_get_irqsave(flags); + + ier = serial_in(up, UART_IER); + serial_out(up, UART_IER, ier | mask); + + if (is_console) + printk_cpu_sync_put_irqrestore(flags); } static void mtk8250_set_flow_ctrl(struct uart_8250_port *up, int mode) diff --git a/drivers/tty/serial/8250/8250_omap.c b/drivers/tty/serial/8250/8250_omap.c index adc85e250822..92a8441ba756 100644 --- a/drivers/tty/serial/8250/8250_omap.c +++ b/drivers/tty/serial/8250/8250_omap.c @@ -328,7 +328,7 @@ static void omap8250_restore_regs(struct uart_8250_port *up) /* drop TCR + TLR access, we setup XON/XOFF later */ serial8250_out_MCR(up, mcr); - serial_out(up, UART_IER, up->ier); + serial8250_set_IER(up, up->ier); serial_out(up, UART_LCR, UART_LCR_CONF_MODE_B); serial_dl_write(up, priv->quot); @@ -518,7 +518,7 @@ static void omap_8250_pm(struct uart_port *port, unsigned int state, serial_out(up, UART_EFR, efr | UART_EFR_ECB); serial_out(up, UART_LCR, 0); - serial_out(up, UART_IER, (state != 0) ? UART_IERX_SLEEP : 0); + serial8250_set_IER(up, (state != 0) ? UART_IERX_SLEEP : 0); serial_out(up, UART_LCR, UART_LCR_CONF_MODE_B); serial_out(up, UART_EFR, efr); serial_out(up, UART_LCR, 0); @@ -641,7 +641,7 @@ static irqreturn_t omap8250_irq(int irq, void *dev_id) /* Synchronize UART_IER access against the console. */ spin_lock(&port->lock); - up->ier = port->serial_in(port, UART_IER); + up->ier = serial8250_in_IER(up); if (up->ier & (UART_IER_RLSI | UART_IER_RDI)) { port->ops->stop_rx(port); } else { @@ -701,7 +701,7 @@ static int omap_8250_startup(struct uart_port *port) goto err; up->ier = UART_IER_RLSI | UART_IER_RDI; - serial_out(up, UART_IER, up->ier); + serial8250_set_IER(up, up->ier); #ifdef CONFIG_PM up->capabilities |= UART_CAP_RPM; @@ -742,7 +742,7 @@ static void omap_8250_shutdown(struct uart_port *port) serial_out(up, UART_OMAP_EFR2, 0x0); up->ier = 0; - serial_out(up, UART_IER, 0); + serial8250_set_IER(up, 0); if (up->dma) serial8250_release_dma(up); @@ -790,7 +790,7 @@ static void omap_8250_unthrottle(struct uart_port *port) up->dma->rx_dma(up); up->ier |= UART_IER_RLSI | UART_IER_RDI; port->read_status_mask |= UART_LSR_DR; - serial_out(up, UART_IER, up->ier); + serial8250_set_IER(up, up->ier); spin_unlock_irqrestore(&port->lock, flags); pm_runtime_mark_last_busy(port->dev); @@ -881,7 +881,7 @@ static void __dma_rx_complete(void *param) __dma_rx_do_complete(p); if (!priv->throttled) { p->ier |= UART_IER_RLSI | UART_IER_RDI; - serial_out(p, UART_IER, p->ier); + serial8250_set_IER(p, p->ier); if (!(priv->habit & UART_HAS_EFR2)) omap_8250_rx_dma(p); } @@ -938,7 +938,7 @@ static int omap_8250_rx_dma(struct uart_8250_port *p) * callback to run. */ p->ier &= ~(UART_IER_RLSI | UART_IER_RDI); - serial_out(p, UART_IER, p->ier); + serial8250_set_IER(p, p->ier); } goto out; } @@ -1151,12 +1151,12 @@ static void am654_8250_handle_rx_dma(struct uart_8250_port *up, u8 iir, * periodic timeouts, re-enable interrupts. */ up->ier &= ~(UART_IER_RLSI | UART_IER_RDI); - serial_out(up, UART_IER, up->ier); + serial8250_set_IER(up, up->ier); omap_8250_rx_dma_flush(up); serial_in(up, UART_IIR); serial_out(up, UART_OMAP_EFR2, 0x0); up->ier |= UART_IER_RLSI | UART_IER_RDI; - serial_out(up, UART_IER, up->ier); + serial8250_set_IER(up, up->ier); } } diff --git a/drivers/tty/serial/8250/8250_port.c b/drivers/tty/serial/8250/8250_port.c index 38760bd6e0c2..b3719e8258f7 100644 --- a/drivers/tty/serial/8250/8250_port.c +++ b/drivers/tty/serial/8250/8250_port.c @@ -744,7 +744,7 @@ static void serial8250_set_sleep(struct uart_8250_port *p, int sleep) serial_out(p, UART_EFR, UART_EFR_ECB); serial_out(p, UART_LCR, 0); } - serial_out(p, UART_IER, sleep ? UART_IERX_SLEEP : 0); + serial8250_set_IER(p, sleep ? UART_IERX_SLEEP : 0); if (p->capabilities & UART_CAP_EFR) { serial_out(p, UART_LCR, UART_LCR_CONF_MODE_B); serial_out(p, UART_EFR, efr); @@ -755,12 +755,29 @@ static void serial8250_set_sleep(struct uart_8250_port *p, int sleep) serial8250_rpm_put(p); } -static void serial8250_clear_IER(struct uart_8250_port *up) +static unsigned int serial8250_clear_IER(struct uart_8250_port *up) { + struct uart_port *port = &up->port; + unsigned int clearval = 0; + unsigned long flags; + bool is_console; + unsigned int prior; + + is_console = uart_console(port); + if (up->capabilities & UART_CAP_UUE) - serial_out(up, UART_IER, UART_IER_UUE); - else - serial_out(up, UART_IER, 0); + clearval = UART_IER_UUE; + + if (is_console) + printk_cpu_sync_get_irqsave(flags); + + prior = serial_in(up, UART_IER); + serial_out(up, UART_IER, clearval); + + if (is_console) + printk_cpu_sync_put_irqrestore(flags); + + return prior; } #ifdef CONFIG_SERIAL_8250_RSA @@ -1026,8 +1043,11 @@ static int broken_efr(struct uart_8250_port *up) */ static void autoconfig_16550a(struct uart_8250_port *up) { + struct uart_port *port = &up->port; unsigned char status1, status2; unsigned int iersave; + unsigned long flags; + bool is_console; up->port.type = PORT_16550A; up->capabilities |= UART_CAP_FIFO; @@ -1139,6 +1159,11 @@ static void autoconfig_16550a(struct uart_8250_port *up) return; } + is_console = uart_console(port); + + if (is_console) + printk_cpu_sync_get_irqsave(flags); + /* * Try writing and reading the UART_IER_UUE bit (b6). * If it works, this is probably one of the Xscale platform's @@ -1174,6 +1199,9 @@ static void autoconfig_16550a(struct uart_8250_port *up) } serial_out(up, UART_IER, iersave); + if (is_console) + printk_cpu_sync_put_irqrestore(flags); + /* * We distinguish between 16550A and U6 16550A by counting * how many bytes are in the FIFO. @@ -1196,8 +1224,10 @@ static void autoconfig(struct uart_8250_port *up) unsigned char status1, scratch, scratch2, scratch3; unsigned char save_lcr, save_mcr; struct uart_port *port = &up->port; + unsigned long cs_flags; unsigned long flags; unsigned int old_capabilities; + bool is_console; if (!port->iobase && !port->mapbase && !port->membase) return; @@ -1215,6 +1245,11 @@ static void autoconfig(struct uart_8250_port *up) up->bugs = 0; if (!(port->flags & UPF_BUGGY_UART)) { + is_console = uart_console(port); + + if (is_console) + printk_cpu_sync_get_irqsave(cs_flags); + /* * Do a simple existence test first; if we fail this, * there's no point trying anything else. @@ -1244,6 +1279,10 @@ static void autoconfig(struct uart_8250_port *up) #endif scratch3 = serial_in(up, UART_IER) & 0x0f; serial_out(up, UART_IER, scratch); + + if (is_console) + printk_cpu_sync_put_irqrestore(cs_flags); + if (scratch2 != 0 || scratch3 != 0x0F) { /* * We failed; there's nothing here @@ -1367,7 +1406,9 @@ static void autoconfig_irq(struct uart_8250_port *up) unsigned char save_mcr, save_ier; unsigned char save_ICP = 0; unsigned int ICP = 0; + unsigned long flags; unsigned long irqs; + bool is_console; int irq; if (port->flags & UPF_FOURPORT) { @@ -1377,8 +1418,12 @@ static void autoconfig_irq(struct uart_8250_port *up) inb_p(ICP); } - if (uart_console(port)) + is_console = uart_console(port); + + if (is_console) { console_lock(); + printk_cpu_sync_get_irqsave(flags); + } /* forget possible initially masked and pending IRQ */ probe_irq_off(probe_irq_on()); @@ -1410,8 +1455,10 @@ static void autoconfig_irq(struct uart_8250_port *up) if (port->flags & UPF_FOURPORT) outb_p(save_ICP, ICP); - if (uart_console(port)) + if (is_console) { + printk_cpu_sync_put_irqrestore(flags); console_unlock(); + } port->irq = (irq > 0) ? irq : 0; } @@ -1424,7 +1471,7 @@ static void serial8250_stop_rx(struct uart_port *port) up->ier &= ~(UART_IER_RLSI | UART_IER_RDI); up->port.read_status_mask &= ~UART_LSR_DR; - serial_port_out(port, UART_IER, up->ier); + serial8250_set_IER(up, up->ier); serial8250_rpm_put(up); } @@ -1454,7 +1501,7 @@ void serial8250_em485_stop_tx(struct uart_8250_port *p) serial8250_clear_and_reinit_fifos(p); p->ier |= UART_IER_RLSI | UART_IER_RDI; - serial_port_out(&p->port, UART_IER, p->ier); + serial8250_set_IER(p, p->ier); } } EXPORT_SYMBOL_GPL(serial8250_em485_stop_tx); @@ -1703,7 +1750,7 @@ static void serial8250_disable_ms(struct uart_port *port) mctrl_gpio_disable_ms(up->gpios); up->ier &= ~UART_IER_MSI; - serial_port_out(port, UART_IER, up->ier); + serial8250_set_IER(up, up->ier); } static void serial8250_enable_ms(struct uart_port *port) @@ -1719,7 +1766,7 @@ static void serial8250_enable_ms(struct uart_port *port) up->ier |= UART_IER_MSI; serial8250_rpm_get(up); - serial_port_out(port, UART_IER, up->ier); + serial8250_set_IER(up, up->ier); serial8250_rpm_put(up); } @@ -2171,8 +2218,7 @@ static void serial8250_put_poll_char(struct uart_port *port, /* * First save the IER then disable the interrupts */ - ier = serial_port_in(port, UART_IER); - serial8250_clear_IER(up); + ier = serial8250_clear_IER(up); wait_for_xmitr(up, UART_LSR_BOTH_EMPTY); /* @@ -2185,7 +2231,7 @@ static void serial8250_put_poll_char(struct uart_port *port, * and restore the IER */ wait_for_xmitr(up, UART_LSR_BOTH_EMPTY); - serial_port_out(port, UART_IER, ier); + serial8250_set_IER(up, ier); serial8250_rpm_put(up); } @@ -2194,8 +2240,10 @@ static void serial8250_put_poll_char(struct uart_port *port, int serial8250_do_startup(struct uart_port *port) { struct uart_8250_port *up = up_to_u8250p(port); + unsigned long cs_flags; unsigned long flags; unsigned char iir; + bool is_console; int retval; u16 lsr; @@ -2216,7 +2264,7 @@ int serial8250_do_startup(struct uart_port *port) up->acr = 0; serial_port_out(port, UART_LCR, UART_LCR_CONF_MODE_B); serial_port_out(port, UART_EFR, UART_EFR_ECB); - serial_port_out(port, UART_IER, 0); + serial8250_set_IER(up, 0); serial_port_out(port, UART_LCR, 0); serial_icr_write(up, UART_CSR, 0); /* Reset the UART */ serial_port_out(port, UART_LCR, UART_LCR_CONF_MODE_B); @@ -2226,7 +2274,7 @@ int serial8250_do_startup(struct uart_port *port) if (port->type == PORT_DA830) { /* Reset the port */ - serial_port_out(port, UART_IER, 0); + serial8250_set_IER(up, 0); serial_port_out(port, UART_DA830_PWREMU_MGMT, 0); mdelay(10); @@ -2325,6 +2373,8 @@ int serial8250_do_startup(struct uart_port *port) if (retval) goto out; + is_console = uart_console(port); + if (port->irq && !(up->port.flags & UPF_NO_THRE_TEST)) { unsigned char iir1; @@ -2341,6 +2391,9 @@ int serial8250_do_startup(struct uart_port *port) */ spin_lock_irqsave(&port->lock, flags); + if (is_console) + printk_cpu_sync_get_irqsave(cs_flags); + wait_for_xmitr(up, UART_LSR_THRE); serial_port_out_sync(port, UART_IER, UART_IER_THRI); udelay(1); /* allow THRE to set */ @@ -2351,6 +2404,9 @@ int serial8250_do_startup(struct uart_port *port) iir = serial_port_in(port, UART_IIR); serial_port_out(port, UART_IER, 0); + if (is_console) + printk_cpu_sync_put_irqrestore(cs_flags); + spin_unlock_irqrestore(&port->lock, flags); if (port->irqflags & IRQF_SHARED) @@ -2405,10 +2461,14 @@ int serial8250_do_startup(struct uart_port *port) * Do a quick test to see if we receive an interrupt when we enable * the TX irq. */ + if (is_console) + printk_cpu_sync_get_irqsave(cs_flags); serial_port_out(port, UART_IER, UART_IER_THRI); lsr = serial_port_in(port, UART_LSR); iir = serial_port_in(port, UART_IIR); serial_port_out(port, UART_IER, 0); + if (is_console) + printk_cpu_sync_put_irqrestore(cs_flags); if (lsr & UART_LSR_TEMT && iir & UART_IIR_NO_INT) { if (!(up->bugs & UART_BUG_TXEN)) { @@ -2440,7 +2500,7 @@ int serial8250_do_startup(struct uart_port *port) if (up->dma) { const char *msg = NULL; - if (uart_console(port)) + if (is_console) msg = "forbid DMA for kernel console"; else if (serial8250_request_dma(up)) msg = "failed to request DMA"; @@ -2491,7 +2551,7 @@ void serial8250_do_shutdown(struct uart_port *port) */ spin_lock_irqsave(&port->lock, flags); up->ier = 0; - serial_port_out(port, UART_IER, 0); + serial8250_set_IER(up, 0); spin_unlock_irqrestore(&port->lock, flags); synchronize_irq(port->irq); @@ -2853,7 +2913,7 @@ serial8250_do_set_termios(struct uart_port *port, struct ktermios *termios, if (up->capabilities & UART_CAP_RTOIE) up->ier |= UART_IER_RTOIE; - serial_port_out(port, UART_IER, up->ier); + serial8250_set_IER(up, up->ier); if (up->capabilities & UART_CAP_EFR) { unsigned char efr = 0; @@ -3318,7 +3378,7 @@ EXPORT_SYMBOL_GPL(serial8250_set_defaults); #ifdef CONFIG_SERIAL_8250_CONSOLE -static void serial8250_console_putchar(struct uart_port *port, unsigned char ch) +static void serial8250_console_putchar_locked(struct uart_port *port, unsigned char ch) { struct uart_8250_port *up = up_to_u8250p(port); @@ -3326,6 +3386,18 @@ static void serial8250_console_putchar(struct uart_port *port, unsigned char ch) serial_port_out(port, UART_TX, ch); } +static void serial8250_console_putchar(struct uart_port *port, unsigned char ch) +{ + struct uart_8250_port *up = up_to_u8250p(port); + unsigned long flags; + + wait_for_xmitr(up, UART_LSR_THRE); + + printk_cpu_sync_get_irqsave(flags); + serial8250_console_putchar_locked(port, ch); + printk_cpu_sync_put_irqrestore(flags); +} + /* * Restore serial console when h/w power-off detected */ @@ -3352,6 +3424,32 @@ static void serial8250_console_restore(struct uart_8250_port *up) serial8250_out_MCR(up, up->mcr | UART_MCR_DTR | UART_MCR_RTS); } +void serial8250_console_write_atomic(struct uart_8250_port *up, + const char *s, unsigned int count) +{ + struct uart_port *port = &up->port; + unsigned long flags; + unsigned int ier; + + printk_cpu_sync_get_irqsave(flags); + + touch_nmi_watchdog(); + + ier = serial8250_clear_IER(up); + + if (atomic_fetch_inc(&up->console_printing)) { + uart_console_write(port, "\n", 1, + serial8250_console_putchar_locked); + } + uart_console_write(port, s, count, serial8250_console_putchar_locked); + atomic_dec(&up->console_printing); + + wait_for_xmitr(up, UART_LSR_BOTH_EMPTY); + serial8250_set_IER(up, ier); + + printk_cpu_sync_put_irqrestore(flags); +} + /* * Print a string to the serial port using the device FIFO * @@ -3397,20 +3495,15 @@ void serial8250_console_write(struct uart_8250_port *up, const char *s, struct uart_port *port = &up->port; unsigned long flags; unsigned int ier, use_fifo; - int locked = 1; touch_nmi_watchdog(); - if (oops_in_progress) - locked = spin_trylock_irqsave(&port->lock, flags); - else - spin_lock_irqsave(&port->lock, flags); + spin_lock_irqsave(&port->lock, flags); /* * First save the IER then disable the interrupts */ - ier = serial_port_in(port, UART_IER); - serial8250_clear_IER(up); + ier = serial8250_clear_IER(up); /* check scratch reg to see if port powered off during system sleep */ if (up->canary && (up->canary != serial_port_in(port, UART_SCR))) { @@ -3444,10 +3537,12 @@ void serial8250_console_write(struct uart_8250_port *up, const char *s, */ !(up->port.flags & UPF_CONS_FLOW); + atomic_inc(&up->console_printing); if (likely(use_fifo)) serial8250_console_fifo_write(up, s, count); else uart_console_write(port, s, count, serial8250_console_putchar); + atomic_dec(&up->console_printing); /* * Finally, wait for transmitter to become empty @@ -3460,8 +3555,7 @@ void serial8250_console_write(struct uart_8250_port *up, const char *s, if (em485->tx_stopped) up->rs485_stop_tx(up); } - - serial_port_out(port, UART_IER, ier); + serial8250_set_IER(up, ier); /* * The receive handling will happen properly because the @@ -3473,8 +3567,7 @@ void serial8250_console_write(struct uart_8250_port *up, const char *s, if (up->msr_saved_flags) serial8250_modem_status(up); - if (locked) - spin_unlock_irqrestore(&port->lock, flags); + spin_unlock_irqrestore(&port->lock, flags); } static unsigned int probe_baud(struct uart_port *port) @@ -3494,6 +3587,7 @@ static unsigned int probe_baud(struct uart_port *port) int serial8250_console_setup(struct uart_port *port, char *options, bool probe) { + struct uart_8250_port *up = up_to_u8250p(port); int baud = 9600; int bits = 8; int parity = 'n'; @@ -3503,6 +3597,8 @@ int serial8250_console_setup(struct uart_port *port, char *options, bool probe) if (!port->iobase && !port->membase) return -ENODEV; + atomic_set(&up->console_printing, 0); + if (options) uart_parse_options(options, &baud, &parity, &bits, &flow); else if (probe) diff --git a/drivers/tty/serial/8250/Kconfig b/drivers/tty/serial/8250/Kconfig index 583a340f9934..1f31320820a6 100644 --- a/drivers/tty/serial/8250/Kconfig +++ b/drivers/tty/serial/8250/Kconfig @@ -9,6 +9,7 @@ config SERIAL_8250 depends on !S390 select SERIAL_CORE select SERIAL_MCTRL_GPIO if GPIOLIB + select HAVE_ATOMIC_CONSOLE help This selects whether you want to include the driver for the standard serial ports. The standard answer is Y. People who might say N diff --git a/drivers/tty/serial/amba-pl011.c b/drivers/tty/serial/amba-pl011.c index 0a1cc36f93aa..51a8176050f8 100644 --- a/drivers/tty/serial/amba-pl011.c +++ b/drivers/tty/serial/amba-pl011.c @@ -2320,18 +2320,24 @@ pl011_console_write(struct console *co, const char *s, unsigned int count) { struct uart_amba_port *uap = amba_ports[co->index]; unsigned int old_cr = 0, new_cr; - unsigned long flags; + unsigned long flags = 0; int locked = 1; clk_enable(uap->clk); - local_irq_save(flags); + /* + * local_irq_save(flags); + * + * This local_irq_save() is nonsense. If we come in via sysrq + * handling then interrupts are already disabled. Aside of + * that the port.sysrq check is racy on SMP regardless. + */ if (uap->port.sysrq) locked = 0; else if (oops_in_progress) - locked = spin_trylock(&uap->port.lock); + locked = spin_trylock_irqsave(&uap->port.lock, flags); else - spin_lock(&uap->port.lock); + spin_lock_irqsave(&uap->port.lock, flags); /* * First save the CR then disable the interrupts @@ -2357,8 +2363,7 @@ pl011_console_write(struct console *co, const char *s, unsigned int count) pl011_write(old_cr, uap, REG_CR); if (locked) - spin_unlock(&uap->port.lock); - local_irq_restore(flags); + spin_unlock_irqrestore(&uap->port.lock, flags); clk_disable(uap->clk); } diff --git a/drivers/tty/serial/omap-serial.c b/drivers/tty/serial/omap-serial.c index 7d0d2718ef59..aa216fdbcb1d 100644 --- a/drivers/tty/serial/omap-serial.c +++ b/drivers/tty/serial/omap-serial.c @@ -1241,13 +1241,10 @@ serial_omap_console_write(struct console *co, const char *s, unsigned int ier; int locked = 1; - local_irq_save(flags); - if (up->port.sysrq) - locked = 0; - else if (oops_in_progress) - locked = spin_trylock(&up->port.lock); + if (up->port.sysrq || oops_in_progress) + locked = spin_trylock_irqsave(&up->port.lock, flags); else - spin_lock(&up->port.lock); + spin_lock_irqsave(&up->port.lock, flags); /* * First save the IER then disable the interrupts @@ -1274,8 +1271,7 @@ serial_omap_console_write(struct console *co, const char *s, check_modem_status(up); if (locked) - spin_unlock(&up->port.lock); - local_irq_restore(flags); + spin_unlock_irqrestore(&up->port.lock, flags); } static int __init diff --git a/drivers/tty/sysrq.c b/drivers/tty/sysrq.c index d2b2720db6ca..18e623325887 100644 --- a/drivers/tty/sysrq.c +++ b/drivers/tty/sysrq.c @@ -581,6 +581,7 @@ void __handle_sysrq(int key, bool check_mask) rcu_sysrq_start(); rcu_read_lock(); + printk_prefer_direct_enter(); /* * Raise the apparent loglevel to maximum so that the sysrq header * is shown to provide the user with positive feedback. We do not @@ -622,6 +623,7 @@ void __handle_sysrq(int key, bool check_mask) pr_cont("\n"); console_loglevel = orig_log_level; } + printk_prefer_direct_exit(); rcu_read_unlock(); rcu_sysrq_end(); diff --git a/drivers/vdpa/vdpa_user/iova_domain.h b/drivers/vdpa/vdpa_user/iova_domain.h index 4e0e50e7ac15..173e979b84a9 100644 --- a/drivers/vdpa/vdpa_user/iova_domain.h +++ b/drivers/vdpa/vdpa_user/iova_domain.h @@ -14,7 +14,6 @@ #include #include #include -#include #define IOVA_START_PFN 1 diff --git a/drivers/virtio/Kconfig b/drivers/virtio/Kconfig index 106b47ab9f4f..d0b8375d10ec 100644 --- a/drivers/virtio/Kconfig +++ b/drivers/virtio/Kconfig @@ -145,6 +145,14 @@ config VIRTIO_INPUT If unsure, say M. +config VIRTIO_TRANS + tristate "VirtIO transfer test driver" + depends on VIRTIO + help + This driver is for VirtIO transfer performance test + between front-end and back-end driver. + If unsure, say N. + config VIRTIO_MMIO tristate "Platform bus driver for memory mapped virtio devices" depends on HAS_IOMEM && HAS_DMA diff --git a/drivers/virtio/Makefile b/drivers/virtio/Makefile index 7960f6493913..bdf34908610c 100644 --- a/drivers/virtio/Makefile +++ b/drivers/virtio/Makefile @@ -13,3 +13,4 @@ obj-$(CONFIG_VIRTIO_INPUT) += virtio_input.o obj-$(CONFIG_VIRTIO_VDPA) += virtio_vdpa.o obj-$(CONFIG_VIRTIO_MEM) += virtio_mem.o obj-$(CONFIG_VIRTIO_DMA_SHARED_BUFFER) += virtio_dma_buf.o +obj-$(CONFIG_VIRTIO_TRANS) += virtio_trans.o diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c index dec3cba88458..c65a22c6631d 100644 --- a/drivers/virtio/virtio_mmio.c +++ b/drivers/virtio/virtio_mmio.c @@ -3,6 +3,7 @@ * Virtio memory mapped device driver * * Copyright 2011-2014, ARM Ltd. + * Copyright 2022-2023 NXP * * This module allows virtio devices to be used over a virtual, memory mapped * platform device. @@ -60,7 +61,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -69,7 +72,7 @@ #include #include #include - +#include /* The alignment to use between consumer and producer parts of vring. @@ -81,6 +84,13 @@ #define to_virtio_mmio_device(_plat_dev) \ container_of(_plat_dev, struct virtio_mmio_device, vdev) +struct virtio_mmio_wr_op { + uint64_t phy_base; + uint32_t offset; + uint32_t value; + uint8_t len; +}; + struct virtio_mmio_device { struct virtio_device vdev; struct platform_device *pdev; @@ -91,6 +101,16 @@ struct virtio_mmio_device { /* a list of queues so we can dispatch IRQs */ spinlock_t lock; struct list_head virtqueues; + + /* Hypervisor_less virtio */ + bool is_hypervisor_less; + phys_addr_t phys; + struct virtio_mmio_wr_op *wr_ops; + dma_addr_t wr_ops_phys; + struct mbox_client mbox_wr_cl; + struct mbox_client mbox_irq_cl; + struct mbox_chan *mmio_wr_ch; + struct mbox_chan *mmio_irq_ch; }; struct virtio_mmio_vq_info { @@ -101,7 +121,72 @@ struct virtio_mmio_vq_info { struct list_head node; }; +static void vmd_write(struct virtio_mmio_device *vm_dev, u32 val, u32 off, u8 len) +{ + void __iomem *addr = vm_dev->base + off; + struct virtio_mmio_wr_op *wr_ops = vm_dev->wr_ops; + int timeout; + int ret; + + switch (len) { + case 1: + writeb(val, addr); + break; + case 2: + writew(val, addr); + break; + case 4: + writel(val, addr); + break; + default: + return; + } + + if (vm_dev->is_hypervisor_less) { + + timeout = 1000000; + writel(0, vm_dev->base + VIRTIO_MMIO_WD_STATUS); + + wr_ops->phy_base = (uint64_t)vm_dev->phys; + wr_ops->offset = off; + wr_ops->value = val; + wr_ops->len = len; + + ret = mbox_send_message(vm_dev->mmio_wr_ch, &vm_dev->wr_ops_phys); + if (ret < 0) + dev_err(&vm_dev->vdev.dev, "mmio write error: base: %llx offset: %x, val: %x, len: %d\n", + wr_ops->phy_base, wr_ops->offset, + wr_ops->value, wr_ops->len); + + /* Unnecessary to sync for virtqueue notification */ + if (off == VIRTIO_MMIO_QUEUE_NOTIFY) + return; + + /* poll backend write operation complete */ + while (timeout-- && !readl(vm_dev->base + VIRTIO_MMIO_WD_STATUS)) + udelay(1); + + if (timeout < 0) + dev_err(&vm_dev->vdev.dev, "mmio write timeout: base: %llx offset: %x, val: %x, len: %d\n", + wr_ops->phy_base, wr_ops->offset, + wr_ops->value, wr_ops->len); + } +} + +static void vmd_writel(struct virtio_mmio_device *vm_dev, u32 val, u32 off) +{ + vmd_write(vm_dev, val, off, 4); +} + +static void vmd_writew(struct virtio_mmio_device *vm_dev, u32 val, u32 off) +{ + vmd_write(vm_dev, val, off, 2); +} +static void vmd_writeb(struct virtio_mmio_device *vm_dev, u32 val, u32 off) +{ + vmd_write(vm_dev, val, off, 1); +} /* Configuration interface */ @@ -110,11 +195,11 @@ static u64 vm_get_features(struct virtio_device *vdev) struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev); u64 features; - writel(1, vm_dev->base + VIRTIO_MMIO_DEVICE_FEATURES_SEL); + vmd_writel(vm_dev, 1, VIRTIO_MMIO_DEVICE_FEATURES_SEL); features = readl(vm_dev->base + VIRTIO_MMIO_DEVICE_FEATURES); features <<= 32; - writel(0, vm_dev->base + VIRTIO_MMIO_DEVICE_FEATURES_SEL); + vmd_writel(vm_dev, 0, VIRTIO_MMIO_DEVICE_FEATURES_SEL); features |= readl(vm_dev->base + VIRTIO_MMIO_DEVICE_FEATURES); return features; @@ -134,13 +219,13 @@ static int vm_finalize_features(struct virtio_device *vdev) return -EINVAL; } - writel(1, vm_dev->base + VIRTIO_MMIO_DRIVER_FEATURES_SEL); - writel((u32)(vdev->features >> 32), - vm_dev->base + VIRTIO_MMIO_DRIVER_FEATURES); + vmd_writel(vm_dev, 1, VIRTIO_MMIO_DRIVER_FEATURES_SEL); + vmd_writel(vm_dev, (u32)(vdev->features >> 32), + VIRTIO_MMIO_DRIVER_FEATURES); - writel(0, vm_dev->base + VIRTIO_MMIO_DRIVER_FEATURES_SEL); - writel((u32)vdev->features, - vm_dev->base + VIRTIO_MMIO_DRIVER_FEATURES); + vmd_writel(vm_dev, 0, VIRTIO_MMIO_DRIVER_FEATURES_SEL); + vmd_writel(vm_dev, (u32)vdev->features, + VIRTIO_MMIO_DRIVER_FEATURES); return 0; } @@ -191,17 +276,13 @@ static void vm_set(struct virtio_device *vdev, unsigned int offset, const void *buf, unsigned int len) { struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev); - void __iomem *base = vm_dev->base + VIRTIO_MMIO_CONFIG; + u32 base = VIRTIO_MMIO_CONFIG; u8 b; __le16 w; __le32 l; if (vm_dev->version == 1) { - const u8 *ptr = buf; - int i; - - for (i = 0; i < len; i++) - writeb(ptr[i], base + offset + i); + vmd_writel(vm_dev, *(u32 *)buf, base + offset); return; } @@ -209,21 +290,21 @@ static void vm_set(struct virtio_device *vdev, unsigned int offset, switch (len) { case 1: memcpy(&b, buf, sizeof b); - writeb(b, base + offset); + vmd_writeb(vm_dev, b, base + offset); break; case 2: memcpy(&w, buf, sizeof w); - writew(le16_to_cpu(w), base + offset); + vmd_writew(vm_dev, le16_to_cpu(w), base + offset); break; case 4: memcpy(&l, buf, sizeof l); - writel(le32_to_cpu(l), base + offset); + vmd_writel(vm_dev, le32_to_cpu(l), base + offset); break; case 8: memcpy(&l, buf, sizeof l); - writel(le32_to_cpu(l), base + offset); + vmd_writel(vm_dev, le32_to_cpu(l), base + offset); memcpy(&l, buf + sizeof l, sizeof l); - writel(le32_to_cpu(l), base + offset + sizeof l); + vmd_writel(vm_dev, le32_to_cpu(l), base + offset + sizeof l); break; default: BUG(); @@ -259,7 +340,7 @@ static void vm_set_status(struct virtio_device *vdev, u8 status) * that the cache coherent memory writes have completed * before writing to the MMIO region. */ - writel(status, vm_dev->base + VIRTIO_MMIO_STATUS); + vmd_writel(vm_dev, status, VIRTIO_MMIO_STATUS); } static void vm_reset(struct virtio_device *vdev) @@ -267,7 +348,7 @@ static void vm_reset(struct virtio_device *vdev) struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev); /* 0 status means a reset. */ - writel(0, vm_dev->base + VIRTIO_MMIO_STATUS); + vmd_writel(vm_dev, 0, VIRTIO_MMIO_STATUS); } @@ -281,10 +362,40 @@ static bool vm_notify(struct virtqueue *vq) /* We write the queue's selector into the notification register to * signal the other end */ - writel(vq->index, vm_dev->base + VIRTIO_MMIO_QUEUE_NOTIFY); + vmd_writel(vm_dev, vq->index, VIRTIO_MMIO_QUEUE_NOTIFY); return true; } +/* Notify all virtqueues on an interrupt. */ +static void virtio_mmio_irq_callback(struct mbox_client *c, void *msg) +{ + struct virtio_mmio_device *vm_dev = container_of(c, + struct virtio_mmio_device, mbox_irq_cl); + struct virtio_mmio_vq_info *info; + unsigned long status; + unsigned long flags; + + /* Read and acknowledge interrupts */ + status = readl(vm_dev->base + VIRTIO_MMIO_INTERRUPT_STATUS); + + /* + * For Hypervisor-less virtio, write ACK without notification, since + * in mailbox RX channel callback function sometimes the timeout occur + * calling the vmd_writel(). + */ + writel(status, vm_dev->base + VIRTIO_MMIO_INTERRUPT_ACK); + + if (unlikely(status & VIRTIO_MMIO_INT_CONFIG)) + virtio_config_changed(&vm_dev->vdev); + + if (likely(status & VIRTIO_MMIO_INT_VRING)) { + spin_lock_irqsave(&vm_dev->lock, flags); + list_for_each_entry(info, &vm_dev->virtqueues, node) + vring_interrupt(0, info->vq); + spin_unlock_irqrestore(&vm_dev->lock, flags); + } +} + /* Notify all virtqueues on an interrupt. */ static irqreturn_t vm_interrupt(int irq, void *opaque) { @@ -296,7 +407,7 @@ static irqreturn_t vm_interrupt(int irq, void *opaque) /* Read and acknowledge interrupts */ status = readl(vm_dev->base + VIRTIO_MMIO_INTERRUPT_STATUS); - writel(status, vm_dev->base + VIRTIO_MMIO_INTERRUPT_ACK); + vmd_writel(vm_dev, status, VIRTIO_MMIO_INTERRUPT_ACK); if (unlikely(status & VIRTIO_MMIO_INT_CONFIG)) { virtio_config_changed(&vm_dev->vdev); @@ -327,11 +438,11 @@ static void vm_del_vq(struct virtqueue *vq) spin_unlock_irqrestore(&vm_dev->lock, flags); /* Select and deactivate the queue */ - writel(index, vm_dev->base + VIRTIO_MMIO_QUEUE_SEL); + vmd_writel(vm_dev, index, VIRTIO_MMIO_QUEUE_SEL); if (vm_dev->version == 1) { - writel(0, vm_dev->base + VIRTIO_MMIO_QUEUE_PFN); + vmd_writel(vm_dev, 0, VIRTIO_MMIO_QUEUE_PFN); } else { - writel(0, vm_dev->base + VIRTIO_MMIO_QUEUE_READY); + vmd_writel(vm_dev, 0, VIRTIO_MMIO_QUEUE_READY); WARN_ON(readl(vm_dev->base + VIRTIO_MMIO_QUEUE_READY)); } @@ -348,7 +459,10 @@ static void vm_del_vqs(struct virtio_device *vdev) list_for_each_entry_safe(vq, n, &vdev->vqs, list) vm_del_vq(vq); - free_irq(platform_get_irq(vm_dev->pdev, 0), vm_dev); + if (vm_dev->is_hypervisor_less) + mbox_free_channel(vm_dev->mmio_irq_ch); + else + free_irq(platform_get_irq(vm_dev->pdev, 0), vm_dev); } static void vm_synchronize_cbs(struct virtio_device *vdev) @@ -373,7 +487,7 @@ static struct virtqueue *vm_setup_vq(struct virtio_device *vdev, unsigned int in return NULL; /* Select the queue we're interested in */ - writel(index, vm_dev->base + VIRTIO_MMIO_QUEUE_SEL); + vmd_writel(vm_dev, index, VIRTIO_MMIO_QUEUE_SEL); /* Queue shouldn't already be set up. */ if (readl(vm_dev->base + (vm_dev->version == 1 ? @@ -406,7 +520,7 @@ static struct virtqueue *vm_setup_vq(struct virtio_device *vdev, unsigned int in vq->num_max = num; /* Activate the queue */ - writel(virtqueue_get_vring_size(vq), vm_dev->base + VIRTIO_MMIO_QUEUE_NUM); + vmd_writel(vm_dev, virtqueue_get_vring_size(vq), VIRTIO_MMIO_QUEUE_NUM); if (vm_dev->version == 1) { u64 q_pfn = virtqueue_get_desc_addr(vq) >> PAGE_SHIFT; @@ -423,27 +537,27 @@ static struct virtqueue *vm_setup_vq(struct virtio_device *vdev, unsigned int in goto error_bad_pfn; } - writel(PAGE_SIZE, vm_dev->base + VIRTIO_MMIO_QUEUE_ALIGN); - writel(q_pfn, vm_dev->base + VIRTIO_MMIO_QUEUE_PFN); + vmd_writel(vm_dev, PAGE_SIZE, VIRTIO_MMIO_QUEUE_ALIGN); + vmd_writel(vm_dev, q_pfn, VIRTIO_MMIO_QUEUE_PFN); } else { u64 addr; addr = virtqueue_get_desc_addr(vq); - writel((u32)addr, vm_dev->base + VIRTIO_MMIO_QUEUE_DESC_LOW); - writel((u32)(addr >> 32), - vm_dev->base + VIRTIO_MMIO_QUEUE_DESC_HIGH); + vmd_writel(vm_dev, (u32)addr, VIRTIO_MMIO_QUEUE_DESC_LOW); + vmd_writel(vm_dev, (u32)(addr >> 32), + VIRTIO_MMIO_QUEUE_DESC_HIGH); addr = virtqueue_get_avail_addr(vq); - writel((u32)addr, vm_dev->base + VIRTIO_MMIO_QUEUE_AVAIL_LOW); - writel((u32)(addr >> 32), - vm_dev->base + VIRTIO_MMIO_QUEUE_AVAIL_HIGH); + vmd_writel(vm_dev, (u32)addr, VIRTIO_MMIO_QUEUE_AVAIL_LOW); + vmd_writel(vm_dev, (u32)(addr >> 32), + VIRTIO_MMIO_QUEUE_AVAIL_HIGH); addr = virtqueue_get_used_addr(vq); - writel((u32)addr, vm_dev->base + VIRTIO_MMIO_QUEUE_USED_LOW); - writel((u32)(addr >> 32), - vm_dev->base + VIRTIO_MMIO_QUEUE_USED_HIGH); + vmd_writel(vm_dev, (u32)addr, VIRTIO_MMIO_QUEUE_USED_LOW); + vmd_writel(vm_dev, (u32)(addr >> 32), + VIRTIO_MMIO_QUEUE_USED_HIGH); - writel(1, vm_dev->base + VIRTIO_MMIO_QUEUE_READY); + vmd_writel(vm_dev, 1, VIRTIO_MMIO_QUEUE_READY); } vq->priv = info; @@ -459,9 +573,9 @@ static struct virtqueue *vm_setup_vq(struct virtio_device *vdev, unsigned int in vring_del_virtqueue(vq); error_new_virtqueue: if (vm_dev->version == 1) { - writel(0, vm_dev->base + VIRTIO_MMIO_QUEUE_PFN); + vmd_writel(vm_dev, 0, VIRTIO_MMIO_QUEUE_PFN); } else { - writel(0, vm_dev->base + VIRTIO_MMIO_QUEUE_READY); + vmd_writel(vm_dev, 0, VIRTIO_MMIO_QUEUE_READY); WARN_ON(readl(vm_dev->base + VIRTIO_MMIO_QUEUE_READY)); } kfree(info); @@ -470,6 +584,28 @@ static struct virtqueue *vm_setup_vq(struct virtio_device *vdev, unsigned int in return ERR_PTR(err); } +static int virtio_mmio_irq_channel_init(struct virtio_mmio_device *vm_dev) +{ + struct platform_device *pdev = vm_dev->pdev; + struct device *dev = &pdev->dev; + struct mbox_client *cl; + int ret = 0; + + cl = &vm_dev->mbox_irq_cl; + cl->dev = dev; + cl->rx_callback = virtio_mmio_irq_callback; + + vm_dev->mmio_irq_ch = mbox_request_channel_byname(cl, "mmioirq"); + if (IS_ERR(vm_dev->mmio_irq_ch)) { + ret = PTR_ERR(vm_dev->mmio_irq_ch); + dev_err(cl->dev, "failed to request mbox irq chan, ret %d\n", + ret); + return ret; + } + + return ret; +} + static int vm_find_vqs(struct virtio_device *vdev, unsigned int nvqs, struct virtqueue *vqs[], vq_callback_t *callbacks[], @@ -478,16 +614,24 @@ static int vm_find_vqs(struct virtio_device *vdev, unsigned int nvqs, struct irq_affinity *desc) { struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev); - int irq = platform_get_irq(vm_dev->pdev, 0); + int irq; int i, err, queue_idx = 0; - if (irq < 0) - return irq; + if (vm_dev->is_hypervisor_less) { + err = virtio_mmio_irq_channel_init(vm_dev); + if (err) + return err; + } else { + irq = platform_get_irq(vm_dev->pdev, 0); + if (irq < 0) + return irq; - err = request_irq(irq, vm_interrupt, IRQF_SHARED, - dev_name(&vdev->dev), vm_dev); - if (err) - return err; + err = request_irq(irq, vm_interrupt, IRQF_SHARED, + dev_name(&vdev->dev), vm_dev); + + if (err) + return err; + } if (of_property_read_bool(vm_dev->pdev->dev.of_node, "wakeup-source")) enable_irq_wake(irq); @@ -523,7 +667,7 @@ static bool vm_get_shm_region(struct virtio_device *vdev, u64 len, addr; /* Select the region we're interested in */ - writel(id, vm_dev->base + VIRTIO_MMIO_SHM_SEL); + vmd_writel(vm_dev, id, VIRTIO_MMIO_SHM_SEL); /* Read the region size */ len = (u64) readl(vm_dev->base + VIRTIO_MMIO_SHM_LEN_LOW); @@ -594,6 +738,27 @@ static void virtio_mmio_release_dev(struct device *_d) kfree(vm_dev); } +static int virtio_mmio_mbox_channel_init(struct virtio_mmio_device *vm_dev) +{ + struct platform_device *pdev = vm_dev->pdev; + struct device *dev = &pdev->dev; + struct mbox_client *cl; + int ret = 0; + + cl = &vm_dev->mbox_wr_cl; + cl->dev = dev; + + vm_dev->mmio_wr_ch = mbox_request_channel_byname(cl, "mmiowr"); + if (IS_ERR(vm_dev->mmio_wr_ch)) { + ret = PTR_ERR(vm_dev->mmio_wr_ch); + dev_err(cl->dev, "failed to request mbox chan mmio, ret %d\n", + ret); + return ret; + } + + return ret; +} + /* Platform device */ static int virtio_mmio_probe(struct platform_device *pdev) @@ -601,11 +766,19 @@ static int virtio_mmio_probe(struct platform_device *pdev) struct virtio_mmio_device *vm_dev; unsigned long magic; int rc; + struct resource *res; vm_dev = kzalloc(sizeof(*vm_dev), GFP_KERNEL); if (!vm_dev) return -ENOMEM; + if (of_property_read_bool(pdev->dev.of_node, "hypervisor_less")) { + vm_dev->is_hypervisor_less = true; + rc = of_reserved_mem_device_init(&pdev->dev); + if (rc) + dev_info(&pdev->dev, "Device specific DMA pool is not available\n"); + } + vm_dev->vdev.dev.parent = &pdev->dev; vm_dev->vdev.dev.release = virtio_mmio_release_dev; vm_dev->vdev.config = &virtio_mmio_config_ops; @@ -613,15 +786,19 @@ static int virtio_mmio_probe(struct platform_device *pdev) INIT_LIST_HEAD(&vm_dev->virtqueues); spin_lock_init(&vm_dev->lock); - vm_dev->base = devm_platform_ioremap_resource(pdev, 0); - if (IS_ERR(vm_dev->base)) - return PTR_ERR(vm_dev->base); + vm_dev->base = devm_platform_get_and_ioremap_resource(pdev, 0, &res); + + if (IS_ERR(vm_dev->base)) { + rc = PTR_ERR(vm_dev->base); + goto err; + } /* Check magic value */ magic = readl(vm_dev->base + VIRTIO_MMIO_MAGIC_VALUE); if (magic != ('v' | 'i' << 8 | 'r' << 16 | 't' << 24)) { dev_warn(&pdev->dev, "Wrong magic value 0x%08lx!\n", magic); - return -ENODEV; + rc = -ENODEV; + goto err; } /* Check device version */ @@ -629,7 +806,8 @@ static int virtio_mmio_probe(struct platform_device *pdev) if (vm_dev->version < 1 || vm_dev->version > 2) { dev_err(&pdev->dev, "Version %ld not supported!\n", vm_dev->version); - return -ENXIO; + rc = -ENXIO; + goto err; } vm_dev->vdev.id.device = readl(vm_dev->base + VIRTIO_MMIO_DEVICE_ID); @@ -638,12 +816,25 @@ static int virtio_mmio_probe(struct platform_device *pdev) * virtio-mmio device with an ID 0 is a (dummy) placeholder * with no function. End probing now with no error reported. */ - return -ENODEV; + rc = -ENODEV; + goto err; } vm_dev->vdev.id.vendor = readl(vm_dev->base + VIRTIO_MMIO_VENDOR_ID); + if (vm_dev->is_hypervisor_less) { + vm_dev->phys = res->start; + + vm_dev->wr_ops = vm_dev->base + VIRTIO_MMIO_RW_OPS_MEM_OFFSET; + vm_dev->wr_ops_phys = res->start + VIRTIO_MMIO_RW_OPS_MEM_OFFSET; + + /* Initialize mailbox mmio channel. */ + rc = virtio_mmio_mbox_channel_init(vm_dev); + if (rc) + goto err; + } + if (vm_dev->version == 1) { - writel(PAGE_SIZE, vm_dev->base + VIRTIO_MMIO_GUEST_PAGE_SIZE); + vmd_writel(vm_dev, PAGE_SIZE, VIRTIO_MMIO_GUEST_PAGE_SIZE); rc = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64)); /* @@ -668,11 +859,20 @@ static int virtio_mmio_probe(struct platform_device *pdev) put_device(&vm_dev->vdev.dev); return rc; + +err: + if (vm_dev->is_hypervisor_less) + of_reserved_mem_device_release(&pdev->dev); + + return rc; } static int virtio_mmio_remove(struct platform_device *pdev) { struct virtio_mmio_device *vm_dev = platform_get_drvdata(pdev); + + if (vm_dev->is_hypervisor_less) + mbox_free_channel(vm_dev->mmio_wr_ch); unregister_virtio_device(&vm_dev->vdev); return 0; diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index 7d320f799ca1..dae64a48bf9e 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -572,8 +572,6 @@ static inline int virtqueue_add_split(struct virtqueue *_vq, /* FIXME: for historical reasons, we force a notify here if * there are outgoing parts to the buffer. Presumably the * host should service the ring ASAP. */ - if (out_sgs) - vq->notify(&vq->vq); if (indirect) kfree(desc); END_USE(vq); diff --git a/drivers/virtio/virtio_trans.c b/drivers/virtio/virtio_trans.c new file mode 100644 index 000000000000..4fe106af4162 --- /dev/null +++ b/drivers/virtio/virtio_trans.c @@ -0,0 +1,793 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright 2022-2023 NXP + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define MAX_TEST_LEN 2048 +#define VT_TIMES 1000 +#define INDIRECT_NUM 1 +#define WAIT_TO_MS 2000 + +#define VT_TXRX_BIT (1 << 0) +#define VT_B_COPY_BIT (1 << 1) +#define VT_F_COPY_BIT (1 << 2) +#define VT_B_MODE_BIT (1 << 3) +#define VT_F_MODE_BIT (1 << 4) + +static u32 poll_delay = 10; + +struct virtio_trans { + /* The virtio device we're associated with */ + struct virtio_device *vdev; + struct device *dev; + wait_queue_head_t waitqueue; + struct work_struct config_work; + + struct virtqueue *tx_vq, *rx_vq; + void *tx_buf, *rx_buf; + spinlock_t txvq_lock; + spinlock_t rxvq_lock; + struct scatterlist *tx_sgl; + struct scatterlist *rx_sgl; + size_t tx_vring_size; + size_t rx_vring_size; + + bool vt_running; + u32 regression; + size_t pkt_size; + bool tx; + bool do_copy; + bool poll_mode; + bool back_poll_mode; + void *data_buf; +}; + +static void tx_intr(struct virtqueue *vq) +{ + struct virtio_trans *vt = vq->vdev->priv; + + if (vt->poll_mode) + return; + + if (!vt->vt_running) + return; + + if (!vt->tx) + return; + + wake_up_interruptible(&vt->waitqueue); +} + +static void virttrans_init_sg(struct scatterlist *sg, size_t sg_num, void *base, size_t len) +{ + int i; + + sg_init_table(sg, sg_num); + if (likely(is_vmalloc_addr(base))) + for (i = 0; i < sg_num; i++) + sg_set_page(&sg[i], vmalloc_to_page(base + i * len), len, + offset_in_page(base)); + else + for (i = 0; i < sg_num; i++) + sg_set_buf(&sg[i], base + i * len, len); +} + +/* call this func with lock */ +static void virttrans_queue_txbuf(struct virtio_trans *vt, + struct scatterlist *sg, + void *buf, size_t pkt_size) +{ + if (vt->do_copy) + memcpy(buf, vt->data_buf, pkt_size); + virtqueue_add_outbuf(vt->tx_vq, sg, INDIRECT_NUM, buf, GFP_ATOMIC); +} + +static void virttrans_reclaim_txvq(struct virtio_trans *vt) +{ + void *buf; + u32 len; + + spin_lock(&vt->txvq_lock); + while ((buf = virtqueue_get_buf(vt->tx_vq, &len)) != NULL) + ; + spin_unlock(&vt->txvq_lock); + +} + +static void virttrans_fill_txvq(struct virtio_trans *vt, size_t pkt_size) +{ + int i; + + spin_lock(&vt->txvq_lock); + for (i = 0; i < vt->tx_vring_size; i++) { + virttrans_init_sg(&vt->tx_sgl[i * INDIRECT_NUM], INDIRECT_NUM, + vt->tx_buf + i * MAX_TEST_LEN * INDIRECT_NUM, + pkt_size); + virttrans_queue_txbuf(vt, &vt->tx_sgl[i * INDIRECT_NUM], + vt->tx_buf + i * MAX_TEST_LEN * + INDIRECT_NUM, pkt_size); + } + spin_unlock(&vt->txvq_lock); +} + +static void *vt_get_tx_buf(struct virtio_trans *vt) +{ + void *buf; + int len; + + spin_lock(&vt->txvq_lock); + buf = virtqueue_get_buf(vt->tx_vq, &len); + spin_unlock(&vt->txvq_lock); + + return buf; +} +static int tx_pkts(struct virtio_trans *vt, size_t pkt_size, u32 times) +{ + + ktime_t start_time, stop_time; + u32 cnt = times; + u64 time_period; + u32 tx_count; + int err; + int i; + u64 idx = 0; + void *buf; + + if (pkt_size > MAX_TEST_LEN) { + dev_err(vt->dev, "pkt_size must be less than 0x%x\n", + MAX_TEST_LEN); + return -EINVAL; + } + + start_time = ktime_get(); + + while (vt->vt_running && cnt > 0) { + buf = vt_get_tx_buf(vt); + while (!buf) { + if (!vt->back_poll_mode) + virtqueue_kick(vt->tx_vq); + + if (vt->poll_mode) { + udelay(poll_delay); + buf = vt_get_tx_buf(vt); + continue; + } + + err = wait_event_interruptible_timeout(vt->waitqueue, + (buf = vt_get_tx_buf(vt)), + msecs_to_jiffies(WAIT_TO_MS)); + if (err == -ERESTARTSYS) { + dev_info(vt->dev, "%s: interrupt the waiting for tx buffer by signal\n", + __func__); + goto out; + } + } + + spin_lock(&vt->txvq_lock); + virttrans_queue_txbuf(vt, &vt->tx_sgl[idx % vt->tx_vring_size], + buf, vt->pkt_size); + idx++; + spin_unlock(&vt->txvq_lock); + + cnt -= INDIRECT_NUM; + } + +out: + i = 100; + while (i--) { + virtqueue_kick(vt->tx_vq); + virtio_cread_le(vt->vdev, struct virtio_trans_config, + tx_count, &tx_count); + if (tx_count >= times - cnt) + break; + mdelay(10); + } + + if (i < 0) + dev_err(vt->dev, "Wait backend completion timeout\n"); + + stop_time = ktime_get(); + time_period = ktime_us_delta(stop_time, start_time); + + pr_info("tx_test: pkt_size (%zu B), pkt_cnt (%d), period (%llu us)\n", + pkt_size, times - cnt, time_period); + + return err; +} + +/* call this func with lock */ +static void virttrans_queue_rxbuf(struct virtio_trans *vt, + struct scatterlist *sg, + void *buf, size_t pkt_size) +{ + if (vt->do_copy) + memcpy(vt->data_buf, buf, pkt_size); + virtqueue_add_inbuf(vt->rx_vq, sg, INDIRECT_NUM, buf, GFP_ATOMIC); +} + +static void virttrans_reclaim_rxvq(struct virtio_trans *vt) +{ + void *buf; + u32 len; + + spin_lock(&vt->rxvq_lock); + while ((buf = virtqueue_get_buf(vt->rx_vq, &len)) != NULL) + ; + spin_unlock(&vt->rxvq_lock); + +} + +static void virttrans_fill_rxvq(struct virtio_trans *vt, size_t pkt_size) +{ + int i; + + spin_lock(&vt->rxvq_lock); + for (i = 0; i < vt->rx_vring_size; i++) { + virttrans_init_sg(&vt->rx_sgl[i * INDIRECT_NUM], INDIRECT_NUM, + vt->rx_buf + i * MAX_TEST_LEN * INDIRECT_NUM, + pkt_size); + virttrans_queue_rxbuf(vt, &vt->rx_sgl[i * INDIRECT_NUM], + vt->rx_buf + i * MAX_TEST_LEN * + INDIRECT_NUM, pkt_size); + } + spin_unlock(&vt->rxvq_lock); +} + +static void rx_intr(struct virtqueue *vq) +{ + struct virtio_trans *vt = vq->vdev->priv; + + if (vt->poll_mode) + return; + + if (!vt->vt_running) + return; + + if (vt->tx) + return; + + wake_up_interruptible(&vt->waitqueue); +} + +static void *vt_get_rx_buf(struct virtio_trans *vt) +{ + void *buf; + int len; + + spin_lock(&vt->rxvq_lock); + buf = virtqueue_get_buf(vt->rx_vq, &len); + spin_unlock(&vt->rxvq_lock); + + return buf; +} + +static int rx_pkts(struct virtio_trans *vt, size_t pkt_size, u32 times) +{ + + ktime_t start_time, stop_time; + u32 cnt = times; + u64 time_period; + int err; + void *buf; + u64 idx = 0; + + if (pkt_size > MAX_TEST_LEN) { + dev_err(vt->dev, "pkt_size must be less than 0x%x\n", + MAX_TEST_LEN); + return -EINVAL; + } + + start_time = ktime_get(); + while (vt->vt_running && cnt > 0) { + buf = vt_get_rx_buf(vt); + while (!buf) { + if (!vt->back_poll_mode) + virtqueue_kick(vt->rx_vq); + + if (vt->poll_mode) { + udelay(poll_delay); + buf = vt_get_rx_buf(vt); + continue; + } + + err = wait_event_interruptible_timeout(vt->waitqueue, + (buf = vt_get_rx_buf(vt)), + msecs_to_jiffies(WAIT_TO_MS)); + if (err == -ERESTARTSYS) { + dev_info(vt->dev, "%s: interrupt the waiting for rx buffer by signal\n", + __func__); + goto out; + } + } + + spin_lock(&vt->rxvq_lock); + virttrans_queue_rxbuf(vt, &vt->rx_sgl[idx % vt->rx_vring_size], + buf, vt->pkt_size); + idx++; + spin_unlock(&vt->rxvq_lock); + + cnt -= INDIRECT_NUM; + } + +out: + stop_time = ktime_get(); + time_period = ktime_us_delta(stop_time, start_time); + + pr_info("rx_test: pkt_size (%zu B), pkt_cnt (%d), period (%llu us)\n", + pkt_size, times - cnt, time_period); + + return 0; +} + +static ssize_t vt_control_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct virtio_trans *vt = dev_get_drvdata(dev); + u32 control; + + virtio_cread_le(vt->vdev, struct virtio_trans_config, control, + &control); + + return sprintf(buf, "0: stop\n1: start\ncontrol = %u\n", control); +} + +static ssize_t vt_control_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t cnt) +{ + struct virtio_trans *vt = dev_get_drvdata(dev); + u32 control; + unsigned long val; + + if (kstrtoul(buf, 0, &val) < 0 || (val > 1)) { + dev_err(vt->dev, "Invalid param\n"); + pr_info("0: stop\n"); + pr_info("1: start\n"); + return -EINVAL; + } + + virtio_cread_le(vt->vdev, struct virtio_trans_config, control, + &control); + + if (!val) { + control &= ~VT_CTRL_START; + virtio_cwrite_le(vt->vdev, struct virtio_trans_config, control, + &control); + vt->vt_running = 0; + if (!vt->tx) + virttrans_reclaim_rxvq(vt); + else + virttrans_reclaim_txvq(vt); + + vt->regression = 0; + return cnt; + } + + if (vt->vt_running) { + dev_err(vt->dev, "Try again after this test completed\n"); + return -EAGAIN; + } + + vt->vt_running = 1; + control |= VT_CTRL_START; + + if (!vt->tx) { + virttrans_fill_rxvq(vt, vt->pkt_size); + virtio_cwrite_le(vt->vdev, struct virtio_trans_config, control, + &control); + rx_pkts(vt, vt->pkt_size, vt->regression); + } else { + virttrans_fill_txvq(vt, vt->pkt_size); + virtio_cwrite_le(vt->vdev, struct virtio_trans_config, control, + &control); + tx_pkts(vt, vt->pkt_size, vt->regression); + } + + control &= ~VT_CTRL_START; + virtio_cwrite_le(vt->vdev, struct virtio_trans_config, control, + &control); + + vt->vt_running = 0; + vt->regression = 0; + + return cnt; +} + +static ssize_t vt_config_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t cnt) +{ + struct virtio_trans *vt = dev_get_drvdata(dev); + u32 config; + unsigned long val; + + if (vt->vt_running) { + dev_err(dev, "Try again after this test completed\n"); + return -EAGAIN; + } + + if (kstrtoul(buf, 0, &val) < 0 || (val & ~0x1f)) { + dev_err(dev, "Invalid param\n"); + pr_info("bit[0]:\n"); + pr_info("\t0: TX\n"); + pr_info("\t1: RX\n"); + pr_info("bit[1]:\n"); + pr_info("\t0: Backend do NOT copy buffer\n"); + pr_info("\t1: Backend do copy buffer\n"); + pr_info("bit[2]:\n"); + pr_info("\t0: Frontend do NOT copy buffer\n"); + pr_info("\t1: Frontend do copy buffer\n"); + pr_info("bit[3]:\n"); + pr_info("\t0: Backend interrupt mode\n"); + pr_info("\t1: Backend polling mode\n"); + pr_info("bit[4]:\n"); + pr_info("\t0: Frontend interrupt mode\n"); + pr_info("\t1: Frontend polling mode\n"); + return -EINVAL; + } + + if (!vt->pkt_size) + vt->pkt_size = 64; + + if (!vt->regression) { + virtio_cwrite_le(vt->vdev, struct virtio_trans_config, + regression, &vt->regression); + vt->regression = VT_TIMES; + } + + virtio_cread_le(vt->vdev, struct virtio_trans_config, config, &config); + + config &= ~(VT_CFG_TX | VT_CFG_RX | VT_CFG_COPY | + VT_CFG_B_POLL | VT_CFG_F_POLL); + vt->do_copy = false; + vt->back_poll_mode = false; + vt->poll_mode = false; + + if (val & VT_TXRX_BIT) { + vt->tx = false; + config |= VT_CFG_RX; + } else { + vt->tx = true; + config |= VT_CFG_TX; + } + + if (val & VT_B_COPY_BIT) + config |= VT_CFG_COPY; + + if (val & VT_F_COPY_BIT) + vt->do_copy = true; + + if (val & VT_B_MODE_BIT) { + vt->back_poll_mode = true; + config |= VT_CFG_B_POLL; + } + + if (val & VT_F_MODE_BIT) { + vt->poll_mode = true; + config |= VT_CFG_F_POLL; + } + + pr_info("*********************************************************\n"); + pr_info("Front-end: %s mode\n", vt->poll_mode ? "poll" : "interrupt"); + pr_info("Back-end: %s mode\n", vt->back_poll_mode ? "poll" : "interrupt"); + pr_info("Front-end: do %scopy buffer\n", vt->do_copy ? "" : "NOT "); + pr_info("Back-end: do %scopy buffer\n", (config & VT_CFG_COPY) ? "" : "NOT"); + + pr_info("\tTest case: %s\n\tpkt_size: <%zu>\n\tregress times: <%d>\n\n", + vt->tx ? "TX" : "RX", vt->pkt_size, vt->regression); + + virtio_cwrite_le(vt->vdev, struct virtio_trans_config, config, &config); + + return cnt; +} +static ssize_t vt_config_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct virtio_trans *vt = dev_get_drvdata(dev); + u32 config; + + virtio_cread_le(vt->vdev, struct virtio_trans_config, config, &config); + + return sprintf(buf, "0x%x\n", config); +} + +static ssize_t vt_pkt_size_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t cnt) +{ + struct virtio_trans *vt = dev_get_drvdata(dev); + u32 val; + + if (vt->vt_running) { + dev_err(dev, "Try again after this test completed\n"); + return -EAGAIN; + } + + if (kstrtou32(buf, 0, &val) < 0 || (val > MAX_TEST_LEN)) { + dev_err(dev, "pkt_size must be less then 0x%x\n", MAX_TEST_LEN); + return -EINVAL; + } + + vt->pkt_size = val; + virtio_cwrite_le(vt->vdev, struct virtio_trans_config, pkt_size, &val); + + dev_dbg(dev, "Update pkt_size to %u\n", val); + + return cnt; +} +static ssize_t vt_pkt_size_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct virtio_trans *vt = dev_get_drvdata(dev); + + return sprintf(buf, "%ld\n", vt->pkt_size); +} + +static ssize_t vt_regression_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t cnt) +{ + struct virtio_trans *vt = dev_get_drvdata(dev); + u32 val; + + if (vt->vt_running) { + dev_err(dev, "Try again after this test completed\n"); + return -EAGAIN; + } + + if (kstrtou32(buf, 0, &val) < 0 || val <= 0) { + dev_err(dev, "Invalid regression\n"); + return -EINVAL; + } + + vt->regression = val; + + virtio_cwrite_le(vt->vdev, struct virtio_trans_config, regression, + &val); + dev_dbg(dev, "Update regression to %u\n", val); + + return cnt; +} +static ssize_t vt_regression_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct virtio_trans *vt = dev_get_drvdata(dev); + + return sprintf(buf, "%u\n", vt->regression); +} + +static DEVICE_ATTR_RW(vt_control); +static DEVICE_ATTR_RW(vt_config); +static DEVICE_ATTR_RW(vt_pkt_size); +static DEVICE_ATTR_RW(vt_regression); + +static struct attribute *sysfs_entries[] = { + &dev_attr_vt_config.attr, + &dev_attr_vt_control.attr, + &dev_attr_vt_pkt_size.attr, + &dev_attr_vt_regression.attr, + NULL +}; + +static const struct attribute_group vt_attribute_group = { + .name = NULL, /* put in device directory */ + .attrs = sysfs_entries, +}; + +static void config_intr(struct virtio_device *vdev) +{ + struct virtio_trans *vt = vdev->priv; + + schedule_work(&vt->config_work); +} + +static void config_work_handler(struct work_struct *work) +{ + /* handle the config change */ +} + +static int virttrans_init_vqs(struct virtio_trans *vt) +{ + struct virtqueue *vqs[2]; + vq_callback_t *cbs[] = { tx_intr, rx_intr}; + static const char * const names[] = { "tx", "rx" }; + int err; + + err = virtio_find_vqs(vt->vdev, 2, vqs, cbs, names, NULL); + if (err) { + dev_err(vt->dev, "Failed to find vqs\n"); + return err; + } + vt->tx_vq = vqs[0]; + vt->rx_vq = vqs[1]; + + return 0; +} + +static int virttrans_probe(struct virtio_device *vdev) +{ + struct virtio_trans *vt; + int err; + dma_addr_t tx_buf_dma; + + vt = kzalloc(sizeof(*vt), GFP_KERNEL); + if (!vt) + return -ENOMEM; + + vdev->priv = vt; + vt->vdev = vdev; + vt->dev = &vdev->dev; + vt->pkt_size = 64; + vt->regression = VT_TIMES; + + dev_set_drvdata(vt->dev, vt); + + err = virttrans_init_vqs(vt); + if (err) { + dev_err(vt->dev, "Failed to init virtqueues\n"); + goto err_init_vq; + } + + vt->tx_vring_size = virtqueue_get_vring_size(vt->tx_vq); + vt->rx_vring_size = virtqueue_get_vring_size(vt->rx_vq); + + vt->data_buf = kzalloc(MAX_TEST_LEN, GFP_KERNEL); + if (!vt->data_buf) { + err = -ENOMEM; + dev_err(vt->dev, "Failed to alloc data buffer\n"); + goto alloc_data_buf; + } + + memset(vt->data_buf, 0xa5, MAX_TEST_LEN); + + vt->tx_sgl = kzalloc(INDIRECT_NUM * (vt->rx_vring_size + + vt->tx_vring_size) * sizeof(struct scatterlist), + GFP_KERNEL); + + if (!vt->tx_sgl) { + err = -ENOMEM; + dev_err(vt->dev, "Failed to alloc SG descriptors\n"); + goto alloc_sgl; + } + + vt->rx_sgl = vt->tx_sgl + INDIRECT_NUM * vt->tx_vring_size; + + vt->tx_buf = dma_alloc_coherent(vdev->dev.parent, MAX_TEST_LEN * + INDIRECT_NUM * (vt->rx_vring_size + + vt->tx_vring_size), &tx_buf_dma, + GFP_KERNEL); + if (!vt->tx_buf) { + err = -ENOMEM; + dev_err(vt->dev, "Failed to alloc Vring buffers\n"); + goto alloc_buf; + } + + vt->rx_buf = vt->tx_buf + + MAX_TEST_LEN * INDIRECT_NUM * vt->tx_vring_size; + + spin_lock_init(&vt->txvq_lock); + spin_lock_init(&vt->rxvq_lock); + init_waitqueue_head(&vt->waitqueue); + INIT_WORK(&vt->config_work, &config_work_handler); + + err = sysfs_create_group(&vt->dev->kobj, &vt_attribute_group); + if (err) { + dev_err(vt->dev, "Failed to create sysfs device attributes\n"); + goto sysfs; + } else { + /* + * Generate a udev event so that appropriate + * symlinks can be created based on udev + * rules. + */ + kobject_uevent(&vt->dev->kobj, KOBJ_CHANGE); + } + + virtio_device_ready(vt->vdev); + + return 0; + +sysfs: + dma_free_coherent(vdev->dev.parent, MAX_TEST_LEN * INDIRECT_NUM * + (vt->rx_vring_size + vt->tx_vring_size), + vt->tx_buf, tx_buf_dma); +alloc_buf: + kfree(vt->tx_sgl); +alloc_sgl: + kfree(vt->data_buf); +alloc_data_buf: + /* TODO: vtrans_deinit_vq(); */ +err_init_vq: + kfree(vt); + + return err; +} + +static void virttrans_remove(struct virtio_device *vdev) +{ + struct virtio_trans *vt = vdev->priv; + void *buf; + + virtio_break_device(vdev); + flush_work(&vt->config_work); + vdev->config->reset(vdev); + cancel_work_sync(&vt->config_work); + + sysfs_remove_group(&vt->dev->kobj, &vt_attribute_group); + + while ((buf = virtqueue_detach_unused_buf(vt->tx_vq)) != NULL) + kfree(buf); + vdev->config->del_vqs(vdev); + + kfree(vt); +} + +static const struct virtio_device_id id_table[] = { + { VIRTIO_ID_TRANS, VIRTIO_DEV_ANY_ID }, + { 0 }, +}; + +static const unsigned int features[] = { + /* none */ +}; + + +static struct virtio_driver virtio_transfer = { + .feature_table = features, + .feature_table_size = ARRAY_SIZE(features), + .driver.name = KBUILD_MODNAME, + .driver.owner = THIS_MODULE, + .id_table = id_table, + .probe = virttrans_probe, + .remove = virttrans_remove, + .config_changed = config_intr, +}; + +static int __init virtio_transfer_init(void) +{ + int err; + + err = register_virtio_driver(&virtio_transfer); + if (err < 0) { + pr_err("Failed to register virtio driver\n"); + return err; + } + return 0; +} + +static void __exit virtio_transfer_exit(void) +{ + unregister_virtio_driver(&virtio_transfer); +} + +module_init(virtio_transfer_init); +module_exit(virtio_transfer_exit); + +MODULE_DEVICE_TABLE(virtio, id_table); +MODULE_DESCRIPTION("VirtIO transfer test driver"); +MODULE_LICENSE("GPL"); -- 2.34.1