[Feature]add MT2731_MP2_MR2_SVN388 baseline version
Change-Id: Ief04314834b31e27effab435d3ca8ba33b499059
diff --git a/src/kernel/linux/v4.14/drivers/pci/host/Kconfig b/src/kernel/linux/v4.14/drivers/pci/host/Kconfig
new file mode 100644
index 0000000..ba4b023
--- /dev/null
+++ b/src/kernel/linux/v4.14/drivers/pci/host/Kconfig
@@ -0,0 +1,224 @@
+menu "PCI host controller drivers"
+ depends on PCI
+
+config PCI_MVEBU
+ bool "Marvell EBU PCIe controller"
+ depends on ARCH_MVEBU || ARCH_DOVE
+ depends on ARM
+ depends on OF
+
+config PCI_AARDVARK
+ bool "Aardvark PCIe controller"
+ depends on ARCH_MVEBU && ARM64
+ depends on OF
+ depends on PCI_MSI_IRQ_DOMAIN
+ help
+ Add support for Aardvark 64bit PCIe Host Controller. This
+ controller is part of the South Bridge of the Marvel Armada
+ 3700 SoC.
+
+config PCIE_XILINX_NWL
+ bool "NWL PCIe Core"
+ depends on ARCH_ZYNQMP
+ depends on PCI_MSI_IRQ_DOMAIN
+ help
+ Say 'Y' here if you want kernel support for Xilinx
+ NWL PCIe controller. The controller can act as Root Port
+ or End Point. The current option selection will only
+ support root port enabling.
+
+config PCI_FTPCI100
+ bool "Faraday Technology FTPCI100 PCI controller"
+ depends on OF
+ depends on ARM
+ default ARCH_GEMINI
+
+config PCI_TEGRA
+ bool "NVIDIA Tegra PCIe controller"
+ depends on ARCH_TEGRA
+ help
+ Say Y here if you want support for the PCIe host controller found
+ on NVIDIA Tegra SoCs.
+
+config PCI_RCAR_GEN2
+ bool "Renesas R-Car Gen2 Internal PCI controller"
+ depends on ARM
+ depends on ARCH_RENESAS || COMPILE_TEST
+ help
+ Say Y here if you want internal PCI support on R-Car Gen2 SoC.
+ There are 3 internal PCI controllers available with a single
+ built-in EHCI/OHCI host controller present on each one.
+
+config PCIE_RCAR
+ bool "Renesas R-Car PCIe controller"
+ depends on ARCH_RENESAS || (ARM && COMPILE_TEST)
+ depends on PCI_MSI_IRQ_DOMAIN
+ help
+ Say Y here if you want PCIe controller support on R-Car SoCs.
+
+config PCI_HOST_COMMON
+ bool
+ select PCI_ECAM
+
+config PCI_HOST_GENERIC
+ bool "Generic PCI host controller"
+ depends on (ARM || ARM64) && OF
+ select PCI_HOST_COMMON
+ select IRQ_DOMAIN
+ help
+ Say Y here if you want to support a simple generic PCI host
+ controller, such as the one emulated by kvmtool.
+
+config PCIE_XILINX
+ bool "Xilinx AXI PCIe host bridge support"
+ depends on ARCH_ZYNQ || MICROBLAZE || (MIPS && PCI_DRIVERS_GENERIC)
+ help
+ Say 'Y' here if you want kernel to support the Xilinx AXI PCIe
+ Host Bridge driver.
+
+config PCI_XGENE
+ bool "X-Gene PCIe controller"
+ depends on ARM64
+ depends on OF || (ACPI && PCI_QUIRKS)
+ select PCIEPORTBUS
+ help
+ Say Y here if you want internal PCI support on APM X-Gene SoC.
+ There are 5 internal PCIe ports available. Each port is GEN3 capable
+ and have varied lanes from x1 to x8.
+
+config PCI_XGENE_MSI
+ bool "X-Gene v1 PCIe MSI feature"
+ depends on PCI_XGENE
+ depends on PCI_MSI_IRQ_DOMAIN
+ default y
+ help
+ Say Y here if you want PCIe MSI support for the APM X-Gene v1 SoC.
+ This MSI driver supports 5 PCIe ports on the APM X-Gene v1 SoC.
+
+config PCI_VERSATILE
+ bool "ARM Versatile PB PCI controller"
+ depends on ARCH_VERSATILE
+
+config PCIE_IPROC
+ tristate
+ select PCI_DOMAINS
+ help
+ This enables the iProc PCIe core controller support for Broadcom's
+ iProc family of SoCs. An appropriate bus interface driver needs
+ to be enabled to select this.
+
+config PCIE_IPROC_PLATFORM
+ tristate "Broadcom iProc PCIe platform bus driver"
+ depends on ARCH_BCM_IPROC || (ARM && COMPILE_TEST)
+ depends on OF
+ select PCIE_IPROC
+ default ARCH_BCM_IPROC
+ help
+ Say Y here if you want to use the Broadcom iProc PCIe controller
+ through the generic platform bus interface
+
+config PCIE_IPROC_BCMA
+ tristate "Broadcom iProc PCIe BCMA bus driver"
+ depends on ARM && (ARCH_BCM_IPROC || COMPILE_TEST)
+ select PCIE_IPROC
+ select BCMA
+ default ARCH_BCM_5301X
+ help
+ Say Y here if you want to use the Broadcom iProc PCIe controller
+ through the BCMA bus interface
+
+config PCIE_IPROC_MSI
+ bool "Broadcom iProc PCIe MSI support"
+ depends on PCIE_IPROC_PLATFORM || PCIE_IPROC_BCMA
+ depends on PCI_MSI_IRQ_DOMAIN
+ default ARCH_BCM_IPROC
+ help
+ Say Y here if you want to enable MSI support for Broadcom's iProc
+ PCIe controller
+
+config PCIE_ALTERA
+ bool "Altera PCIe controller"
+ depends on ARM || NIOS2
+ depends on OF_PCI
+ select PCI_DOMAINS
+ help
+ Say Y here if you want to enable PCIe controller support on Altera
+ FPGA.
+
+config PCIE_ALTERA_MSI
+ bool "Altera PCIe MSI feature"
+ depends on PCIE_ALTERA
+ depends on PCI_MSI_IRQ_DOMAIN
+ help
+ Say Y here if you want PCIe MSI support for the Altera FPGA.
+ This MSI driver supports Altera MSI to GIC controller IP.
+
+config PCI_HOST_THUNDER_PEM
+ bool "Cavium Thunder PCIe controller to off-chip devices"
+ depends on ARM64
+ depends on OF || (ACPI && PCI_QUIRKS)
+ select PCI_HOST_COMMON
+ help
+ Say Y here if you want PCIe support for CN88XX Cavium Thunder SoCs.
+
+config PCI_HOST_THUNDER_ECAM
+ bool "Cavium Thunder ECAM controller to on-chip devices on pass-1.x silicon"
+ depends on ARM64
+ depends on OF || (ACPI && PCI_QUIRKS)
+ select PCI_HOST_COMMON
+ help
+ Say Y here if you want ECAM support for CN88XX-Pass-1.x Cavium Thunder SoCs.
+
+config PCIE_ROCKCHIP
+ tristate "Rockchip PCIe controller"
+ depends on ARCH_ROCKCHIP || COMPILE_TEST
+ depends on OF
+ depends on PCI_MSI_IRQ_DOMAIN
+ select MFD_SYSCON
+ help
+ Say Y here if you want internal PCI support on Rockchip SoC.
+ There is 1 internal PCIe port available to support GEN2 with
+ 4 slots.
+
+config PCIE_MEDIATEK
+ tristate "MediaTek PCIe controller"
+ depends on (ARM || ARM64) && (ARCH_MEDIATEK || COMPILE_TEST)
+ depends on OF
+ depends on PCI
+ depends on PCI_MSI_IRQ_DOMAIN
+ select PCIEPORTBUS
+ help
+ Say Y here if you want to enable PCIe controller support on
+ MediaTek SoCs.
+
+config PCIE_TANGO_SMP8759
+ bool "Tango SMP8759 PCIe controller (DANGEROUS)"
+ depends on ARCH_TANGO && PCI_MSI && OF
+ depends on BROKEN
+ select PCI_HOST_COMMON
+ help
+ Say Y here to enable PCIe controller support for Sigma Designs
+ Tango SMP8759-based systems.
+
+ Note: The SMP8759 controller multiplexes PCI config and MMIO
+ accesses, and Linux doesn't provide a way to serialize them.
+ This can lead to data corruption if drivers perform concurrent
+ config and MMIO accesses.
+
+config VMD
+ depends on PCI_MSI && X86_64 && SRCU
+ tristate "Intel Volume Management Device Driver"
+ default N
+ ---help---
+ Adds support for the Intel Volume Management Device (VMD). VMD is a
+ secondary PCI host bridge that allows PCI Express root ports,
+ and devices attached to them, to be removed from the default
+ PCI domain and placed within the VMD domain. This provides
+ more bus resources than are otherwise possible with a
+ single domain. If you know your system provides one of these and
+ has devices attached to it, say Y; if you are not sure, say N.
+
+ To compile this driver as a module, choose M here: the
+ module will be called vmd.
+
+endmenu
diff --git a/src/kernel/linux/v4.14/drivers/pci/host/Makefile b/src/kernel/linux/v4.14/drivers/pci/host/Makefile
new file mode 100644
index 0000000..95f5b80
--- /dev/null
+++ b/src/kernel/linux/v4.14/drivers/pci/host/Makefile
@@ -0,0 +1,38 @@
+# SPDX-License-Identifier: GPL-2.0
+obj-$(CONFIG_PCI_FTPCI100) += pci-ftpci100.o
+obj-$(CONFIG_PCI_HYPERV) += pci-hyperv.o
+obj-$(CONFIG_PCI_MVEBU) += pci-mvebu.o
+obj-$(CONFIG_PCI_AARDVARK) += pci-aardvark.o
+obj-$(CONFIG_PCI_TEGRA) += pci-tegra.o
+obj-$(CONFIG_PCI_RCAR_GEN2) += pci-rcar-gen2.o
+obj-$(CONFIG_PCIE_RCAR) += pcie-rcar.o
+obj-$(CONFIG_PCI_HOST_COMMON) += pci-host-common.o
+obj-$(CONFIG_PCI_HOST_GENERIC) += pci-host-generic.o
+obj-$(CONFIG_PCIE_XILINX) += pcie-xilinx.o
+obj-$(CONFIG_PCIE_XILINX_NWL) += pcie-xilinx-nwl.o
+obj-$(CONFIG_PCI_XGENE_MSI) += pci-xgene-msi.o
+obj-$(CONFIG_PCI_VERSATILE) += pci-versatile.o
+obj-$(CONFIG_PCIE_IPROC) += pcie-iproc.o
+obj-$(CONFIG_PCIE_IPROC_MSI) += pcie-iproc-msi.o
+obj-$(CONFIG_PCIE_IPROC_PLATFORM) += pcie-iproc-platform.o
+obj-$(CONFIG_PCIE_IPROC_BCMA) += pcie-iproc-bcma.o
+obj-$(CONFIG_PCIE_ALTERA) += pcie-altera.o
+obj-$(CONFIG_PCIE_ALTERA_MSI) += pcie-altera-msi.o
+obj-$(CONFIG_PCIE_ROCKCHIP) += pcie-rockchip.o
+obj-$(CONFIG_PCIE_MEDIATEK) += pcie-mediatek.o
+obj-$(CONFIG_PCIE_TANGO_SMP8759) += pcie-tango.o
+obj-$(CONFIG_VMD) += vmd.o
+
+# The following drivers are for devices that use the generic ACPI
+# pci_root.c driver but don't support standard ECAM config access.
+# They contain MCFG quirks to replace the generic ECAM accessors with
+# device-specific ones that are shared with the DT driver.
+
+# The ACPI driver is generic and should not require driver-specific
+# config options to be enabled, so we always build these drivers on
+# ARM64 and use internal ifdefs to only build the pieces we need
+# depending on whether ACPI, the DT driver, or both are enabled.
+
+obj-$(CONFIG_ARM64) += pci-thunder-ecam.o
+obj-$(CONFIG_ARM64) += pci-thunder-pem.o
+obj-$(CONFIG_ARM64) += pci-xgene.o
diff --git a/src/kernel/linux/v4.14/drivers/pci/host/pci-aardvark.c b/src/kernel/linux/v4.14/drivers/pci/host/pci-aardvark.c
new file mode 100644
index 0000000..c1db09f
--- /dev/null
+++ b/src/kernel/linux/v4.14/drivers/pci/host/pci-aardvark.c
@@ -0,0 +1,977 @@
+/*
+ * Driver for the Aardvark PCIe controller, used on Marvell Armada
+ * 3700.
+ *
+ * Copyright (C) 2016 Marvell
+ *
+ * Author: Hezi Shahmoon <hezi.shahmoon@marvell.com>
+ *
+ * This file is licensed under the terms of the GNU General Public
+ * License version 2. This program is licensed "as is" without any
+ * warranty of any kind, whether express or implied.
+ */
+
+#include <linux/delay.h>
+#include <linux/interrupt.h>
+#include <linux/irq.h>
+#include <linux/irqdomain.h>
+#include <linux/kernel.h>
+#include <linux/pci.h>
+#include <linux/init.h>
+#include <linux/platform_device.h>
+#include <linux/of_address.h>
+#include <linux/of_pci.h>
+
+/* PCIe core registers */
+#define PCIE_CORE_CMD_STATUS_REG 0x4
+#define PCIE_CORE_CMD_IO_ACCESS_EN BIT(0)
+#define PCIE_CORE_CMD_MEM_ACCESS_EN BIT(1)
+#define PCIE_CORE_CMD_MEM_IO_REQ_EN BIT(2)
+#define PCIE_CORE_DEV_CTRL_STATS_REG 0xc8
+#define PCIE_CORE_DEV_CTRL_STATS_RELAX_ORDER_DISABLE (0 << 4)
+#define PCIE_CORE_DEV_CTRL_STATS_MAX_PAYLOAD_SZ_SHIFT 5
+#define PCIE_CORE_DEV_CTRL_STATS_SNOOP_DISABLE (0 << 11)
+#define PCIE_CORE_DEV_CTRL_STATS_MAX_RD_REQ_SIZE_SHIFT 12
+#define PCIE_CORE_DEV_CTRL_STATS_MAX_RD_REQ_SZ 0x2
+#define PCIE_CORE_LINK_CTRL_STAT_REG 0xd0
+#define PCIE_CORE_LINK_L0S_ENTRY BIT(0)
+#define PCIE_CORE_LINK_TRAINING BIT(5)
+#define PCIE_CORE_LINK_WIDTH_SHIFT 20
+#define PCIE_CORE_ERR_CAPCTL_REG 0x118
+#define PCIE_CORE_ERR_CAPCTL_ECRC_CHK_TX BIT(5)
+#define PCIE_CORE_ERR_CAPCTL_ECRC_CHK_TX_EN BIT(6)
+#define PCIE_CORE_ERR_CAPCTL_ECRC_CHCK BIT(7)
+#define PCIE_CORE_ERR_CAPCTL_ECRC_CHCK_RCV BIT(8)
+
+/* PIO registers base address and register offsets */
+#define PIO_BASE_ADDR 0x4000
+#define PIO_CTRL (PIO_BASE_ADDR + 0x0)
+#define PIO_CTRL_TYPE_MASK GENMASK(3, 0)
+#define PIO_CTRL_ADDR_WIN_DISABLE BIT(24)
+#define PIO_STAT (PIO_BASE_ADDR + 0x4)
+#define PIO_COMPLETION_STATUS_SHIFT 7
+#define PIO_COMPLETION_STATUS_MASK GENMASK(9, 7)
+#define PIO_COMPLETION_STATUS_OK 0
+#define PIO_COMPLETION_STATUS_UR 1
+#define PIO_COMPLETION_STATUS_CRS 2
+#define PIO_COMPLETION_STATUS_CA 4
+#define PIO_NON_POSTED_REQ BIT(0)
+#define PIO_ADDR_LS (PIO_BASE_ADDR + 0x8)
+#define PIO_ADDR_MS (PIO_BASE_ADDR + 0xc)
+#define PIO_WR_DATA (PIO_BASE_ADDR + 0x10)
+#define PIO_WR_DATA_STRB (PIO_BASE_ADDR + 0x14)
+#define PIO_RD_DATA (PIO_BASE_ADDR + 0x18)
+#define PIO_START (PIO_BASE_ADDR + 0x1c)
+#define PIO_ISR (PIO_BASE_ADDR + 0x20)
+#define PIO_ISRM (PIO_BASE_ADDR + 0x24)
+
+/* Aardvark Control registers */
+#define CONTROL_BASE_ADDR 0x4800
+#define PCIE_CORE_CTRL0_REG (CONTROL_BASE_ADDR + 0x0)
+#define PCIE_GEN_SEL_MSK 0x3
+#define PCIE_GEN_SEL_SHIFT 0x0
+#define SPEED_GEN_1 0
+#define SPEED_GEN_2 1
+#define SPEED_GEN_3 2
+#define IS_RC_MSK 1
+#define IS_RC_SHIFT 2
+#define LANE_CNT_MSK 0x18
+#define LANE_CNT_SHIFT 0x3
+#define LANE_COUNT_1 (0 << LANE_CNT_SHIFT)
+#define LANE_COUNT_2 (1 << LANE_CNT_SHIFT)
+#define LANE_COUNT_4 (2 << LANE_CNT_SHIFT)
+#define LANE_COUNT_8 (3 << LANE_CNT_SHIFT)
+#define LINK_TRAINING_EN BIT(6)
+#define LEGACY_INTA BIT(28)
+#define LEGACY_INTB BIT(29)
+#define LEGACY_INTC BIT(30)
+#define LEGACY_INTD BIT(31)
+#define PCIE_CORE_CTRL1_REG (CONTROL_BASE_ADDR + 0x4)
+#define HOT_RESET_GEN BIT(0)
+#define PCIE_CORE_CTRL2_REG (CONTROL_BASE_ADDR + 0x8)
+#define PCIE_CORE_CTRL2_RESERVED 0x7
+#define PCIE_CORE_CTRL2_TD_ENABLE BIT(4)
+#define PCIE_CORE_CTRL2_STRICT_ORDER_ENABLE BIT(5)
+#define PCIE_CORE_CTRL2_OB_WIN_ENABLE BIT(6)
+#define PCIE_CORE_CTRL2_MSI_ENABLE BIT(10)
+#define PCIE_ISR0_REG (CONTROL_BASE_ADDR + 0x40)
+#define PCIE_ISR0_MASK_REG (CONTROL_BASE_ADDR + 0x44)
+#define PCIE_ISR0_MSI_INT_PENDING BIT(24)
+#define PCIE_ISR0_INTX_ASSERT(val) BIT(16 + (val))
+#define PCIE_ISR0_INTX_DEASSERT(val) BIT(20 + (val))
+#define PCIE_ISR0_ALL_MASK GENMASK(26, 0)
+#define PCIE_ISR1_REG (CONTROL_BASE_ADDR + 0x48)
+#define PCIE_ISR1_MASK_REG (CONTROL_BASE_ADDR + 0x4C)
+#define PCIE_ISR1_POWER_STATE_CHANGE BIT(4)
+#define PCIE_ISR1_FLUSH BIT(5)
+#define PCIE_ISR1_INTX_ASSERT(val) BIT(8 + (val))
+#define PCIE_ISR1_ALL_MASK GENMASK(11, 4)
+#define PCIE_MSI_ADDR_LOW_REG (CONTROL_BASE_ADDR + 0x50)
+#define PCIE_MSI_ADDR_HIGH_REG (CONTROL_BASE_ADDR + 0x54)
+#define PCIE_MSI_STATUS_REG (CONTROL_BASE_ADDR + 0x58)
+#define PCIE_MSI_MASK_REG (CONTROL_BASE_ADDR + 0x5C)
+#define PCIE_MSI_PAYLOAD_REG (CONTROL_BASE_ADDR + 0x9C)
+
+/* PCIe window configuration */
+#define OB_WIN_BASE_ADDR 0x4c00
+#define OB_WIN_BLOCK_SIZE 0x20
+#define OB_WIN_REG_ADDR(win, offset) (OB_WIN_BASE_ADDR + \
+ OB_WIN_BLOCK_SIZE * (win) + \
+ (offset))
+#define OB_WIN_MATCH_LS(win) OB_WIN_REG_ADDR(win, 0x00)
+#define OB_WIN_MATCH_MS(win) OB_WIN_REG_ADDR(win, 0x04)
+#define OB_WIN_REMAP_LS(win) OB_WIN_REG_ADDR(win, 0x08)
+#define OB_WIN_REMAP_MS(win) OB_WIN_REG_ADDR(win, 0x0c)
+#define OB_WIN_MASK_LS(win) OB_WIN_REG_ADDR(win, 0x10)
+#define OB_WIN_MASK_MS(win) OB_WIN_REG_ADDR(win, 0x14)
+#define OB_WIN_ACTIONS(win) OB_WIN_REG_ADDR(win, 0x18)
+
+/* PCIe window types */
+#define OB_PCIE_MEM 0x0
+#define OB_PCIE_IO 0x4
+
+/* LMI registers base address and register offsets */
+#define LMI_BASE_ADDR 0x6000
+#define CFG_REG (LMI_BASE_ADDR + 0x0)
+#define LTSSM_SHIFT 24
+#define LTSSM_MASK 0x3f
+#define LTSSM_L0 0x10
+#define RC_BAR_CONFIG 0x300
+
+/* PCIe core controller registers */
+#define CTRL_CORE_BASE_ADDR 0x18000
+#define CTRL_CONFIG_REG (CTRL_CORE_BASE_ADDR + 0x0)
+#define CTRL_MODE_SHIFT 0x0
+#define CTRL_MODE_MASK 0x1
+#define PCIE_CORE_MODE_DIRECT 0x0
+#define PCIE_CORE_MODE_COMMAND 0x1
+
+/* PCIe Central Interrupts Registers */
+#define CENTRAL_INT_BASE_ADDR 0x1b000
+#define HOST_CTRL_INT_STATUS_REG (CENTRAL_INT_BASE_ADDR + 0x0)
+#define HOST_CTRL_INT_MASK_REG (CENTRAL_INT_BASE_ADDR + 0x4)
+#define PCIE_IRQ_CMDQ_INT BIT(0)
+#define PCIE_IRQ_MSI_STATUS_INT BIT(1)
+#define PCIE_IRQ_CMD_SENT_DONE BIT(3)
+#define PCIE_IRQ_DMA_INT BIT(4)
+#define PCIE_IRQ_IB_DXFERDONE BIT(5)
+#define PCIE_IRQ_OB_DXFERDONE BIT(6)
+#define PCIE_IRQ_OB_RXFERDONE BIT(7)
+#define PCIE_IRQ_COMPQ_INT BIT(12)
+#define PCIE_IRQ_DIR_RD_DDR_DET BIT(13)
+#define PCIE_IRQ_DIR_WR_DDR_DET BIT(14)
+#define PCIE_IRQ_CORE_INT BIT(16)
+#define PCIE_IRQ_CORE_INT_PIO BIT(17)
+#define PCIE_IRQ_DPMU_INT BIT(18)
+#define PCIE_IRQ_PCIE_MIS_INT BIT(19)
+#define PCIE_IRQ_MSI_INT1_DET BIT(20)
+#define PCIE_IRQ_MSI_INT2_DET BIT(21)
+#define PCIE_IRQ_RC_DBELL_DET BIT(22)
+#define PCIE_IRQ_EP_STATUS BIT(23)
+#define PCIE_IRQ_ALL_MASK 0xfff0fb
+#define PCIE_IRQ_ENABLE_INTS_MASK PCIE_IRQ_CORE_INT
+
+/* Transaction types */
+#define PCIE_CONFIG_RD_TYPE0 0x8
+#define PCIE_CONFIG_RD_TYPE1 0x9
+#define PCIE_CONFIG_WR_TYPE0 0xa
+#define PCIE_CONFIG_WR_TYPE1 0xb
+
+#define PCIE_CONF_BUS(bus) (((bus) & 0xff) << 20)
+#define PCIE_CONF_DEV(dev) (((dev) & 0x1f) << 15)
+#define PCIE_CONF_FUNC(fun) (((fun) & 0x7) << 12)
+#define PCIE_CONF_REG(reg) ((reg) & 0xffc)
+#define PCIE_CONF_ADDR(bus, devfn, where) \
+ (PCIE_CONF_BUS(bus) | PCIE_CONF_DEV(PCI_SLOT(devfn)) | \
+ PCIE_CONF_FUNC(PCI_FUNC(devfn)) | PCIE_CONF_REG(where))
+
+#define PIO_TIMEOUT_MS 1
+
+#define LINK_WAIT_MAX_RETRIES 10
+#define LINK_WAIT_USLEEP_MIN 90000
+#define LINK_WAIT_USLEEP_MAX 100000
+
+#define MSI_IRQ_NUM 32
+
+struct advk_pcie {
+ struct platform_device *pdev;
+ void __iomem *base;
+ struct list_head resources;
+ struct irq_domain *irq_domain;
+ struct irq_chip irq_chip;
+ struct irq_domain *msi_domain;
+ struct irq_domain *msi_inner_domain;
+ struct irq_chip msi_bottom_irq_chip;
+ struct irq_chip msi_irq_chip;
+ struct msi_domain_info msi_domain_info;
+ DECLARE_BITMAP(msi_used, MSI_IRQ_NUM);
+ struct mutex msi_used_lock;
+ u16 msi_msg;
+ int root_bus_nr;
+};
+
+static inline void advk_writel(struct advk_pcie *pcie, u32 val, u64 reg)
+{
+ writel(val, pcie->base + reg);
+}
+
+static inline u32 advk_readl(struct advk_pcie *pcie, u64 reg)
+{
+ return readl(pcie->base + reg);
+}
+
+static int advk_pcie_link_up(struct advk_pcie *pcie)
+{
+ u32 val, ltssm_state;
+
+ val = advk_readl(pcie, CFG_REG);
+ ltssm_state = (val >> LTSSM_SHIFT) & LTSSM_MASK;
+ return ltssm_state >= LTSSM_L0;
+}
+
+static int advk_pcie_wait_for_link(struct advk_pcie *pcie)
+{
+ struct device *dev = &pcie->pdev->dev;
+ int retries;
+
+ /* check if the link is up or not */
+ for (retries = 0; retries < LINK_WAIT_MAX_RETRIES; retries++) {
+ if (advk_pcie_link_up(pcie)) {
+ dev_info(dev, "link up\n");
+ return 0;
+ }
+
+ usleep_range(LINK_WAIT_USLEEP_MIN, LINK_WAIT_USLEEP_MAX);
+ }
+
+ dev_err(dev, "link never came up\n");
+ return -ETIMEDOUT;
+}
+
+/*
+ * Set PCIe address window register which could be used for memory
+ * mapping.
+ */
+static void advk_pcie_set_ob_win(struct advk_pcie *pcie,
+ u32 win_num, u32 match_ms,
+ u32 match_ls, u32 mask_ms,
+ u32 mask_ls, u32 remap_ms,
+ u32 remap_ls, u32 action)
+{
+ advk_writel(pcie, match_ls, OB_WIN_MATCH_LS(win_num));
+ advk_writel(pcie, match_ms, OB_WIN_MATCH_MS(win_num));
+ advk_writel(pcie, mask_ms, OB_WIN_MASK_MS(win_num));
+ advk_writel(pcie, mask_ls, OB_WIN_MASK_LS(win_num));
+ advk_writel(pcie, remap_ms, OB_WIN_REMAP_MS(win_num));
+ advk_writel(pcie, remap_ls, OB_WIN_REMAP_LS(win_num));
+ advk_writel(pcie, action, OB_WIN_ACTIONS(win_num));
+ advk_writel(pcie, match_ls | BIT(0), OB_WIN_MATCH_LS(win_num));
+}
+
+static void advk_pcie_setup_hw(struct advk_pcie *pcie)
+{
+ u32 reg;
+ int i;
+
+ /* Point PCIe unit MBUS decode windows to DRAM space */
+ for (i = 0; i < 8; i++)
+ advk_pcie_set_ob_win(pcie, i, 0, 0, 0, 0, 0, 0, 0);
+
+ /* Set to Direct mode */
+ reg = advk_readl(pcie, CTRL_CONFIG_REG);
+ reg &= ~(CTRL_MODE_MASK << CTRL_MODE_SHIFT);
+ reg |= ((PCIE_CORE_MODE_DIRECT & CTRL_MODE_MASK) << CTRL_MODE_SHIFT);
+ advk_writel(pcie, reg, CTRL_CONFIG_REG);
+
+ /* Set PCI global control register to RC mode */
+ reg = advk_readl(pcie, PCIE_CORE_CTRL0_REG);
+ reg |= (IS_RC_MSK << IS_RC_SHIFT);
+ advk_writel(pcie, reg, PCIE_CORE_CTRL0_REG);
+
+ /* Set Advanced Error Capabilities and Control PF0 register */
+ reg = PCIE_CORE_ERR_CAPCTL_ECRC_CHK_TX |
+ PCIE_CORE_ERR_CAPCTL_ECRC_CHK_TX_EN |
+ PCIE_CORE_ERR_CAPCTL_ECRC_CHCK |
+ PCIE_CORE_ERR_CAPCTL_ECRC_CHCK_RCV;
+ advk_writel(pcie, reg, PCIE_CORE_ERR_CAPCTL_REG);
+
+ /* Set PCIe Device Control and Status 1 PF0 register */
+ reg = PCIE_CORE_DEV_CTRL_STATS_RELAX_ORDER_DISABLE |
+ (7 << PCIE_CORE_DEV_CTRL_STATS_MAX_PAYLOAD_SZ_SHIFT) |
+ PCIE_CORE_DEV_CTRL_STATS_SNOOP_DISABLE |
+ (PCIE_CORE_DEV_CTRL_STATS_MAX_RD_REQ_SZ <<
+ PCIE_CORE_DEV_CTRL_STATS_MAX_RD_REQ_SIZE_SHIFT);
+ advk_writel(pcie, reg, PCIE_CORE_DEV_CTRL_STATS_REG);
+
+ /* Program PCIe Control 2 to disable strict ordering */
+ reg = PCIE_CORE_CTRL2_RESERVED |
+ PCIE_CORE_CTRL2_TD_ENABLE;
+ advk_writel(pcie, reg, PCIE_CORE_CTRL2_REG);
+
+ /* Set GEN2 */
+ reg = advk_readl(pcie, PCIE_CORE_CTRL0_REG);
+ reg &= ~PCIE_GEN_SEL_MSK;
+ reg |= SPEED_GEN_2;
+ advk_writel(pcie, reg, PCIE_CORE_CTRL0_REG);
+
+ /* Set lane X1 */
+ reg = advk_readl(pcie, PCIE_CORE_CTRL0_REG);
+ reg &= ~LANE_CNT_MSK;
+ reg |= LANE_COUNT_1;
+ advk_writel(pcie, reg, PCIE_CORE_CTRL0_REG);
+
+ /* Enable link training */
+ reg = advk_readl(pcie, PCIE_CORE_CTRL0_REG);
+ reg |= LINK_TRAINING_EN;
+ advk_writel(pcie, reg, PCIE_CORE_CTRL0_REG);
+
+ /* Enable MSI */
+ reg = advk_readl(pcie, PCIE_CORE_CTRL2_REG);
+ reg |= PCIE_CORE_CTRL2_MSI_ENABLE;
+ advk_writel(pcie, reg, PCIE_CORE_CTRL2_REG);
+
+ /* Clear all interrupts */
+ advk_writel(pcie, PCIE_ISR0_ALL_MASK, PCIE_ISR0_REG);
+ advk_writel(pcie, PCIE_ISR1_ALL_MASK, PCIE_ISR1_REG);
+ advk_writel(pcie, PCIE_IRQ_ALL_MASK, HOST_CTRL_INT_STATUS_REG);
+
+ /* Disable All ISR0/1 Sources */
+ reg = PCIE_ISR0_ALL_MASK;
+ reg &= ~PCIE_ISR0_MSI_INT_PENDING;
+ advk_writel(pcie, reg, PCIE_ISR0_MASK_REG);
+
+ advk_writel(pcie, PCIE_ISR1_ALL_MASK, PCIE_ISR1_MASK_REG);
+
+ /* Unmask all MSI's */
+ advk_writel(pcie, 0, PCIE_MSI_MASK_REG);
+
+ /* Enable summary interrupt for GIC SPI source */
+ reg = PCIE_IRQ_ALL_MASK & (~PCIE_IRQ_ENABLE_INTS_MASK);
+ advk_writel(pcie, reg, HOST_CTRL_INT_MASK_REG);
+
+ reg = advk_readl(pcie, PCIE_CORE_CTRL2_REG);
+ reg |= PCIE_CORE_CTRL2_OB_WIN_ENABLE;
+ advk_writel(pcie, reg, PCIE_CORE_CTRL2_REG);
+
+ /* Bypass the address window mapping for PIO */
+ reg = advk_readl(pcie, PIO_CTRL);
+ reg |= PIO_CTRL_ADDR_WIN_DISABLE;
+ advk_writel(pcie, reg, PIO_CTRL);
+
+ /* Start link training */
+ reg = advk_readl(pcie, PCIE_CORE_LINK_CTRL_STAT_REG);
+ reg |= PCIE_CORE_LINK_TRAINING;
+ advk_writel(pcie, reg, PCIE_CORE_LINK_CTRL_STAT_REG);
+
+ advk_pcie_wait_for_link(pcie);
+
+ reg = advk_readl(pcie, PCIE_CORE_CMD_STATUS_REG);
+ reg |= PCIE_CORE_CMD_MEM_ACCESS_EN |
+ PCIE_CORE_CMD_IO_ACCESS_EN |
+ PCIE_CORE_CMD_MEM_IO_REQ_EN;
+ advk_writel(pcie, reg, PCIE_CORE_CMD_STATUS_REG);
+}
+
+static void advk_pcie_check_pio_status(struct advk_pcie *pcie)
+{
+ struct device *dev = &pcie->pdev->dev;
+ u32 reg;
+ unsigned int status;
+ char *strcomp_status, *str_posted;
+
+ reg = advk_readl(pcie, PIO_STAT);
+ status = (reg & PIO_COMPLETION_STATUS_MASK) >>
+ PIO_COMPLETION_STATUS_SHIFT;
+
+ if (!status)
+ return;
+
+ switch (status) {
+ case PIO_COMPLETION_STATUS_UR:
+ strcomp_status = "UR";
+ break;
+ case PIO_COMPLETION_STATUS_CRS:
+ strcomp_status = "CRS";
+ break;
+ case PIO_COMPLETION_STATUS_CA:
+ strcomp_status = "CA";
+ break;
+ default:
+ strcomp_status = "Unknown";
+ break;
+ }
+
+ if (reg & PIO_NON_POSTED_REQ)
+ str_posted = "Non-posted";
+ else
+ str_posted = "Posted";
+
+ dev_err(dev, "%s PIO Response Status: %s, %#x @ %#x\n",
+ str_posted, strcomp_status, reg, advk_readl(pcie, PIO_ADDR_LS));
+}
+
+static int advk_pcie_wait_pio(struct advk_pcie *pcie)
+{
+ struct device *dev = &pcie->pdev->dev;
+ unsigned long timeout;
+
+ timeout = jiffies + msecs_to_jiffies(PIO_TIMEOUT_MS);
+
+ while (time_before(jiffies, timeout)) {
+ u32 start, isr;
+
+ start = advk_readl(pcie, PIO_START);
+ isr = advk_readl(pcie, PIO_ISR);
+ if (!start && isr)
+ return 0;
+ }
+
+ dev_err(dev, "config read/write timed out\n");
+ return -ETIMEDOUT;
+}
+
+static int advk_pcie_rd_conf(struct pci_bus *bus, u32 devfn,
+ int where, int size, u32 *val)
+{
+ struct advk_pcie *pcie = bus->sysdata;
+ u32 reg;
+ int ret;
+
+ if ((bus->number == pcie->root_bus_nr) && PCI_SLOT(devfn) != 0) {
+ *val = 0xffffffff;
+ return PCIBIOS_DEVICE_NOT_FOUND;
+ }
+
+ /* Start PIO */
+ advk_writel(pcie, 0, PIO_START);
+ advk_writel(pcie, 1, PIO_ISR);
+
+ /* Program the control register */
+ reg = advk_readl(pcie, PIO_CTRL);
+ reg &= ~PIO_CTRL_TYPE_MASK;
+ if (bus->number == pcie->root_bus_nr)
+ reg |= PCIE_CONFIG_RD_TYPE0;
+ else
+ reg |= PCIE_CONFIG_RD_TYPE1;
+ advk_writel(pcie, reg, PIO_CTRL);
+
+ /* Program the address registers */
+ reg = PCIE_CONF_ADDR(bus->number, devfn, where);
+ advk_writel(pcie, reg, PIO_ADDR_LS);
+ advk_writel(pcie, 0, PIO_ADDR_MS);
+
+ /* Program the data strobe */
+ advk_writel(pcie, 0xf, PIO_WR_DATA_STRB);
+
+ /* Start the transfer */
+ advk_writel(pcie, 1, PIO_START);
+
+ ret = advk_pcie_wait_pio(pcie);
+ if (ret < 0)
+ return PCIBIOS_SET_FAILED;
+
+ advk_pcie_check_pio_status(pcie);
+
+ /* Get the read result */
+ *val = advk_readl(pcie, PIO_RD_DATA);
+ if (size == 1)
+ *val = (*val >> (8 * (where & 3))) & 0xff;
+ else if (size == 2)
+ *val = (*val >> (8 * (where & 3))) & 0xffff;
+
+ return PCIBIOS_SUCCESSFUL;
+}
+
+static int advk_pcie_wr_conf(struct pci_bus *bus, u32 devfn,
+ int where, int size, u32 val)
+{
+ struct advk_pcie *pcie = bus->sysdata;
+ u32 reg;
+ u32 data_strobe = 0x0;
+ int offset;
+ int ret;
+
+ if ((bus->number == pcie->root_bus_nr) && PCI_SLOT(devfn) != 0)
+ return PCIBIOS_DEVICE_NOT_FOUND;
+
+ if (where % size)
+ return PCIBIOS_SET_FAILED;
+
+ /* Start PIO */
+ advk_writel(pcie, 0, PIO_START);
+ advk_writel(pcie, 1, PIO_ISR);
+
+ /* Program the control register */
+ reg = advk_readl(pcie, PIO_CTRL);
+ reg &= ~PIO_CTRL_TYPE_MASK;
+ if (bus->number == pcie->root_bus_nr)
+ reg |= PCIE_CONFIG_WR_TYPE0;
+ else
+ reg |= PCIE_CONFIG_WR_TYPE1;
+ advk_writel(pcie, reg, PIO_CTRL);
+
+ /* Program the address registers */
+ reg = PCIE_CONF_ADDR(bus->number, devfn, where);
+ advk_writel(pcie, reg, PIO_ADDR_LS);
+ advk_writel(pcie, 0, PIO_ADDR_MS);
+
+ /* Calculate the write strobe */
+ offset = where & 0x3;
+ reg = val << (8 * offset);
+ data_strobe = GENMASK(size - 1, 0) << offset;
+
+ /* Program the data register */
+ advk_writel(pcie, reg, PIO_WR_DATA);
+
+ /* Program the data strobe */
+ advk_writel(pcie, data_strobe, PIO_WR_DATA_STRB);
+
+ /* Start the transfer */
+ advk_writel(pcie, 1, PIO_START);
+
+ ret = advk_pcie_wait_pio(pcie);
+ if (ret < 0)
+ return PCIBIOS_SET_FAILED;
+
+ advk_pcie_check_pio_status(pcie);
+
+ return PCIBIOS_SUCCESSFUL;
+}
+
+static struct pci_ops advk_pcie_ops = {
+ .read = advk_pcie_rd_conf,
+ .write = advk_pcie_wr_conf,
+};
+
+static void advk_msi_irq_compose_msi_msg(struct irq_data *data,
+ struct msi_msg *msg)
+{
+ struct advk_pcie *pcie = irq_data_get_irq_chip_data(data);
+ phys_addr_t msi_msg = virt_to_phys(&pcie->msi_msg);
+
+ msg->address_lo = lower_32_bits(msi_msg);
+ msg->address_hi = upper_32_bits(msi_msg);
+ msg->data = data->irq;
+}
+
+static int advk_msi_set_affinity(struct irq_data *irq_data,
+ const struct cpumask *mask, bool force)
+{
+ return -EINVAL;
+}
+
+static int advk_msi_irq_domain_alloc(struct irq_domain *domain,
+ unsigned int virq,
+ unsigned int nr_irqs, void *args)
+{
+ struct advk_pcie *pcie = domain->host_data;
+ int hwirq, i;
+
+ mutex_lock(&pcie->msi_used_lock);
+ hwirq = bitmap_find_next_zero_area(pcie->msi_used, MSI_IRQ_NUM,
+ 0, nr_irqs, 0);
+ if (hwirq >= MSI_IRQ_NUM) {
+ mutex_unlock(&pcie->msi_used_lock);
+ return -ENOSPC;
+ }
+
+ bitmap_set(pcie->msi_used, hwirq, nr_irqs);
+ mutex_unlock(&pcie->msi_used_lock);
+
+ for (i = 0; i < nr_irqs; i++)
+ irq_domain_set_info(domain, virq + i, hwirq + i,
+ &pcie->msi_bottom_irq_chip,
+ domain->host_data, handle_simple_irq,
+ NULL, NULL);
+
+ return hwirq;
+}
+
+static void advk_msi_irq_domain_free(struct irq_domain *domain,
+ unsigned int virq, unsigned int nr_irqs)
+{
+ struct irq_data *d = irq_domain_get_irq_data(domain, virq);
+ struct advk_pcie *pcie = domain->host_data;
+
+ mutex_lock(&pcie->msi_used_lock);
+ bitmap_clear(pcie->msi_used, d->hwirq, nr_irqs);
+ mutex_unlock(&pcie->msi_used_lock);
+}
+
+static const struct irq_domain_ops advk_msi_domain_ops = {
+ .alloc = advk_msi_irq_domain_alloc,
+ .free = advk_msi_irq_domain_free,
+};
+
+static void advk_pcie_irq_mask(struct irq_data *d)
+{
+ struct advk_pcie *pcie = d->domain->host_data;
+ irq_hw_number_t hwirq = irqd_to_hwirq(d);
+ u32 mask;
+
+ mask = advk_readl(pcie, PCIE_ISR1_MASK_REG);
+ mask |= PCIE_ISR1_INTX_ASSERT(hwirq);
+ advk_writel(pcie, mask, PCIE_ISR1_MASK_REG);
+}
+
+static void advk_pcie_irq_unmask(struct irq_data *d)
+{
+ struct advk_pcie *pcie = d->domain->host_data;
+ irq_hw_number_t hwirq = irqd_to_hwirq(d);
+ u32 mask;
+
+ mask = advk_readl(pcie, PCIE_ISR1_MASK_REG);
+ mask &= ~PCIE_ISR1_INTX_ASSERT(hwirq);
+ advk_writel(pcie, mask, PCIE_ISR1_MASK_REG);
+}
+
+static int advk_pcie_irq_map(struct irq_domain *h,
+ unsigned int virq, irq_hw_number_t hwirq)
+{
+ struct advk_pcie *pcie = h->host_data;
+
+ advk_pcie_irq_mask(irq_get_irq_data(virq));
+ irq_set_status_flags(virq, IRQ_LEVEL);
+ irq_set_chip_and_handler(virq, &pcie->irq_chip,
+ handle_level_irq);
+ irq_set_chip_data(virq, pcie);
+
+ return 0;
+}
+
+static const struct irq_domain_ops advk_pcie_irq_domain_ops = {
+ .map = advk_pcie_irq_map,
+ .xlate = irq_domain_xlate_onecell,
+};
+
+static int advk_pcie_init_msi_irq_domain(struct advk_pcie *pcie)
+{
+ struct device *dev = &pcie->pdev->dev;
+ struct device_node *node = dev->of_node;
+ struct irq_chip *bottom_ic, *msi_ic;
+ struct msi_domain_info *msi_di;
+ phys_addr_t msi_msg_phys;
+
+ mutex_init(&pcie->msi_used_lock);
+
+ bottom_ic = &pcie->msi_bottom_irq_chip;
+
+ bottom_ic->name = "MSI";
+ bottom_ic->irq_compose_msi_msg = advk_msi_irq_compose_msi_msg;
+ bottom_ic->irq_set_affinity = advk_msi_set_affinity;
+
+ msi_ic = &pcie->msi_irq_chip;
+ msi_ic->name = "advk-MSI";
+
+ msi_di = &pcie->msi_domain_info;
+ msi_di->flags = MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS |
+ MSI_FLAG_MULTI_PCI_MSI;
+ msi_di->chip = msi_ic;
+
+ msi_msg_phys = virt_to_phys(&pcie->msi_msg);
+
+ advk_writel(pcie, lower_32_bits(msi_msg_phys),
+ PCIE_MSI_ADDR_LOW_REG);
+ advk_writel(pcie, upper_32_bits(msi_msg_phys),
+ PCIE_MSI_ADDR_HIGH_REG);
+
+ pcie->msi_inner_domain =
+ irq_domain_add_linear(NULL, MSI_IRQ_NUM,
+ &advk_msi_domain_ops, pcie);
+ if (!pcie->msi_inner_domain)
+ return -ENOMEM;
+
+ pcie->msi_domain =
+ pci_msi_create_irq_domain(of_node_to_fwnode(node),
+ msi_di, pcie->msi_inner_domain);
+ if (!pcie->msi_domain) {
+ irq_domain_remove(pcie->msi_inner_domain);
+ return -ENOMEM;
+ }
+
+ return 0;
+}
+
+static void advk_pcie_remove_msi_irq_domain(struct advk_pcie *pcie)
+{
+ irq_domain_remove(pcie->msi_domain);
+ irq_domain_remove(pcie->msi_inner_domain);
+}
+
+static int advk_pcie_init_irq_domain(struct advk_pcie *pcie)
+{
+ struct device *dev = &pcie->pdev->dev;
+ struct device_node *node = dev->of_node;
+ struct device_node *pcie_intc_node;
+ struct irq_chip *irq_chip;
+
+ pcie_intc_node = of_get_next_child(node, NULL);
+ if (!pcie_intc_node) {
+ dev_err(dev, "No PCIe Intc node found\n");
+ return -ENODEV;
+ }
+
+ irq_chip = &pcie->irq_chip;
+
+ irq_chip->name = devm_kasprintf(dev, GFP_KERNEL, "%s-irq",
+ dev_name(dev));
+ if (!irq_chip->name) {
+ of_node_put(pcie_intc_node);
+ return -ENOMEM;
+ }
+
+ irq_chip->irq_mask = advk_pcie_irq_mask;
+ irq_chip->irq_mask_ack = advk_pcie_irq_mask;
+ irq_chip->irq_unmask = advk_pcie_irq_unmask;
+
+ pcie->irq_domain =
+ irq_domain_add_linear(pcie_intc_node, PCI_NUM_INTX,
+ &advk_pcie_irq_domain_ops, pcie);
+ if (!pcie->irq_domain) {
+ dev_err(dev, "Failed to get a INTx IRQ domain\n");
+ of_node_put(pcie_intc_node);
+ return -ENOMEM;
+ }
+
+ return 0;
+}
+
+static void advk_pcie_remove_irq_domain(struct advk_pcie *pcie)
+{
+ irq_domain_remove(pcie->irq_domain);
+}
+
+static void advk_pcie_handle_msi(struct advk_pcie *pcie)
+{
+ u32 msi_val, msi_mask, msi_status, msi_idx;
+ u16 msi_data;
+
+ msi_mask = advk_readl(pcie, PCIE_MSI_MASK_REG);
+ msi_val = advk_readl(pcie, PCIE_MSI_STATUS_REG);
+ msi_status = msi_val & ~msi_mask;
+
+ for (msi_idx = 0; msi_idx < MSI_IRQ_NUM; msi_idx++) {
+ if (!(BIT(msi_idx) & msi_status))
+ continue;
+
+ advk_writel(pcie, BIT(msi_idx), PCIE_MSI_STATUS_REG);
+ msi_data = advk_readl(pcie, PCIE_MSI_PAYLOAD_REG) & 0xFF;
+ generic_handle_irq(msi_data);
+ }
+
+ advk_writel(pcie, PCIE_ISR0_MSI_INT_PENDING,
+ PCIE_ISR0_REG);
+}
+
+static void advk_pcie_handle_int(struct advk_pcie *pcie)
+{
+ u32 isr0_val, isr0_mask, isr0_status;
+ u32 isr1_val, isr1_mask, isr1_status;
+ int i, virq;
+
+ isr0_val = advk_readl(pcie, PCIE_ISR0_REG);
+ isr0_mask = advk_readl(pcie, PCIE_ISR0_MASK_REG);
+ isr0_status = isr0_val & ((~isr0_mask) & PCIE_ISR0_ALL_MASK);
+
+ isr1_val = advk_readl(pcie, PCIE_ISR1_REG);
+ isr1_mask = advk_readl(pcie, PCIE_ISR1_MASK_REG);
+ isr1_status = isr1_val & ((~isr1_mask) & PCIE_ISR1_ALL_MASK);
+
+ if (!isr0_status && !isr1_status) {
+ advk_writel(pcie, isr0_val, PCIE_ISR0_REG);
+ advk_writel(pcie, isr1_val, PCIE_ISR1_REG);
+ return;
+ }
+
+ /* Process MSI interrupts */
+ if (isr0_status & PCIE_ISR0_MSI_INT_PENDING)
+ advk_pcie_handle_msi(pcie);
+
+ /* Process legacy interrupts */
+ for (i = 0; i < PCI_NUM_INTX; i++) {
+ if (!(isr1_status & PCIE_ISR1_INTX_ASSERT(i)))
+ continue;
+
+ advk_writel(pcie, PCIE_ISR1_INTX_ASSERT(i),
+ PCIE_ISR1_REG);
+
+ virq = irq_find_mapping(pcie->irq_domain, i);
+ generic_handle_irq(virq);
+ }
+}
+
+static irqreturn_t advk_pcie_irq_handler(int irq, void *arg)
+{
+ struct advk_pcie *pcie = arg;
+ u32 status;
+
+ status = advk_readl(pcie, HOST_CTRL_INT_STATUS_REG);
+ if (!(status & PCIE_IRQ_CORE_INT))
+ return IRQ_NONE;
+
+ advk_pcie_handle_int(pcie);
+
+ /* Clear interrupt */
+ advk_writel(pcie, PCIE_IRQ_CORE_INT, HOST_CTRL_INT_STATUS_REG);
+
+ return IRQ_HANDLED;
+}
+
+static int advk_pcie_parse_request_of_pci_ranges(struct advk_pcie *pcie)
+{
+ int err, res_valid = 0;
+ struct device *dev = &pcie->pdev->dev;
+ struct device_node *np = dev->of_node;
+ struct resource_entry *win, *tmp;
+ resource_size_t iobase;
+
+ INIT_LIST_HEAD(&pcie->resources);
+
+ err = of_pci_get_host_bridge_resources(np, 0, 0xff, &pcie->resources,
+ &iobase);
+ if (err)
+ return err;
+
+ err = devm_request_pci_bus_resources(dev, &pcie->resources);
+ if (err)
+ goto out_release_res;
+
+ resource_list_for_each_entry_safe(win, tmp, &pcie->resources) {
+ struct resource *res = win->res;
+
+ switch (resource_type(res)) {
+ case IORESOURCE_IO:
+ advk_pcie_set_ob_win(pcie, 1,
+ upper_32_bits(res->start),
+ lower_32_bits(res->start),
+ 0, 0xF8000000, 0,
+ lower_32_bits(res->start),
+ OB_PCIE_IO);
+ err = pci_remap_iospace(res, iobase);
+ if (err) {
+ dev_warn(dev, "error %d: failed to map resource %pR\n",
+ err, res);
+ resource_list_destroy_entry(win);
+ }
+ break;
+ case IORESOURCE_MEM:
+ advk_pcie_set_ob_win(pcie, 0,
+ upper_32_bits(res->start),
+ lower_32_bits(res->start),
+ 0x0, 0xF8000000, 0,
+ lower_32_bits(res->start),
+ (2 << 20) | OB_PCIE_MEM);
+ res_valid |= !(res->flags & IORESOURCE_PREFETCH);
+ break;
+ case IORESOURCE_BUS:
+ pcie->root_bus_nr = res->start;
+ break;
+ }
+ }
+
+ if (!res_valid) {
+ dev_err(dev, "non-prefetchable memory resource required\n");
+ err = -EINVAL;
+ goto out_release_res;
+ }
+
+ return 0;
+
+out_release_res:
+ pci_free_resource_list(&pcie->resources);
+ return err;
+}
+
+static int advk_pcie_probe(struct platform_device *pdev)
+{
+ struct device *dev = &pdev->dev;
+ struct advk_pcie *pcie;
+ struct resource *res;
+ struct pci_bus *bus, *child;
+ struct pci_host_bridge *bridge;
+ int ret, irq;
+
+ bridge = devm_pci_alloc_host_bridge(dev, sizeof(struct advk_pcie));
+ if (!bridge)
+ return -ENOMEM;
+
+ pcie = pci_host_bridge_priv(bridge);
+ pcie->pdev = pdev;
+
+ res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ pcie->base = devm_ioremap_resource(dev, res);
+ if (IS_ERR(pcie->base))
+ return PTR_ERR(pcie->base);
+
+ irq = platform_get_irq(pdev, 0);
+ ret = devm_request_irq(dev, irq, advk_pcie_irq_handler,
+ IRQF_SHARED | IRQF_NO_THREAD, "advk-pcie",
+ pcie);
+ if (ret) {
+ dev_err(dev, "Failed to register interrupt\n");
+ return ret;
+ }
+
+ ret = advk_pcie_parse_request_of_pci_ranges(pcie);
+ if (ret) {
+ dev_err(dev, "Failed to parse resources\n");
+ return ret;
+ }
+
+ advk_pcie_setup_hw(pcie);
+
+ ret = advk_pcie_init_irq_domain(pcie);
+ if (ret) {
+ dev_err(dev, "Failed to initialize irq\n");
+ return ret;
+ }
+
+ ret = advk_pcie_init_msi_irq_domain(pcie);
+ if (ret) {
+ dev_err(dev, "Failed to initialize irq\n");
+ advk_pcie_remove_irq_domain(pcie);
+ return ret;
+ }
+
+ list_splice_init(&pcie->resources, &bridge->windows);
+ bridge->dev.parent = dev;
+ bridge->sysdata = pcie;
+ bridge->busnr = 0;
+ bridge->ops = &advk_pcie_ops;
+ bridge->map_irq = of_irq_parse_and_map_pci;
+ bridge->swizzle_irq = pci_common_swizzle;
+
+ ret = pci_scan_root_bus_bridge(bridge);
+ if (ret < 0) {
+ advk_pcie_remove_msi_irq_domain(pcie);
+ advk_pcie_remove_irq_domain(pcie);
+ return ret;
+ }
+
+ bus = bridge->bus;
+
+ pci_bus_size_bridges(bus);
+ pci_bus_assign_resources(bus);
+
+ list_for_each_entry(child, &bus->children, node)
+ pcie_bus_configure_settings(child);
+
+ pci_bus_add_devices(bus);
+ return 0;
+}
+
+static const struct of_device_id advk_pcie_of_match_table[] = {
+ { .compatible = "marvell,armada-3700-pcie", },
+ {},
+};
+
+static struct platform_driver advk_pcie_driver = {
+ .driver = {
+ .name = "advk-pcie",
+ .of_match_table = advk_pcie_of_match_table,
+ /* Driver unloading/unbinding currently not supported */
+ .suppress_bind_attrs = true,
+ },
+ .probe = advk_pcie_probe,
+};
+builtin_platform_driver(advk_pcie_driver);
diff --git a/src/kernel/linux/v4.14/drivers/pci/host/pci-ftpci100.c b/src/kernel/linux/v4.14/drivers/pci/host/pci-ftpci100.c
new file mode 100644
index 0000000..4e6b219
--- /dev/null
+++ b/src/kernel/linux/v4.14/drivers/pci/host/pci-ftpci100.c
@@ -0,0 +1,637 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Support for Faraday Technology FTPC100 PCI Controller
+ *
+ * Copyright (C) 2017 Linus Walleij <linus.walleij@linaro.org>
+ *
+ * Based on the out-of-tree OpenWRT patch for Cortina Gemini:
+ * Copyright (C) 2009 Janos Laube <janos.dev@gmail.com>
+ * Copyright (C) 2009 Paulius Zaleckas <paulius.zaleckas@teltonika.lt>
+ * Based on SL2312 PCI controller code
+ * Storlink (C) 2003
+ */
+
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/of_address.h>
+#include <linux/of_device.h>
+#include <linux/of_irq.h>
+#include <linux/of_pci.h>
+#include <linux/pci.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+#include <linux/irqdomain.h>
+#include <linux/irqchip/chained_irq.h>
+#include <linux/bitops.h>
+#include <linux/irq.h>
+#include <linux/clk.h>
+
+/*
+ * Special configuration registers directly in the first few words
+ * in I/O space.
+ */
+#define PCI_IOSIZE 0x00
+#define PCI_PROT 0x04 /* AHB protection */
+#define PCI_CTRL 0x08 /* PCI control signal */
+#define PCI_SOFTRST 0x10 /* Soft reset counter and response error enable */
+#define PCI_CONFIG 0x28 /* PCI configuration command register */
+#define PCI_DATA 0x2C
+
+#define FARADAY_PCI_STATUS_CMD 0x04 /* Status and command */
+#define FARADAY_PCI_PMC 0x40 /* Power management control */
+#define FARADAY_PCI_PMCSR 0x44 /* Power management status */
+#define FARADAY_PCI_CTRL1 0x48 /* Control register 1 */
+#define FARADAY_PCI_CTRL2 0x4C /* Control register 2 */
+#define FARADAY_PCI_MEM1_BASE_SIZE 0x50 /* Memory base and size #1 */
+#define FARADAY_PCI_MEM2_BASE_SIZE 0x54 /* Memory base and size #2 */
+#define FARADAY_PCI_MEM3_BASE_SIZE 0x58 /* Memory base and size #3 */
+
+#define PCI_STATUS_66MHZ_CAPABLE BIT(21)
+
+/* Bits 31..28 gives INTD..INTA status */
+#define PCI_CTRL2_INTSTS_SHIFT 28
+#define PCI_CTRL2_INTMASK_CMDERR BIT(27)
+#define PCI_CTRL2_INTMASK_PARERR BIT(26)
+/* Bits 25..22 masks INTD..INTA */
+#define PCI_CTRL2_INTMASK_SHIFT 22
+#define PCI_CTRL2_INTMASK_MABRT_RX BIT(21)
+#define PCI_CTRL2_INTMASK_TABRT_RX BIT(20)
+#define PCI_CTRL2_INTMASK_TABRT_TX BIT(19)
+#define PCI_CTRL2_INTMASK_RETRY4 BIT(18)
+#define PCI_CTRL2_INTMASK_SERR_RX BIT(17)
+#define PCI_CTRL2_INTMASK_PERR_RX BIT(16)
+/* Bit 15 reserved */
+#define PCI_CTRL2_MSTPRI_REQ6 BIT(14)
+#define PCI_CTRL2_MSTPRI_REQ5 BIT(13)
+#define PCI_CTRL2_MSTPRI_REQ4 BIT(12)
+#define PCI_CTRL2_MSTPRI_REQ3 BIT(11)
+#define PCI_CTRL2_MSTPRI_REQ2 BIT(10)
+#define PCI_CTRL2_MSTPRI_REQ1 BIT(9)
+#define PCI_CTRL2_MSTPRI_REQ0 BIT(8)
+/* Bits 7..4 reserved */
+/* Bits 3..0 TRDYW */
+
+/*
+ * Memory configs:
+ * Bit 31..20 defines the PCI side memory base
+ * Bit 19..16 (4 bits) defines the size per below
+ */
+#define FARADAY_PCI_MEMBASE_MASK 0xfff00000
+#define FARADAY_PCI_MEMSIZE_1MB 0x0
+#define FARADAY_PCI_MEMSIZE_2MB 0x1
+#define FARADAY_PCI_MEMSIZE_4MB 0x2
+#define FARADAY_PCI_MEMSIZE_8MB 0x3
+#define FARADAY_PCI_MEMSIZE_16MB 0x4
+#define FARADAY_PCI_MEMSIZE_32MB 0x5
+#define FARADAY_PCI_MEMSIZE_64MB 0x6
+#define FARADAY_PCI_MEMSIZE_128MB 0x7
+#define FARADAY_PCI_MEMSIZE_256MB 0x8
+#define FARADAY_PCI_MEMSIZE_512MB 0x9
+#define FARADAY_PCI_MEMSIZE_1GB 0xa
+#define FARADAY_PCI_MEMSIZE_2GB 0xb
+#define FARADAY_PCI_MEMSIZE_SHIFT 16
+
+/*
+ * The DMA base is set to 0x0 for all memory segments, it reflects the
+ * fact that the memory of the host system starts at 0x0.
+ */
+#define FARADAY_PCI_DMA_MEM1_BASE 0x00000000
+#define FARADAY_PCI_DMA_MEM2_BASE 0x00000000
+#define FARADAY_PCI_DMA_MEM3_BASE 0x00000000
+
+/* Defines for PCI configuration command register */
+#define PCI_CONF_ENABLE BIT(31)
+#define PCI_CONF_WHERE(r) ((r) & 0xFC)
+#define PCI_CONF_BUS(b) (((b) & 0xFF) << 16)
+#define PCI_CONF_DEVICE(d) (((d) & 0x1F) << 11)
+#define PCI_CONF_FUNCTION(f) (((f) & 0x07) << 8)
+
+/**
+ * struct faraday_pci_variant - encodes IP block differences
+ * @cascaded_irq: this host has cascaded IRQs from an interrupt controller
+ * embedded in the host bridge.
+ */
+struct faraday_pci_variant {
+ bool cascaded_irq;
+};
+
+struct faraday_pci {
+ struct device *dev;
+ void __iomem *base;
+ struct irq_domain *irqdomain;
+ struct pci_bus *bus;
+ struct clk *bus_clk;
+};
+
+static int faraday_res_to_memcfg(resource_size_t mem_base,
+ resource_size_t mem_size, u32 *val)
+{
+ u32 outval;
+
+ switch (mem_size) {
+ case SZ_1M:
+ outval = FARADAY_PCI_MEMSIZE_1MB;
+ break;
+ case SZ_2M:
+ outval = FARADAY_PCI_MEMSIZE_2MB;
+ break;
+ case SZ_4M:
+ outval = FARADAY_PCI_MEMSIZE_4MB;
+ break;
+ case SZ_8M:
+ outval = FARADAY_PCI_MEMSIZE_8MB;
+ break;
+ case SZ_16M:
+ outval = FARADAY_PCI_MEMSIZE_16MB;
+ break;
+ case SZ_32M:
+ outval = FARADAY_PCI_MEMSIZE_32MB;
+ break;
+ case SZ_64M:
+ outval = FARADAY_PCI_MEMSIZE_64MB;
+ break;
+ case SZ_128M:
+ outval = FARADAY_PCI_MEMSIZE_128MB;
+ break;
+ case SZ_256M:
+ outval = FARADAY_PCI_MEMSIZE_256MB;
+ break;
+ case SZ_512M:
+ outval = FARADAY_PCI_MEMSIZE_512MB;
+ break;
+ case SZ_1G:
+ outval = FARADAY_PCI_MEMSIZE_1GB;
+ break;
+ case SZ_2G:
+ outval = FARADAY_PCI_MEMSIZE_2GB;
+ break;
+ default:
+ return -EINVAL;
+ }
+ outval <<= FARADAY_PCI_MEMSIZE_SHIFT;
+
+ /* This is probably not good */
+ if (mem_base & ~(FARADAY_PCI_MEMBASE_MASK))
+ pr_warn("truncated PCI memory base\n");
+ /* Translate to bridge side address space */
+ outval |= (mem_base & FARADAY_PCI_MEMBASE_MASK);
+ pr_debug("Translated pci base @%pap, size %pap to config %08x\n",
+ &mem_base, &mem_size, outval);
+
+ *val = outval;
+ return 0;
+}
+
+static int faraday_raw_pci_read_config(struct faraday_pci *p, int bus_number,
+ unsigned int fn, int config, int size,
+ u32 *value)
+{
+ writel(PCI_CONF_BUS(bus_number) |
+ PCI_CONF_DEVICE(PCI_SLOT(fn)) |
+ PCI_CONF_FUNCTION(PCI_FUNC(fn)) |
+ PCI_CONF_WHERE(config) |
+ PCI_CONF_ENABLE,
+ p->base + PCI_CONFIG);
+
+ *value = readl(p->base + PCI_DATA);
+
+ if (size == 1)
+ *value = (*value >> (8 * (config & 3))) & 0xFF;
+ else if (size == 2)
+ *value = (*value >> (8 * (config & 3))) & 0xFFFF;
+
+ return PCIBIOS_SUCCESSFUL;
+}
+
+static int faraday_pci_read_config(struct pci_bus *bus, unsigned int fn,
+ int config, int size, u32 *value)
+{
+ struct faraday_pci *p = bus->sysdata;
+
+ dev_dbg(&bus->dev,
+ "[read] slt: %.2d, fnc: %d, cnf: 0x%.2X, val (%d bytes): 0x%.8X\n",
+ PCI_SLOT(fn), PCI_FUNC(fn), config, size, *value);
+
+ return faraday_raw_pci_read_config(p, bus->number, fn, config, size, value);
+}
+
+static int faraday_raw_pci_write_config(struct faraday_pci *p, int bus_number,
+ unsigned int fn, int config, int size,
+ u32 value)
+{
+ int ret = PCIBIOS_SUCCESSFUL;
+
+ writel(PCI_CONF_BUS(bus_number) |
+ PCI_CONF_DEVICE(PCI_SLOT(fn)) |
+ PCI_CONF_FUNCTION(PCI_FUNC(fn)) |
+ PCI_CONF_WHERE(config) |
+ PCI_CONF_ENABLE,
+ p->base + PCI_CONFIG);
+
+ switch (size) {
+ case 4:
+ writel(value, p->base + PCI_DATA);
+ break;
+ case 2:
+ writew(value, p->base + PCI_DATA + (config & 3));
+ break;
+ case 1:
+ writeb(value, p->base + PCI_DATA + (config & 3));
+ break;
+ default:
+ ret = PCIBIOS_BAD_REGISTER_NUMBER;
+ }
+
+ return ret;
+}
+
+static int faraday_pci_write_config(struct pci_bus *bus, unsigned int fn,
+ int config, int size, u32 value)
+{
+ struct faraday_pci *p = bus->sysdata;
+
+ dev_dbg(&bus->dev,
+ "[write] slt: %.2d, fnc: %d, cnf: 0x%.2X, val (%d bytes): 0x%.8X\n",
+ PCI_SLOT(fn), PCI_FUNC(fn), config, size, value);
+
+ return faraday_raw_pci_write_config(p, bus->number, fn, config, size,
+ value);
+}
+
+static struct pci_ops faraday_pci_ops = {
+ .read = faraday_pci_read_config,
+ .write = faraday_pci_write_config,
+};
+
+static void faraday_pci_ack_irq(struct irq_data *d)
+{
+ struct faraday_pci *p = irq_data_get_irq_chip_data(d);
+ unsigned int reg;
+
+ faraday_raw_pci_read_config(p, 0, 0, FARADAY_PCI_CTRL2, 4, ®);
+ reg &= ~(0xF << PCI_CTRL2_INTSTS_SHIFT);
+ reg |= BIT(irqd_to_hwirq(d) + PCI_CTRL2_INTSTS_SHIFT);
+ faraday_raw_pci_write_config(p, 0, 0, FARADAY_PCI_CTRL2, 4, reg);
+}
+
+static void faraday_pci_mask_irq(struct irq_data *d)
+{
+ struct faraday_pci *p = irq_data_get_irq_chip_data(d);
+ unsigned int reg;
+
+ faraday_raw_pci_read_config(p, 0, 0, FARADAY_PCI_CTRL2, 4, ®);
+ reg &= ~((0xF << PCI_CTRL2_INTSTS_SHIFT)
+ | BIT(irqd_to_hwirq(d) + PCI_CTRL2_INTMASK_SHIFT));
+ faraday_raw_pci_write_config(p, 0, 0, FARADAY_PCI_CTRL2, 4, reg);
+}
+
+static void faraday_pci_unmask_irq(struct irq_data *d)
+{
+ struct faraday_pci *p = irq_data_get_irq_chip_data(d);
+ unsigned int reg;
+
+ faraday_raw_pci_read_config(p, 0, 0, FARADAY_PCI_CTRL2, 4, ®);
+ reg &= ~(0xF << PCI_CTRL2_INTSTS_SHIFT);
+ reg |= BIT(irqd_to_hwirq(d) + PCI_CTRL2_INTMASK_SHIFT);
+ faraday_raw_pci_write_config(p, 0, 0, FARADAY_PCI_CTRL2, 4, reg);
+}
+
+static void faraday_pci_irq_handler(struct irq_desc *desc)
+{
+ struct faraday_pci *p = irq_desc_get_handler_data(desc);
+ struct irq_chip *irqchip = irq_desc_get_chip(desc);
+ unsigned int irq_stat, reg, i;
+
+ faraday_raw_pci_read_config(p, 0, 0, FARADAY_PCI_CTRL2, 4, ®);
+ irq_stat = reg >> PCI_CTRL2_INTSTS_SHIFT;
+
+ chained_irq_enter(irqchip, desc);
+
+ for (i = 0; i < 4; i++) {
+ if ((irq_stat & BIT(i)) == 0)
+ continue;
+ generic_handle_irq(irq_find_mapping(p->irqdomain, i));
+ }
+
+ chained_irq_exit(irqchip, desc);
+}
+
+static struct irq_chip faraday_pci_irq_chip = {
+ .name = "PCI",
+ .irq_ack = faraday_pci_ack_irq,
+ .irq_mask = faraday_pci_mask_irq,
+ .irq_unmask = faraday_pci_unmask_irq,
+};
+
+static int faraday_pci_irq_map(struct irq_domain *domain, unsigned int irq,
+ irq_hw_number_t hwirq)
+{
+ irq_set_chip_and_handler(irq, &faraday_pci_irq_chip, handle_level_irq);
+ irq_set_chip_data(irq, domain->host_data);
+
+ return 0;
+}
+
+static const struct irq_domain_ops faraday_pci_irqdomain_ops = {
+ .map = faraday_pci_irq_map,
+};
+
+static int faraday_pci_setup_cascaded_irq(struct faraday_pci *p)
+{
+ struct device_node *intc = of_get_next_child(p->dev->of_node, NULL);
+ int irq;
+ int i;
+
+ if (!intc) {
+ dev_err(p->dev, "missing child interrupt-controller node\n");
+ return -EINVAL;
+ }
+
+ /* All PCI IRQs cascade off this one */
+ irq = of_irq_get(intc, 0);
+ if (irq <= 0) {
+ dev_err(p->dev, "failed to get parent IRQ\n");
+ of_node_put(intc);
+ return irq ?: -EINVAL;
+ }
+
+ p->irqdomain = irq_domain_add_linear(intc, PCI_NUM_INTX,
+ &faraday_pci_irqdomain_ops, p);
+ of_node_put(intc);
+ if (!p->irqdomain) {
+ dev_err(p->dev, "failed to create Gemini PCI IRQ domain\n");
+ return -EINVAL;
+ }
+
+ irq_set_chained_handler_and_data(irq, faraday_pci_irq_handler, p);
+
+ for (i = 0; i < 4; i++)
+ irq_create_mapping(p->irqdomain, i);
+
+ return 0;
+}
+
+static int pci_dma_range_parser_init(struct of_pci_range_parser *parser,
+ struct device_node *node)
+{
+ const int na = 3, ns = 2;
+ int rlen;
+
+ parser->node = node;
+ parser->pna = of_n_addr_cells(node);
+ parser->np = parser->pna + na + ns;
+
+ parser->range = of_get_property(node, "dma-ranges", &rlen);
+ if (!parser->range)
+ return -ENOENT;
+ parser->end = parser->range + rlen / sizeof(__be32);
+
+ return 0;
+}
+
+static int faraday_pci_parse_map_dma_ranges(struct faraday_pci *p,
+ struct device_node *np)
+{
+ struct of_pci_range range;
+ struct of_pci_range_parser parser;
+ struct device *dev = p->dev;
+ u32 confreg[3] = {
+ FARADAY_PCI_MEM1_BASE_SIZE,
+ FARADAY_PCI_MEM2_BASE_SIZE,
+ FARADAY_PCI_MEM3_BASE_SIZE,
+ };
+ int i = 0;
+ u32 val;
+
+ if (pci_dma_range_parser_init(&parser, np)) {
+ dev_err(dev, "missing dma-ranges property\n");
+ return -EINVAL;
+ }
+
+ /*
+ * Get the dma-ranges from the device tree
+ */
+ for_each_of_pci_range(&parser, &range) {
+ u64 end = range.pci_addr + range.size - 1;
+ int ret;
+
+ ret = faraday_res_to_memcfg(range.pci_addr, range.size, &val);
+ if (ret) {
+ dev_err(dev,
+ "DMA range %d: illegal MEM resource size\n", i);
+ return -EINVAL;
+ }
+
+ dev_info(dev, "DMA MEM%d BASE: 0x%016llx -> 0x%016llx config %08x\n",
+ i + 1, range.pci_addr, end, val);
+ if (i <= 2) {
+ faraday_raw_pci_write_config(p, 0, 0, confreg[i],
+ 4, val);
+ } else {
+ dev_err(dev, "ignore extraneous dma-range %d\n", i);
+ break;
+ }
+
+ i++;
+ }
+
+ return 0;
+}
+
+static int faraday_pci_probe(struct platform_device *pdev)
+{
+ struct device *dev = &pdev->dev;
+ const struct faraday_pci_variant *variant =
+ of_device_get_match_data(dev);
+ struct resource *regs;
+ resource_size_t io_base;
+ struct resource_entry *win;
+ struct faraday_pci *p;
+ struct resource *mem;
+ struct resource *io;
+ struct pci_host_bridge *host;
+ struct clk *clk;
+ unsigned char max_bus_speed = PCI_SPEED_33MHz;
+ unsigned char cur_bus_speed = PCI_SPEED_33MHz;
+ int ret;
+ u32 val;
+ LIST_HEAD(res);
+
+ host = devm_pci_alloc_host_bridge(dev, sizeof(*p));
+ if (!host)
+ return -ENOMEM;
+
+ host->dev.parent = dev;
+ host->ops = &faraday_pci_ops;
+ host->busnr = 0;
+ host->msi = NULL;
+ host->map_irq = of_irq_parse_and_map_pci;
+ host->swizzle_irq = pci_common_swizzle;
+ p = pci_host_bridge_priv(host);
+ host->sysdata = p;
+ p->dev = dev;
+
+ /* Retrieve and enable optional clocks */
+ clk = devm_clk_get(dev, "PCLK");
+ if (IS_ERR(clk))
+ return PTR_ERR(clk);
+ ret = clk_prepare_enable(clk);
+ if (ret) {
+ dev_err(dev, "could not prepare PCLK\n");
+ return ret;
+ }
+ p->bus_clk = devm_clk_get(dev, "PCICLK");
+ if (IS_ERR(p->bus_clk))
+ return PTR_ERR(clk);
+ ret = clk_prepare_enable(p->bus_clk);
+ if (ret) {
+ dev_err(dev, "could not prepare PCICLK\n");
+ return ret;
+ }
+
+ regs = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ p->base = devm_ioremap_resource(dev, regs);
+ if (IS_ERR(p->base))
+ return PTR_ERR(p->base);
+
+ ret = of_pci_get_host_bridge_resources(dev->of_node, 0, 0xff,
+ &res, &io_base);
+ if (ret)
+ return ret;
+
+ ret = devm_request_pci_bus_resources(dev, &res);
+ if (ret)
+ return ret;
+
+ /* Get the I/O and memory ranges from DT */
+ resource_list_for_each_entry(win, &res) {
+ switch (resource_type(win->res)) {
+ case IORESOURCE_IO:
+ io = win->res;
+ io->name = "Gemini PCI I/O";
+ if (!faraday_res_to_memcfg(io->start - win->offset,
+ resource_size(io), &val)) {
+ /* setup I/O space size */
+ writel(val, p->base + PCI_IOSIZE);
+ } else {
+ dev_err(dev, "illegal IO mem size\n");
+ return -EINVAL;
+ }
+ ret = pci_remap_iospace(io, io_base);
+ if (ret) {
+ dev_warn(dev, "error %d: failed to map resource %pR\n",
+ ret, io);
+ continue;
+ }
+ break;
+ case IORESOURCE_MEM:
+ mem = win->res;
+ mem->name = "Gemini PCI MEM";
+ break;
+ case IORESOURCE_BUS:
+ break;
+ default:
+ break;
+ }
+ }
+
+ /* Setup hostbridge */
+ val = readl(p->base + PCI_CTRL);
+ val |= PCI_COMMAND_IO;
+ val |= PCI_COMMAND_MEMORY;
+ val |= PCI_COMMAND_MASTER;
+ writel(val, p->base + PCI_CTRL);
+ /* Mask and clear all interrupts */
+ faraday_raw_pci_write_config(p, 0, 0, FARADAY_PCI_CTRL2 + 2, 2, 0xF000);
+ if (variant->cascaded_irq) {
+ ret = faraday_pci_setup_cascaded_irq(p);
+ if (ret) {
+ dev_err(dev, "failed to setup cascaded IRQ\n");
+ return ret;
+ }
+ }
+
+ /* Check bus clock if we can gear up to 66 MHz */
+ if (!IS_ERR(p->bus_clk)) {
+ unsigned long rate;
+ u32 val;
+
+ faraday_raw_pci_read_config(p, 0, 0,
+ FARADAY_PCI_STATUS_CMD, 4, &val);
+ rate = clk_get_rate(p->bus_clk);
+
+ if ((rate == 33000000) && (val & PCI_STATUS_66MHZ_CAPABLE)) {
+ dev_info(dev, "33MHz bus is 66MHz capable\n");
+ max_bus_speed = PCI_SPEED_66MHz;
+ ret = clk_set_rate(p->bus_clk, 66000000);
+ if (ret)
+ dev_err(dev, "failed to set bus clock\n");
+ } else {
+ dev_info(dev, "33MHz only bus\n");
+ max_bus_speed = PCI_SPEED_33MHz;
+ }
+
+ /* Bumping the clock may fail so read back the rate */
+ rate = clk_get_rate(p->bus_clk);
+ if (rate == 33000000)
+ cur_bus_speed = PCI_SPEED_33MHz;
+ if (rate == 66000000)
+ cur_bus_speed = PCI_SPEED_66MHz;
+ }
+
+ ret = faraday_pci_parse_map_dma_ranges(p, dev->of_node);
+ if (ret)
+ return ret;
+
+ list_splice_init(&res, &host->windows);
+ ret = pci_scan_root_bus_bridge(host);
+ if (ret) {
+ dev_err(dev, "failed to scan host: %d\n", ret);
+ return ret;
+ }
+ p->bus = host->bus;
+ p->bus->max_bus_speed = max_bus_speed;
+ p->bus->cur_bus_speed = cur_bus_speed;
+
+ pci_bus_assign_resources(p->bus);
+ pci_bus_add_devices(p->bus);
+ pci_free_resource_list(&res);
+
+ return 0;
+}
+
+/*
+ * We encode bridge variants here, we have at least two so it doesn't
+ * hurt to have infrastructure to encompass future variants as well.
+ */
+const struct faraday_pci_variant faraday_regular = {
+ .cascaded_irq = true,
+};
+
+const struct faraday_pci_variant faraday_dual = {
+ .cascaded_irq = false,
+};
+
+static const struct of_device_id faraday_pci_of_match[] = {
+ {
+ .compatible = "faraday,ftpci100",
+ .data = &faraday_regular,
+ },
+ {
+ .compatible = "faraday,ftpci100-dual",
+ .data = &faraday_dual,
+ },
+ {},
+};
+
+static struct platform_driver faraday_pci_driver = {
+ .driver = {
+ .name = "ftpci100",
+ .of_match_table = of_match_ptr(faraday_pci_of_match),
+ .suppress_bind_attrs = true,
+ },
+ .probe = faraday_pci_probe,
+};
+builtin_platform_driver(faraday_pci_driver);
diff --git a/src/kernel/linux/v4.14/drivers/pci/host/pci-host-common.c b/src/kernel/linux/v4.14/drivers/pci/host/pci-host-common.c
new file mode 100644
index 0000000..148896f
--- /dev/null
+++ b/src/kernel/linux/v4.14/drivers/pci/host/pci-host-common.c
@@ -0,0 +1,180 @@
+/*
+ * Generic PCI host driver common code
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ * Copyright (C) 2014 ARM Limited
+ *
+ * Author: Will Deacon <will.deacon@arm.com>
+ */
+
+#include <linux/kernel.h>
+#include <linux/of_address.h>
+#include <linux/of_pci.h>
+#include <linux/pci-ecam.h>
+#include <linux/platform_device.h>
+
+static int gen_pci_parse_request_of_pci_ranges(struct device *dev,
+ struct list_head *resources, struct resource **bus_range)
+{
+ int err, res_valid = 0;
+ struct device_node *np = dev->of_node;
+ resource_size_t iobase;
+ struct resource_entry *win, *tmp;
+
+ err = of_pci_get_host_bridge_resources(np, 0, 0xff, resources, &iobase);
+ if (err)
+ return err;
+
+ err = devm_request_pci_bus_resources(dev, resources);
+ if (err)
+ return err;
+
+ resource_list_for_each_entry_safe(win, tmp, resources) {
+ struct resource *res = win->res;
+
+ switch (resource_type(res)) {
+ case IORESOURCE_IO:
+ err = devm_pci_remap_iospace(dev, res, iobase);
+ if (err) {
+ dev_warn(dev, "error %d: failed to map resource %pR\n",
+ err, res);
+ resource_list_destroy_entry(win);
+ }
+ break;
+ case IORESOURCE_MEM:
+ res_valid |= !(res->flags & IORESOURCE_PREFETCH);
+ break;
+ case IORESOURCE_BUS:
+ *bus_range = res;
+ break;
+ }
+ }
+
+ if (res_valid)
+ return 0;
+
+ dev_err(dev, "non-prefetchable memory resource required\n");
+ return -EINVAL;
+}
+
+static void gen_pci_unmap_cfg(void *ptr)
+{
+ pci_ecam_free((struct pci_config_window *)ptr);
+}
+
+static struct pci_config_window *gen_pci_init(struct device *dev,
+ struct list_head *resources, struct pci_ecam_ops *ops)
+{
+ int err;
+ struct resource cfgres;
+ struct resource *bus_range = NULL;
+ struct pci_config_window *cfg;
+
+ /* Parse our PCI ranges and request their resources */
+ err = gen_pci_parse_request_of_pci_ranges(dev, resources, &bus_range);
+ if (err)
+ goto err_out;
+
+ err = of_address_to_resource(dev->of_node, 0, &cfgres);
+ if (err) {
+ dev_err(dev, "missing \"reg\" property\n");
+ goto err_out;
+ }
+
+ cfg = pci_ecam_create(dev, &cfgres, bus_range, ops);
+ if (IS_ERR(cfg)) {
+ err = PTR_ERR(cfg);
+ goto err_out;
+ }
+
+ err = devm_add_action(dev, gen_pci_unmap_cfg, cfg);
+ if (err) {
+ gen_pci_unmap_cfg(cfg);
+ goto err_out;
+ }
+ return cfg;
+
+err_out:
+ pci_free_resource_list(resources);
+ return ERR_PTR(err);
+}
+
+int pci_host_common_probe(struct platform_device *pdev,
+ struct pci_ecam_ops *ops)
+{
+ const char *type;
+ struct device *dev = &pdev->dev;
+ struct device_node *np = dev->of_node;
+ struct pci_bus *bus, *child;
+ struct pci_host_bridge *bridge;
+ struct pci_config_window *cfg;
+ struct list_head resources;
+ int ret;
+
+ bridge = devm_pci_alloc_host_bridge(dev, 0);
+ if (!bridge)
+ return -ENOMEM;
+
+ type = of_get_property(np, "device_type", NULL);
+ if (!type || strcmp(type, "pci")) {
+ dev_err(dev, "invalid \"device_type\" %s\n", type);
+ return -EINVAL;
+ }
+
+ of_pci_check_probe_only();
+
+ /* Parse and map our Configuration Space windows */
+ INIT_LIST_HEAD(&resources);
+ cfg = gen_pci_init(dev, &resources, ops);
+ if (IS_ERR(cfg))
+ return PTR_ERR(cfg);
+
+ /* Do not reassign resources if probe only */
+ if (!pci_has_flag(PCI_PROBE_ONLY))
+ pci_add_flags(PCI_REASSIGN_ALL_RSRC | PCI_REASSIGN_ALL_BUS);
+
+ list_splice_init(&resources, &bridge->windows);
+ bridge->dev.parent = dev;
+ bridge->sysdata = cfg;
+ bridge->busnr = cfg->busr.start;
+ bridge->ops = &ops->pci_ops;
+ bridge->map_irq = of_irq_parse_and_map_pci;
+ bridge->swizzle_irq = pci_common_swizzle;
+
+ ret = pci_scan_root_bus_bridge(bridge);
+ if (ret < 0) {
+ dev_err(dev, "Scanning root bridge failed");
+ return ret;
+ }
+
+ bus = bridge->bus;
+
+ /*
+ * We insert PCI resources into the iomem_resource and
+ * ioport_resource trees in either pci_bus_claim_resources()
+ * or pci_bus_assign_resources().
+ */
+ if (pci_has_flag(PCI_PROBE_ONLY)) {
+ pci_bus_claim_resources(bus);
+ } else {
+ pci_bus_size_bridges(bus);
+ pci_bus_assign_resources(bus);
+
+ list_for_each_entry(child, &bus->children, node)
+ pcie_bus_configure_settings(child);
+ }
+
+ pci_bus_add_devices(bus);
+ return 0;
+}
diff --git a/src/kernel/linux/v4.14/drivers/pci/host/pci-host-generic.c b/src/kernel/linux/v4.14/drivers/pci/host/pci-host-generic.c
new file mode 100644
index 0000000..7d709a7
--- /dev/null
+++ b/src/kernel/linux/v4.14/drivers/pci/host/pci-host-generic.c
@@ -0,0 +1,67 @@
+/*
+ * Simple, generic PCI host controller driver targetting firmware-initialised
+ * systems and virtual machines (e.g. the PCI emulation provided by kvmtool).
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ * Copyright (C) 2014 ARM Limited
+ *
+ * Author: Will Deacon <will.deacon@arm.com>
+ */
+
+#include <linux/kernel.h>
+#include <linux/init.h>
+#include <linux/of_address.h>
+#include <linux/of_pci.h>
+#include <linux/pci-ecam.h>
+#include <linux/platform_device.h>
+
+static struct pci_ecam_ops gen_pci_cfg_cam_bus_ops = {
+ .bus_shift = 16,
+ .pci_ops = {
+ .map_bus = pci_ecam_map_bus,
+ .read = pci_generic_config_read,
+ .write = pci_generic_config_write,
+ }
+};
+
+static const struct of_device_id gen_pci_of_match[] = {
+ { .compatible = "pci-host-cam-generic",
+ .data = &gen_pci_cfg_cam_bus_ops },
+
+ { .compatible = "pci-host-ecam-generic",
+ .data = &pci_generic_ecam_ops },
+
+ { },
+};
+
+static int gen_pci_probe(struct platform_device *pdev)
+{
+ const struct of_device_id *of_id;
+ struct pci_ecam_ops *ops;
+
+ of_id = of_match_node(gen_pci_of_match, pdev->dev.of_node);
+ ops = (struct pci_ecam_ops *)of_id->data;
+
+ return pci_host_common_probe(pdev, ops);
+}
+
+static struct platform_driver gen_pci_driver = {
+ .driver = {
+ .name = "pci-host-generic",
+ .of_match_table = gen_pci_of_match,
+ .suppress_bind_attrs = true,
+ },
+ .probe = gen_pci_probe,
+};
+builtin_platform_driver(gen_pci_driver);
diff --git a/src/kernel/linux/v4.14/drivers/pci/host/pci-hyperv.c b/src/kernel/linux/v4.14/drivers/pci/host/pci-hyperv.c
new file mode 100644
index 0000000..7082568
--- /dev/null
+++ b/src/kernel/linux/v4.14/drivers/pci/host/pci-hyperv.c
@@ -0,0 +1,2796 @@
+/*
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Author:
+ * Jake Oshins <jakeo@microsoft.com>
+ *
+ * This driver acts as a paravirtual front-end for PCI Express root buses.
+ * When a PCI Express function (either an entire device or an SR-IOV
+ * Virtual Function) is being passed through to the VM, this driver exposes
+ * a new bus to the guest VM. This is modeled as a root PCI bus because
+ * no bridges are being exposed to the VM. In fact, with a "Generation 2"
+ * VM within Hyper-V, there may seem to be no PCI bus at all in the VM
+ * until a device as been exposed using this driver.
+ *
+ * Each root PCI bus has its own PCI domain, which is called "Segment" in
+ * the PCI Firmware Specifications. Thus while each device passed through
+ * to the VM using this front-end will appear at "device 0", the domain will
+ * be unique. Typically, each bus will have one PCI function on it, though
+ * this driver does support more than one.
+ *
+ * In order to map the interrupts from the device through to the guest VM,
+ * this driver also implements an IRQ Domain, which handles interrupts (either
+ * MSI or MSI-X) associated with the functions on the bus. As interrupts are
+ * set up, torn down, or reaffined, this driver communicates with the
+ * underlying hypervisor to adjust the mappings in the I/O MMU so that each
+ * interrupt will be delivered to the correct virtual processor at the right
+ * vector. This driver does not support level-triggered (line-based)
+ * interrupts, and will report that the Interrupt Line register in the
+ * function's configuration space is zero.
+ *
+ * The rest of this driver mostly maps PCI concepts onto underlying Hyper-V
+ * facilities. For instance, the configuration space of a function exposed
+ * by Hyper-V is mapped into a single page of memory space, and the
+ * read and write handlers for config space must be aware of this mechanism.
+ * Similarly, device setup and teardown involves messages sent to and from
+ * the PCI back-end driver in Hyper-V.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published
+ * by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
+ * NON INFRINGEMENT. See the GNU General Public License for more
+ * details.
+ *
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/delay.h>
+#include <linux/semaphore.h>
+#include <linux/irqdomain.h>
+#include <linux/irq.h>
+
+#include <asm/irqdomain.h>
+#include <asm/apic.h>
+#include <linux/msi.h>
+#include <linux/hyperv.h>
+#include <linux/refcount.h>
+#include <asm/mshyperv.h>
+
+/*
+ * Protocol versions. The low word is the minor version, the high word the
+ * major version.
+ */
+
+#define PCI_MAKE_VERSION(major, minor) ((u32)(((major) << 16) | (minor)))
+#define PCI_MAJOR_VERSION(version) ((u32)(version) >> 16)
+#define PCI_MINOR_VERSION(version) ((u32)(version) & 0xff)
+
+enum pci_protocol_version_t {
+ PCI_PROTOCOL_VERSION_1_1 = PCI_MAKE_VERSION(1, 1), /* Win10 */
+ PCI_PROTOCOL_VERSION_1_2 = PCI_MAKE_VERSION(1, 2), /* RS1 */
+};
+
+#define CPU_AFFINITY_ALL -1ULL
+
+/*
+ * Supported protocol versions in the order of probing - highest go
+ * first.
+ */
+static enum pci_protocol_version_t pci_protocol_versions[] = {
+ PCI_PROTOCOL_VERSION_1_2,
+ PCI_PROTOCOL_VERSION_1_1,
+};
+
+/*
+ * Protocol version negotiated by hv_pci_protocol_negotiation().
+ */
+static enum pci_protocol_version_t pci_protocol_version;
+
+#define PCI_CONFIG_MMIO_LENGTH 0x2000
+#define CFG_PAGE_OFFSET 0x1000
+#define CFG_PAGE_SIZE (PCI_CONFIG_MMIO_LENGTH - CFG_PAGE_OFFSET)
+
+#define MAX_SUPPORTED_MSI_MESSAGES 0x400
+
+#define STATUS_REVISION_MISMATCH 0xC0000059
+
+/* space for 32bit serial number as string */
+#define SLOT_NAME_SIZE 11
+
+/*
+ * Message Types
+ */
+
+enum pci_message_type {
+ /*
+ * Version 1.1
+ */
+ PCI_MESSAGE_BASE = 0x42490000,
+ PCI_BUS_RELATIONS = PCI_MESSAGE_BASE + 0,
+ PCI_QUERY_BUS_RELATIONS = PCI_MESSAGE_BASE + 1,
+ PCI_POWER_STATE_CHANGE = PCI_MESSAGE_BASE + 4,
+ PCI_QUERY_RESOURCE_REQUIREMENTS = PCI_MESSAGE_BASE + 5,
+ PCI_QUERY_RESOURCE_RESOURCES = PCI_MESSAGE_BASE + 6,
+ PCI_BUS_D0ENTRY = PCI_MESSAGE_BASE + 7,
+ PCI_BUS_D0EXIT = PCI_MESSAGE_BASE + 8,
+ PCI_READ_BLOCK = PCI_MESSAGE_BASE + 9,
+ PCI_WRITE_BLOCK = PCI_MESSAGE_BASE + 0xA,
+ PCI_EJECT = PCI_MESSAGE_BASE + 0xB,
+ PCI_QUERY_STOP = PCI_MESSAGE_BASE + 0xC,
+ PCI_REENABLE = PCI_MESSAGE_BASE + 0xD,
+ PCI_QUERY_STOP_FAILED = PCI_MESSAGE_BASE + 0xE,
+ PCI_EJECTION_COMPLETE = PCI_MESSAGE_BASE + 0xF,
+ PCI_RESOURCES_ASSIGNED = PCI_MESSAGE_BASE + 0x10,
+ PCI_RESOURCES_RELEASED = PCI_MESSAGE_BASE + 0x11,
+ PCI_INVALIDATE_BLOCK = PCI_MESSAGE_BASE + 0x12,
+ PCI_QUERY_PROTOCOL_VERSION = PCI_MESSAGE_BASE + 0x13,
+ PCI_CREATE_INTERRUPT_MESSAGE = PCI_MESSAGE_BASE + 0x14,
+ PCI_DELETE_INTERRUPT_MESSAGE = PCI_MESSAGE_BASE + 0x15,
+ PCI_RESOURCES_ASSIGNED2 = PCI_MESSAGE_BASE + 0x16,
+ PCI_CREATE_INTERRUPT_MESSAGE2 = PCI_MESSAGE_BASE + 0x17,
+ PCI_DELETE_INTERRUPT_MESSAGE2 = PCI_MESSAGE_BASE + 0x18, /* unused */
+ PCI_MESSAGE_MAXIMUM
+};
+
+/*
+ * Structures defining the virtual PCI Express protocol.
+ */
+
+union pci_version {
+ struct {
+ u16 minor_version;
+ u16 major_version;
+ } parts;
+ u32 version;
+} __packed;
+
+/*
+ * Function numbers are 8-bits wide on Express, as interpreted through ARI,
+ * which is all this driver does. This representation is the one used in
+ * Windows, which is what is expected when sending this back and forth with
+ * the Hyper-V parent partition.
+ */
+union win_slot_encoding {
+ struct {
+ u32 dev:5;
+ u32 func:3;
+ u32 reserved:24;
+ } bits;
+ u32 slot;
+} __packed;
+
+/*
+ * Pretty much as defined in the PCI Specifications.
+ */
+struct pci_function_description {
+ u16 v_id; /* vendor ID */
+ u16 d_id; /* device ID */
+ u8 rev;
+ u8 prog_intf;
+ u8 subclass;
+ u8 base_class;
+ u32 subsystem_id;
+ union win_slot_encoding win_slot;
+ u32 ser; /* serial number */
+} __packed;
+
+/**
+ * struct hv_msi_desc
+ * @vector: IDT entry
+ * @delivery_mode: As defined in Intel's Programmer's
+ * Reference Manual, Volume 3, Chapter 8.
+ * @vector_count: Number of contiguous entries in the
+ * Interrupt Descriptor Table that are
+ * occupied by this Message-Signaled
+ * Interrupt. For "MSI", as first defined
+ * in PCI 2.2, this can be between 1 and
+ * 32. For "MSI-X," as first defined in PCI
+ * 3.0, this must be 1, as each MSI-X table
+ * entry would have its own descriptor.
+ * @reserved: Empty space
+ * @cpu_mask: All the target virtual processors.
+ */
+struct hv_msi_desc {
+ u8 vector;
+ u8 delivery_mode;
+ u16 vector_count;
+ u32 reserved;
+ u64 cpu_mask;
+} __packed;
+
+/**
+ * struct hv_msi_desc2 - 1.2 version of hv_msi_desc
+ * @vector: IDT entry
+ * @delivery_mode: As defined in Intel's Programmer's
+ * Reference Manual, Volume 3, Chapter 8.
+ * @vector_count: Number of contiguous entries in the
+ * Interrupt Descriptor Table that are
+ * occupied by this Message-Signaled
+ * Interrupt. For "MSI", as first defined
+ * in PCI 2.2, this can be between 1 and
+ * 32. For "MSI-X," as first defined in PCI
+ * 3.0, this must be 1, as each MSI-X table
+ * entry would have its own descriptor.
+ * @processor_count: number of bits enabled in array.
+ * @processor_array: All the target virtual processors.
+ */
+struct hv_msi_desc2 {
+ u8 vector;
+ u8 delivery_mode;
+ u16 vector_count;
+ u16 processor_count;
+ u16 processor_array[32];
+} __packed;
+
+/**
+ * struct tran_int_desc
+ * @reserved: unused, padding
+ * @vector_count: same as in hv_msi_desc
+ * @data: This is the "data payload" value that is
+ * written by the device when it generates
+ * a message-signaled interrupt, either MSI
+ * or MSI-X.
+ * @address: This is the address to which the data
+ * payload is written on interrupt
+ * generation.
+ */
+struct tran_int_desc {
+ u16 reserved;
+ u16 vector_count;
+ u32 data;
+ u64 address;
+} __packed;
+
+/*
+ * A generic message format for virtual PCI.
+ * Specific message formats are defined later in the file.
+ */
+
+struct pci_message {
+ u32 type;
+} __packed;
+
+struct pci_child_message {
+ struct pci_message message_type;
+ union win_slot_encoding wslot;
+} __packed;
+
+struct pci_incoming_message {
+ struct vmpacket_descriptor hdr;
+ struct pci_message message_type;
+} __packed;
+
+struct pci_response {
+ struct vmpacket_descriptor hdr;
+ s32 status; /* negative values are failures */
+} __packed;
+
+struct pci_packet {
+ void (*completion_func)(void *context, struct pci_response *resp,
+ int resp_packet_size);
+ void *compl_ctxt;
+
+ struct pci_message message[0];
+};
+
+/*
+ * Specific message types supporting the PCI protocol.
+ */
+
+/*
+ * Version negotiation message. Sent from the guest to the host.
+ * The guest is free to try different versions until the host
+ * accepts the version.
+ *
+ * pci_version: The protocol version requested.
+ * is_last_attempt: If TRUE, this is the last version guest will request.
+ * reservedz: Reserved field, set to zero.
+ */
+
+struct pci_version_request {
+ struct pci_message message_type;
+ u32 protocol_version;
+} __packed;
+
+/*
+ * Bus D0 Entry. This is sent from the guest to the host when the virtual
+ * bus (PCI Express port) is ready for action.
+ */
+
+struct pci_bus_d0_entry {
+ struct pci_message message_type;
+ u32 reserved;
+ u64 mmio_base;
+} __packed;
+
+struct pci_bus_relations {
+ struct pci_incoming_message incoming;
+ u32 device_count;
+ struct pci_function_description func[0];
+} __packed;
+
+struct pci_q_res_req_response {
+ struct vmpacket_descriptor hdr;
+ s32 status; /* negative values are failures */
+ u32 probed_bar[6];
+} __packed;
+
+struct pci_set_power {
+ struct pci_message message_type;
+ union win_slot_encoding wslot;
+ u32 power_state; /* In Windows terms */
+ u32 reserved;
+} __packed;
+
+struct pci_set_power_response {
+ struct vmpacket_descriptor hdr;
+ s32 status; /* negative values are failures */
+ union win_slot_encoding wslot;
+ u32 resultant_state; /* In Windows terms */
+ u32 reserved;
+} __packed;
+
+struct pci_resources_assigned {
+ struct pci_message message_type;
+ union win_slot_encoding wslot;
+ u8 memory_range[0x14][6]; /* not used here */
+ u32 msi_descriptors;
+ u32 reserved[4];
+} __packed;
+
+struct pci_resources_assigned2 {
+ struct pci_message message_type;
+ union win_slot_encoding wslot;
+ u8 memory_range[0x14][6]; /* not used here */
+ u32 msi_descriptor_count;
+ u8 reserved[70];
+} __packed;
+
+struct pci_create_interrupt {
+ struct pci_message message_type;
+ union win_slot_encoding wslot;
+ struct hv_msi_desc int_desc;
+} __packed;
+
+struct pci_create_int_response {
+ struct pci_response response;
+ u32 reserved;
+ struct tran_int_desc int_desc;
+} __packed;
+
+struct pci_create_interrupt2 {
+ struct pci_message message_type;
+ union win_slot_encoding wslot;
+ struct hv_msi_desc2 int_desc;
+} __packed;
+
+struct pci_delete_interrupt {
+ struct pci_message message_type;
+ union win_slot_encoding wslot;
+ struct tran_int_desc int_desc;
+} __packed;
+
+struct pci_dev_incoming {
+ struct pci_incoming_message incoming;
+ union win_slot_encoding wslot;
+} __packed;
+
+struct pci_eject_response {
+ struct pci_message message_type;
+ union win_slot_encoding wslot;
+ u32 status;
+} __packed;
+
+static int pci_ring_size = (4 * PAGE_SIZE);
+
+/*
+ * Definitions or interrupt steering hypercall.
+ */
+#define HV_PARTITION_ID_SELF ((u64)-1)
+#define HVCALL_RETARGET_INTERRUPT 0x7e
+
+struct hv_interrupt_entry {
+ u32 source; /* 1 for MSI(-X) */
+ u32 reserved1;
+ u32 address;
+ u32 data;
+};
+
+#define HV_VP_SET_BANK_COUNT_MAX 5 /* current implementation limit */
+
+struct hv_vp_set {
+ u64 format; /* 0 (HvGenericSetSparse4k) */
+ u64 valid_banks;
+ u64 masks[HV_VP_SET_BANK_COUNT_MAX];
+};
+
+/*
+ * flags for hv_device_interrupt_target.flags
+ */
+#define HV_DEVICE_INTERRUPT_TARGET_MULTICAST 1
+#define HV_DEVICE_INTERRUPT_TARGET_PROCESSOR_SET 2
+
+struct hv_device_interrupt_target {
+ u32 vector;
+ u32 flags;
+ union {
+ u64 vp_mask;
+ struct hv_vp_set vp_set;
+ };
+};
+
+struct retarget_msi_interrupt {
+ u64 partition_id; /* use "self" */
+ u64 device_id;
+ struct hv_interrupt_entry int_entry;
+ u64 reserved2;
+ struct hv_device_interrupt_target int_target;
+} __packed;
+
+/*
+ * Driver specific state.
+ */
+
+enum hv_pcibus_state {
+ hv_pcibus_init = 0,
+ hv_pcibus_probed,
+ hv_pcibus_installed,
+ hv_pcibus_removed,
+ hv_pcibus_maximum
+};
+
+struct hv_pcibus_device {
+ struct pci_sysdata sysdata;
+ enum hv_pcibus_state state;
+ atomic_t remove_lock;
+ struct hv_device *hdev;
+ resource_size_t low_mmio_space;
+ resource_size_t high_mmio_space;
+ struct resource *mem_config;
+ struct resource *low_mmio_res;
+ struct resource *high_mmio_res;
+ struct completion *survey_event;
+ struct completion remove_event;
+ struct pci_bus *pci_bus;
+ spinlock_t config_lock; /* Avoid two threads writing index page */
+ spinlock_t device_list_lock; /* Protect lists below */
+ void __iomem *cfg_addr;
+
+ struct list_head resources_for_children;
+
+ struct list_head children;
+ struct list_head dr_list;
+
+ struct msi_domain_info msi_info;
+ struct msi_controller msi_chip;
+ struct irq_domain *irq_domain;
+
+ /* hypercall arg, must not cross page boundary */
+ struct retarget_msi_interrupt retarget_msi_interrupt_params;
+
+ spinlock_t retarget_msi_interrupt_lock;
+
+ struct workqueue_struct *wq;
+};
+
+/*
+ * Tracks "Device Relations" messages from the host, which must be both
+ * processed in order and deferred so that they don't run in the context
+ * of the incoming packet callback.
+ */
+struct hv_dr_work {
+ struct work_struct wrk;
+ struct hv_pcibus_device *bus;
+};
+
+struct hv_dr_state {
+ struct list_head list_entry;
+ u32 device_count;
+ struct pci_function_description func[0];
+};
+
+enum hv_pcichild_state {
+ hv_pcichild_init = 0,
+ hv_pcichild_requirements,
+ hv_pcichild_resourced,
+ hv_pcichild_ejecting,
+ hv_pcichild_maximum
+};
+
+enum hv_pcidev_ref_reason {
+ hv_pcidev_ref_invalid = 0,
+ hv_pcidev_ref_initial,
+ hv_pcidev_ref_by_slot,
+ hv_pcidev_ref_packet,
+ hv_pcidev_ref_pnp,
+ hv_pcidev_ref_childlist,
+ hv_pcidev_irqdata,
+ hv_pcidev_ref_max
+};
+
+struct hv_pci_dev {
+ /* List protected by pci_rescan_remove_lock */
+ struct list_head list_entry;
+ refcount_t refs;
+ enum hv_pcichild_state state;
+ struct pci_slot *pci_slot;
+ struct pci_function_description desc;
+ bool reported_missing;
+ struct hv_pcibus_device *hbus;
+ struct work_struct wrk;
+
+ /*
+ * What would be observed if one wrote 0xFFFFFFFF to a BAR and then
+ * read it back, for each of the BAR offsets within config space.
+ */
+ u32 probed_bar[6];
+};
+
+struct hv_pci_compl {
+ struct completion host_event;
+ s32 completion_status;
+};
+
+static void hv_pci_onchannelcallback(void *context);
+
+/**
+ * hv_pci_generic_compl() - Invoked for a completion packet
+ * @context: Set up by the sender of the packet.
+ * @resp: The response packet
+ * @resp_packet_size: Size in bytes of the packet
+ *
+ * This function is used to trigger an event and report status
+ * for any message for which the completion packet contains a
+ * status and nothing else.
+ */
+static void hv_pci_generic_compl(void *context, struct pci_response *resp,
+ int resp_packet_size)
+{
+ struct hv_pci_compl *comp_pkt = context;
+
+ if (resp_packet_size >= offsetofend(struct pci_response, status))
+ comp_pkt->completion_status = resp->status;
+ else
+ comp_pkt->completion_status = -1;
+
+ complete(&comp_pkt->host_event);
+}
+
+static struct hv_pci_dev *get_pcichild_wslot(struct hv_pcibus_device *hbus,
+ u32 wslot);
+static void get_pcichild(struct hv_pci_dev *hv_pcidev,
+ enum hv_pcidev_ref_reason reason);
+static void put_pcichild(struct hv_pci_dev *hv_pcidev,
+ enum hv_pcidev_ref_reason reason);
+
+static void get_hvpcibus(struct hv_pcibus_device *hv_pcibus);
+static void put_hvpcibus(struct hv_pcibus_device *hv_pcibus);
+
+/*
+ * There is no good way to get notified from vmbus_onoffer_rescind(),
+ * so let's use polling here, since this is not a hot path.
+ */
+static int wait_for_response(struct hv_device *hdev,
+ struct completion *comp)
+{
+ while (true) {
+ if (hdev->channel->rescind) {
+ dev_warn_once(&hdev->device, "The device is gone.\n");
+ return -ENODEV;
+ }
+
+ if (wait_for_completion_timeout(comp, HZ / 10))
+ break;
+ }
+
+ return 0;
+}
+
+/**
+ * devfn_to_wslot() - Convert from Linux PCI slot to Windows
+ * @devfn: The Linux representation of PCI slot
+ *
+ * Windows uses a slightly different representation of PCI slot.
+ *
+ * Return: The Windows representation
+ */
+static u32 devfn_to_wslot(int devfn)
+{
+ union win_slot_encoding wslot;
+
+ wslot.slot = 0;
+ wslot.bits.dev = PCI_SLOT(devfn);
+ wslot.bits.func = PCI_FUNC(devfn);
+
+ return wslot.slot;
+}
+
+/**
+ * wslot_to_devfn() - Convert from Windows PCI slot to Linux
+ * @wslot: The Windows representation of PCI slot
+ *
+ * Windows uses a slightly different representation of PCI slot.
+ *
+ * Return: The Linux representation
+ */
+static int wslot_to_devfn(u32 wslot)
+{
+ union win_slot_encoding slot_no;
+
+ slot_no.slot = wslot;
+ return PCI_DEVFN(slot_no.bits.dev, slot_no.bits.func);
+}
+
+/*
+ * PCI Configuration Space for these root PCI buses is implemented as a pair
+ * of pages in memory-mapped I/O space. Writing to the first page chooses
+ * the PCI function being written or read. Once the first page has been
+ * written to, the following page maps in the entire configuration space of
+ * the function.
+ */
+
+/**
+ * _hv_pcifront_read_config() - Internal PCI config read
+ * @hpdev: The PCI driver's representation of the device
+ * @where: Offset within config space
+ * @size: Size of the transfer
+ * @val: Pointer to the buffer receiving the data
+ */
+static void _hv_pcifront_read_config(struct hv_pci_dev *hpdev, int where,
+ int size, u32 *val)
+{
+ unsigned long flags;
+ void __iomem *addr = hpdev->hbus->cfg_addr + CFG_PAGE_OFFSET + where;
+
+ /*
+ * If the attempt is to read the IDs or the ROM BAR, simulate that.
+ */
+ if (where + size <= PCI_COMMAND) {
+ memcpy(val, ((u8 *)&hpdev->desc.v_id) + where, size);
+ } else if (where >= PCI_CLASS_REVISION && where + size <=
+ PCI_CACHE_LINE_SIZE) {
+ memcpy(val, ((u8 *)&hpdev->desc.rev) + where -
+ PCI_CLASS_REVISION, size);
+ } else if (where >= PCI_SUBSYSTEM_VENDOR_ID && where + size <=
+ PCI_ROM_ADDRESS) {
+ memcpy(val, (u8 *)&hpdev->desc.subsystem_id + where -
+ PCI_SUBSYSTEM_VENDOR_ID, size);
+ } else if (where >= PCI_ROM_ADDRESS && where + size <=
+ PCI_CAPABILITY_LIST) {
+ /* ROM BARs are unimplemented */
+ *val = 0;
+ } else if (where >= PCI_INTERRUPT_LINE && where + size <=
+ PCI_INTERRUPT_PIN) {
+ /*
+ * Interrupt Line and Interrupt PIN are hard-wired to zero
+ * because this front-end only supports message-signaled
+ * interrupts.
+ */
+ *val = 0;
+ } else if (where + size <= CFG_PAGE_SIZE) {
+ spin_lock_irqsave(&hpdev->hbus->config_lock, flags);
+ /* Choose the function to be read. (See comment above) */
+ writel(hpdev->desc.win_slot.slot, hpdev->hbus->cfg_addr);
+ /* Make sure the function was chosen before we start reading. */
+ mb();
+ /* Read from that function's config space. */
+ switch (size) {
+ case 1:
+ *val = readb(addr);
+ break;
+ case 2:
+ *val = readw(addr);
+ break;
+ default:
+ *val = readl(addr);
+ break;
+ }
+ /*
+ * Make sure the write was done before we release the spinlock
+ * allowing consecutive reads/writes.
+ */
+ mb();
+ spin_unlock_irqrestore(&hpdev->hbus->config_lock, flags);
+ } else {
+ dev_err(&hpdev->hbus->hdev->device,
+ "Attempt to read beyond a function's config space.\n");
+ }
+}
+
+static u16 hv_pcifront_get_vendor_id(struct hv_pci_dev *hpdev)
+{
+ u16 ret;
+ unsigned long flags;
+ void __iomem *addr = hpdev->hbus->cfg_addr + CFG_PAGE_OFFSET +
+ PCI_VENDOR_ID;
+
+ spin_lock_irqsave(&hpdev->hbus->config_lock, flags);
+
+ /* Choose the function to be read. (See comment above) */
+ writel(hpdev->desc.win_slot.slot, hpdev->hbus->cfg_addr);
+ /* Make sure the function was chosen before we start reading. */
+ mb();
+ /* Read from that function's config space. */
+ ret = readw(addr);
+ /*
+ * mb() is not required here, because the spin_unlock_irqrestore()
+ * is a barrier.
+ */
+
+ spin_unlock_irqrestore(&hpdev->hbus->config_lock, flags);
+
+ return ret;
+}
+
+/**
+ * _hv_pcifront_write_config() - Internal PCI config write
+ * @hpdev: The PCI driver's representation of the device
+ * @where: Offset within config space
+ * @size: Size of the transfer
+ * @val: The data being transferred
+ */
+static void _hv_pcifront_write_config(struct hv_pci_dev *hpdev, int where,
+ int size, u32 val)
+{
+ unsigned long flags;
+ void __iomem *addr = hpdev->hbus->cfg_addr + CFG_PAGE_OFFSET + where;
+
+ if (where >= PCI_SUBSYSTEM_VENDOR_ID &&
+ where + size <= PCI_CAPABILITY_LIST) {
+ /* SSIDs and ROM BARs are read-only */
+ } else if (where >= PCI_COMMAND && where + size <= CFG_PAGE_SIZE) {
+ spin_lock_irqsave(&hpdev->hbus->config_lock, flags);
+ /* Choose the function to be written. (See comment above) */
+ writel(hpdev->desc.win_slot.slot, hpdev->hbus->cfg_addr);
+ /* Make sure the function was chosen before we start writing. */
+ wmb();
+ /* Write to that function's config space. */
+ switch (size) {
+ case 1:
+ writeb(val, addr);
+ break;
+ case 2:
+ writew(val, addr);
+ break;
+ default:
+ writel(val, addr);
+ break;
+ }
+ /*
+ * Make sure the write was done before we release the spinlock
+ * allowing consecutive reads/writes.
+ */
+ mb();
+ spin_unlock_irqrestore(&hpdev->hbus->config_lock, flags);
+ } else {
+ dev_err(&hpdev->hbus->hdev->device,
+ "Attempt to write beyond a function's config space.\n");
+ }
+}
+
+/**
+ * hv_pcifront_read_config() - Read configuration space
+ * @bus: PCI Bus structure
+ * @devfn: Device/function
+ * @where: Offset from base
+ * @size: Byte/word/dword
+ * @val: Value to be read
+ *
+ * Return: PCIBIOS_SUCCESSFUL on success
+ * PCIBIOS_DEVICE_NOT_FOUND on failure
+ */
+static int hv_pcifront_read_config(struct pci_bus *bus, unsigned int devfn,
+ int where, int size, u32 *val)
+{
+ struct hv_pcibus_device *hbus =
+ container_of(bus->sysdata, struct hv_pcibus_device, sysdata);
+ struct hv_pci_dev *hpdev;
+
+ hpdev = get_pcichild_wslot(hbus, devfn_to_wslot(devfn));
+ if (!hpdev)
+ return PCIBIOS_DEVICE_NOT_FOUND;
+
+ _hv_pcifront_read_config(hpdev, where, size, val);
+
+ put_pcichild(hpdev, hv_pcidev_ref_by_slot);
+ return PCIBIOS_SUCCESSFUL;
+}
+
+/**
+ * hv_pcifront_write_config() - Write configuration space
+ * @bus: PCI Bus structure
+ * @devfn: Device/function
+ * @where: Offset from base
+ * @size: Byte/word/dword
+ * @val: Value to be written to device
+ *
+ * Return: PCIBIOS_SUCCESSFUL on success
+ * PCIBIOS_DEVICE_NOT_FOUND on failure
+ */
+static int hv_pcifront_write_config(struct pci_bus *bus, unsigned int devfn,
+ int where, int size, u32 val)
+{
+ struct hv_pcibus_device *hbus =
+ container_of(bus->sysdata, struct hv_pcibus_device, sysdata);
+ struct hv_pci_dev *hpdev;
+
+ hpdev = get_pcichild_wslot(hbus, devfn_to_wslot(devfn));
+ if (!hpdev)
+ return PCIBIOS_DEVICE_NOT_FOUND;
+
+ _hv_pcifront_write_config(hpdev, where, size, val);
+
+ put_pcichild(hpdev, hv_pcidev_ref_by_slot);
+ return PCIBIOS_SUCCESSFUL;
+}
+
+/* PCIe operations */
+static struct pci_ops hv_pcifront_ops = {
+ .read = hv_pcifront_read_config,
+ .write = hv_pcifront_write_config,
+};
+
+/* Interrupt management hooks */
+static void hv_int_desc_free(struct hv_pci_dev *hpdev,
+ struct tran_int_desc *int_desc)
+{
+ struct pci_delete_interrupt *int_pkt;
+ struct {
+ struct pci_packet pkt;
+ u8 buffer[sizeof(struct pci_delete_interrupt)];
+ } ctxt;
+
+ memset(&ctxt, 0, sizeof(ctxt));
+ int_pkt = (struct pci_delete_interrupt *)&ctxt.pkt.message;
+ int_pkt->message_type.type =
+ PCI_DELETE_INTERRUPT_MESSAGE;
+ int_pkt->wslot.slot = hpdev->desc.win_slot.slot;
+ int_pkt->int_desc = *int_desc;
+ vmbus_sendpacket(hpdev->hbus->hdev->channel, int_pkt, sizeof(*int_pkt),
+ (unsigned long)&ctxt.pkt, VM_PKT_DATA_INBAND, 0);
+ kfree(int_desc);
+}
+
+/**
+ * hv_msi_free() - Free the MSI.
+ * @domain: The interrupt domain pointer
+ * @info: Extra MSI-related context
+ * @irq: Identifies the IRQ.
+ *
+ * The Hyper-V parent partition and hypervisor are tracking the
+ * messages that are in use, keeping the interrupt redirection
+ * table up to date. This callback sends a message that frees
+ * the IRT entry and related tracking nonsense.
+ */
+static void hv_msi_free(struct irq_domain *domain, struct msi_domain_info *info,
+ unsigned int irq)
+{
+ struct hv_pcibus_device *hbus;
+ struct hv_pci_dev *hpdev;
+ struct pci_dev *pdev;
+ struct tran_int_desc *int_desc;
+ struct irq_data *irq_data = irq_domain_get_irq_data(domain, irq);
+ struct msi_desc *msi = irq_data_get_msi_desc(irq_data);
+
+ pdev = msi_desc_to_pci_dev(msi);
+ hbus = info->data;
+ int_desc = irq_data_get_irq_chip_data(irq_data);
+ if (!int_desc)
+ return;
+
+ irq_data->chip_data = NULL;
+ hpdev = get_pcichild_wslot(hbus, devfn_to_wslot(pdev->devfn));
+ if (!hpdev) {
+ kfree(int_desc);
+ return;
+ }
+
+ hv_int_desc_free(hpdev, int_desc);
+ put_pcichild(hpdev, hv_pcidev_ref_by_slot);
+}
+
+static int hv_set_affinity(struct irq_data *data, const struct cpumask *dest,
+ bool force)
+{
+ struct irq_data *parent = data->parent_data;
+
+ return parent->chip->irq_set_affinity(parent, dest, force);
+}
+
+static void hv_irq_mask(struct irq_data *data)
+{
+ pci_msi_mask_irq(data);
+}
+
+/**
+ * hv_irq_unmask() - "Unmask" the IRQ by setting its current
+ * affinity.
+ * @data: Describes the IRQ
+ *
+ * Build new a destination for the MSI and make a hypercall to
+ * update the Interrupt Redirection Table. "Device Logical ID"
+ * is built out of this PCI bus's instance GUID and the function
+ * number of the device.
+ */
+static void hv_irq_unmask(struct irq_data *data)
+{
+ struct msi_desc *msi_desc = irq_data_get_msi_desc(data);
+ struct irq_cfg *cfg = irqd_cfg(data);
+ struct retarget_msi_interrupt *params;
+ struct hv_pcibus_device *hbus;
+ struct cpumask *dest;
+ struct pci_bus *pbus;
+ struct pci_dev *pdev;
+ unsigned long flags;
+ u32 var_size = 0;
+ int cpu_vmbus;
+ int cpu;
+ u64 res;
+
+ dest = irq_data_get_effective_affinity_mask(data);
+ pdev = msi_desc_to_pci_dev(msi_desc);
+ pbus = pdev->bus;
+ hbus = container_of(pbus->sysdata, struct hv_pcibus_device, sysdata);
+
+ spin_lock_irqsave(&hbus->retarget_msi_interrupt_lock, flags);
+
+ params = &hbus->retarget_msi_interrupt_params;
+ memset(params, 0, sizeof(*params));
+ params->partition_id = HV_PARTITION_ID_SELF;
+ params->int_entry.source = 1; /* MSI(-X) */
+ params->int_entry.address = msi_desc->msg.address_lo;
+ params->int_entry.data = msi_desc->msg.data;
+ params->device_id = (hbus->hdev->dev_instance.b[5] << 24) |
+ (hbus->hdev->dev_instance.b[4] << 16) |
+ (hbus->hdev->dev_instance.b[7] << 8) |
+ (hbus->hdev->dev_instance.b[6] & 0xf8) |
+ PCI_FUNC(pdev->devfn);
+ params->int_target.vector = cfg->vector;
+
+ /*
+ * Honoring apic->irq_delivery_mode set to dest_Fixed by
+ * setting the HV_DEVICE_INTERRUPT_TARGET_MULTICAST flag results in a
+ * spurious interrupt storm. Not doing so does not seem to have a
+ * negative effect (yet?).
+ */
+
+ if (pci_protocol_version >= PCI_PROTOCOL_VERSION_1_2) {
+ /*
+ * PCI_PROTOCOL_VERSION_1_2 supports the VP_SET version of the
+ * HVCALL_RETARGET_INTERRUPT hypercall, which also coincides
+ * with >64 VP support.
+ * ms_hyperv.hints & HV_X64_EX_PROCESSOR_MASKS_RECOMMENDED
+ * is not sufficient for this hypercall.
+ */
+ params->int_target.flags |=
+ HV_DEVICE_INTERRUPT_TARGET_PROCESSOR_SET;
+ params->int_target.vp_set.valid_banks =
+ (1ull << HV_VP_SET_BANK_COUNT_MAX) - 1;
+
+ /*
+ * var-sized hypercall, var-size starts after vp_mask (thus
+ * vp_set.format does not count, but vp_set.valid_banks does).
+ */
+ var_size = 1 + HV_VP_SET_BANK_COUNT_MAX;
+
+ for_each_cpu_and(cpu, dest, cpu_online_mask) {
+ cpu_vmbus = hv_cpu_number_to_vp_number(cpu);
+
+ if (cpu_vmbus >= HV_VP_SET_BANK_COUNT_MAX * 64) {
+ dev_err(&hbus->hdev->device,
+ "too high CPU %d", cpu_vmbus);
+ res = 1;
+ goto exit_unlock;
+ }
+
+ params->int_target.vp_set.masks[cpu_vmbus / 64] |=
+ (1ULL << (cpu_vmbus & 63));
+ }
+ } else {
+ for_each_cpu_and(cpu, dest, cpu_online_mask) {
+ params->int_target.vp_mask |=
+ (1ULL << hv_cpu_number_to_vp_number(cpu));
+ }
+ }
+
+ res = hv_do_hypercall(HVCALL_RETARGET_INTERRUPT | (var_size << 17),
+ params, NULL);
+
+exit_unlock:
+ spin_unlock_irqrestore(&hbus->retarget_msi_interrupt_lock, flags);
+
+ if (res) {
+ dev_err(&hbus->hdev->device,
+ "%s() failed: %#llx", __func__, res);
+ return;
+ }
+
+ pci_msi_unmask_irq(data);
+}
+
+struct compose_comp_ctxt {
+ struct hv_pci_compl comp_pkt;
+ struct tran_int_desc int_desc;
+};
+
+static void hv_pci_compose_compl(void *context, struct pci_response *resp,
+ int resp_packet_size)
+{
+ struct compose_comp_ctxt *comp_pkt = context;
+ struct pci_create_int_response *int_resp =
+ (struct pci_create_int_response *)resp;
+
+ comp_pkt->comp_pkt.completion_status = resp->status;
+ comp_pkt->int_desc = int_resp->int_desc;
+ complete(&comp_pkt->comp_pkt.host_event);
+}
+
+static u32 hv_compose_msi_req_v1(
+ struct pci_create_interrupt *int_pkt, struct cpumask *affinity,
+ u32 slot, u8 vector)
+{
+ int_pkt->message_type.type = PCI_CREATE_INTERRUPT_MESSAGE;
+ int_pkt->wslot.slot = slot;
+ int_pkt->int_desc.vector = vector;
+ int_pkt->int_desc.vector_count = 1;
+ int_pkt->int_desc.delivery_mode =
+ (apic->irq_delivery_mode == dest_LowestPrio) ?
+ dest_LowestPrio : dest_Fixed;
+
+ /*
+ * Create MSI w/ dummy vCPU set, overwritten by subsequent retarget in
+ * hv_irq_unmask().
+ */
+ int_pkt->int_desc.cpu_mask = CPU_AFFINITY_ALL;
+
+ return sizeof(*int_pkt);
+}
+
+static u32 hv_compose_msi_req_v2(
+ struct pci_create_interrupt2 *int_pkt, struct cpumask *affinity,
+ u32 slot, u8 vector)
+{
+ int cpu;
+
+ int_pkt->message_type.type = PCI_CREATE_INTERRUPT_MESSAGE2;
+ int_pkt->wslot.slot = slot;
+ int_pkt->int_desc.vector = vector;
+ int_pkt->int_desc.vector_count = 1;
+ int_pkt->int_desc.delivery_mode =
+ (apic->irq_delivery_mode == dest_LowestPrio) ?
+ dest_LowestPrio : dest_Fixed;
+
+ /*
+ * Create MSI w/ dummy vCPU set targeting just one vCPU, overwritten
+ * by subsequent retarget in hv_irq_unmask().
+ */
+ cpu = cpumask_first_and(affinity, cpu_online_mask);
+ int_pkt->int_desc.processor_array[0] =
+ hv_cpu_number_to_vp_number(cpu);
+ int_pkt->int_desc.processor_count = 1;
+
+ return sizeof(*int_pkt);
+}
+
+/**
+ * hv_compose_msi_msg() - Supplies a valid MSI address/data
+ * @data: Everything about this MSI
+ * @msg: Buffer that is filled in by this function
+ *
+ * This function unpacks the IRQ looking for target CPU set, IDT
+ * vector and mode and sends a message to the parent partition
+ * asking for a mapping for that tuple in this partition. The
+ * response supplies a data value and address to which that data
+ * should be written to trigger that interrupt.
+ */
+static void hv_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
+{
+ struct irq_cfg *cfg = irqd_cfg(data);
+ struct hv_pcibus_device *hbus;
+ struct hv_pci_dev *hpdev;
+ struct pci_bus *pbus;
+ struct pci_dev *pdev;
+ struct cpumask *dest;
+ unsigned long flags;
+ struct compose_comp_ctxt comp;
+ struct tran_int_desc *int_desc;
+ struct {
+ struct pci_packet pci_pkt;
+ union {
+ struct pci_create_interrupt v1;
+ struct pci_create_interrupt2 v2;
+ } int_pkts;
+ } __packed ctxt;
+
+ u32 size;
+ int ret;
+
+ pdev = msi_desc_to_pci_dev(irq_data_get_msi_desc(data));
+ dest = irq_data_get_effective_affinity_mask(data);
+ pbus = pdev->bus;
+ hbus = container_of(pbus->sysdata, struct hv_pcibus_device, sysdata);
+ hpdev = get_pcichild_wslot(hbus, devfn_to_wslot(pdev->devfn));
+ if (!hpdev)
+ goto return_null_message;
+
+ /* Free any previous message that might have already been composed. */
+ if (data->chip_data) {
+ int_desc = data->chip_data;
+ data->chip_data = NULL;
+ hv_int_desc_free(hpdev, int_desc);
+ }
+
+ int_desc = kzalloc(sizeof(*int_desc), GFP_ATOMIC);
+ if (!int_desc)
+ goto drop_reference;
+
+ memset(&ctxt, 0, sizeof(ctxt));
+ init_completion(&comp.comp_pkt.host_event);
+ ctxt.pci_pkt.completion_func = hv_pci_compose_compl;
+ ctxt.pci_pkt.compl_ctxt = ∁
+
+ switch (pci_protocol_version) {
+ case PCI_PROTOCOL_VERSION_1_1:
+ size = hv_compose_msi_req_v1(&ctxt.int_pkts.v1,
+ dest,
+ hpdev->desc.win_slot.slot,
+ cfg->vector);
+ break;
+
+ case PCI_PROTOCOL_VERSION_1_2:
+ size = hv_compose_msi_req_v2(&ctxt.int_pkts.v2,
+ dest,
+ hpdev->desc.win_slot.slot,
+ cfg->vector);
+ break;
+
+ default:
+ /* As we only negotiate protocol versions known to this driver,
+ * this path should never hit. However, this is it not a hot
+ * path so we print a message to aid future updates.
+ */
+ dev_err(&hbus->hdev->device,
+ "Unexpected vPCI protocol, update driver.");
+ goto free_int_desc;
+ }
+
+ ret = vmbus_sendpacket(hpdev->hbus->hdev->channel, &ctxt.int_pkts,
+ size, (unsigned long)&ctxt.pci_pkt,
+ VM_PKT_DATA_INBAND,
+ VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
+ if (ret) {
+ dev_err(&hbus->hdev->device,
+ "Sending request for interrupt failed: 0x%x",
+ comp.comp_pkt.completion_status);
+ goto free_int_desc;
+ }
+
+ /*
+ * Since this function is called with IRQ locks held, can't
+ * do normal wait for completion; instead poll.
+ */
+ while (!try_wait_for_completion(&comp.comp_pkt.host_event)) {
+ /* 0xFFFF means an invalid PCI VENDOR ID. */
+ if (hv_pcifront_get_vendor_id(hpdev) == 0xFFFF) {
+ dev_err_once(&hbus->hdev->device,
+ "the device has gone\n");
+ goto free_int_desc;
+ }
+
+ /*
+ * When the higher level interrupt code calls us with
+ * interrupt disabled, we must poll the channel by calling
+ * the channel callback directly when channel->target_cpu is
+ * the current CPU. When the higher level interrupt code
+ * calls us with interrupt enabled, let's add the
+ * local_irq_save()/restore() to avoid race:
+ * hv_pci_onchannelcallback() can also run in tasklet.
+ */
+ local_irq_save(flags);
+
+ if (hbus->hdev->channel->target_cpu == smp_processor_id())
+ hv_pci_onchannelcallback(hbus);
+
+ local_irq_restore(flags);
+
+ if (hpdev->state == hv_pcichild_ejecting) {
+ dev_err_once(&hbus->hdev->device,
+ "the device is being ejected\n");
+ goto free_int_desc;
+ }
+
+ udelay(100);
+ }
+
+ if (comp.comp_pkt.completion_status < 0) {
+ dev_err(&hbus->hdev->device,
+ "Request for interrupt failed: 0x%x",
+ comp.comp_pkt.completion_status);
+ goto free_int_desc;
+ }
+
+ /*
+ * Record the assignment so that this can be unwound later. Using
+ * irq_set_chip_data() here would be appropriate, but the lock it takes
+ * is already held.
+ */
+ *int_desc = comp.int_desc;
+ data->chip_data = int_desc;
+
+ /* Pass up the result. */
+ msg->address_hi = comp.int_desc.address >> 32;
+ msg->address_lo = comp.int_desc.address & 0xffffffff;
+ msg->data = comp.int_desc.data;
+
+ put_pcichild(hpdev, hv_pcidev_ref_by_slot);
+ return;
+
+free_int_desc:
+ kfree(int_desc);
+drop_reference:
+ put_pcichild(hpdev, hv_pcidev_ref_by_slot);
+return_null_message:
+ msg->address_hi = 0;
+ msg->address_lo = 0;
+ msg->data = 0;
+}
+
+/* HW Interrupt Chip Descriptor */
+static struct irq_chip hv_msi_irq_chip = {
+ .name = "Hyper-V PCIe MSI",
+ .irq_compose_msi_msg = hv_compose_msi_msg,
+ .irq_set_affinity = hv_set_affinity,
+ .irq_ack = irq_chip_ack_parent,
+ .irq_mask = hv_irq_mask,
+ .irq_unmask = hv_irq_unmask,
+};
+
+static irq_hw_number_t hv_msi_domain_ops_get_hwirq(struct msi_domain_info *info,
+ msi_alloc_info_t *arg)
+{
+ return arg->msi_hwirq;
+}
+
+static struct msi_domain_ops hv_msi_ops = {
+ .get_hwirq = hv_msi_domain_ops_get_hwirq,
+ .msi_prepare = pci_msi_prepare,
+ .set_desc = pci_msi_set_desc,
+ .msi_free = hv_msi_free,
+};
+
+/**
+ * hv_pcie_init_irq_domain() - Initialize IRQ domain
+ * @hbus: The root PCI bus
+ *
+ * This function creates an IRQ domain which will be used for
+ * interrupts from devices that have been passed through. These
+ * devices only support MSI and MSI-X, not line-based interrupts
+ * or simulations of line-based interrupts through PCIe's
+ * fabric-layer messages. Because interrupts are remapped, we
+ * can support multi-message MSI here.
+ *
+ * Return: '0' on success and error value on failure
+ */
+static int hv_pcie_init_irq_domain(struct hv_pcibus_device *hbus)
+{
+ hbus->msi_info.chip = &hv_msi_irq_chip;
+ hbus->msi_info.ops = &hv_msi_ops;
+ hbus->msi_info.flags = (MSI_FLAG_USE_DEF_DOM_OPS |
+ MSI_FLAG_USE_DEF_CHIP_OPS | MSI_FLAG_MULTI_PCI_MSI |
+ MSI_FLAG_PCI_MSIX);
+ hbus->msi_info.handler = handle_edge_irq;
+ hbus->msi_info.handler_name = "edge";
+ hbus->msi_info.data = hbus;
+ hbus->irq_domain = pci_msi_create_irq_domain(hbus->sysdata.fwnode,
+ &hbus->msi_info,
+ x86_vector_domain);
+ if (!hbus->irq_domain) {
+ dev_err(&hbus->hdev->device,
+ "Failed to build an MSI IRQ domain\n");
+ return -ENODEV;
+ }
+
+ return 0;
+}
+
+/**
+ * get_bar_size() - Get the address space consumed by a BAR
+ * @bar_val: Value that a BAR returned after -1 was written
+ * to it.
+ *
+ * This function returns the size of the BAR, rounded up to 1
+ * page. It has to be rounded up because the hypervisor's page
+ * table entry that maps the BAR into the VM can't specify an
+ * offset within a page. The invariant is that the hypervisor
+ * must place any BARs of smaller than page length at the
+ * beginning of a page.
+ *
+ * Return: Size in bytes of the consumed MMIO space.
+ */
+static u64 get_bar_size(u64 bar_val)
+{
+ return round_up((1 + ~(bar_val & PCI_BASE_ADDRESS_MEM_MASK)),
+ PAGE_SIZE);
+}
+
+/**
+ * survey_child_resources() - Total all MMIO requirements
+ * @hbus: Root PCI bus, as understood by this driver
+ */
+static void survey_child_resources(struct hv_pcibus_device *hbus)
+{
+ struct list_head *iter;
+ struct hv_pci_dev *hpdev;
+ resource_size_t bar_size = 0;
+ unsigned long flags;
+ struct completion *event;
+ u64 bar_val;
+ int i;
+
+ /* If nobody is waiting on the answer, don't compute it. */
+ event = xchg(&hbus->survey_event, NULL);
+ if (!event)
+ return;
+
+ /* If the answer has already been computed, go with it. */
+ if (hbus->low_mmio_space || hbus->high_mmio_space) {
+ complete(event);
+ return;
+ }
+
+ spin_lock_irqsave(&hbus->device_list_lock, flags);
+
+ /*
+ * Due to an interesting quirk of the PCI spec, all memory regions
+ * for a child device are a power of 2 in size and aligned in memory,
+ * so it's sufficient to just add them up without tracking alignment.
+ */
+ list_for_each(iter, &hbus->children) {
+ hpdev = container_of(iter, struct hv_pci_dev, list_entry);
+ for (i = 0; i < 6; i++) {
+ if (hpdev->probed_bar[i] & PCI_BASE_ADDRESS_SPACE_IO)
+ dev_err(&hbus->hdev->device,
+ "There's an I/O BAR in this list!\n");
+
+ if (hpdev->probed_bar[i] != 0) {
+ /*
+ * A probed BAR has all the upper bits set that
+ * can be changed.
+ */
+
+ bar_val = hpdev->probed_bar[i];
+ if (bar_val & PCI_BASE_ADDRESS_MEM_TYPE_64)
+ bar_val |=
+ ((u64)hpdev->probed_bar[++i] << 32);
+ else
+ bar_val |= 0xffffffff00000000ULL;
+
+ bar_size = get_bar_size(bar_val);
+
+ if (bar_val & PCI_BASE_ADDRESS_MEM_TYPE_64)
+ hbus->high_mmio_space += bar_size;
+ else
+ hbus->low_mmio_space += bar_size;
+ }
+ }
+ }
+
+ spin_unlock_irqrestore(&hbus->device_list_lock, flags);
+ complete(event);
+}
+
+/**
+ * prepopulate_bars() - Fill in BARs with defaults
+ * @hbus: Root PCI bus, as understood by this driver
+ *
+ * The core PCI driver code seems much, much happier if the BARs
+ * for a device have values upon first scan. So fill them in.
+ * The algorithm below works down from large sizes to small,
+ * attempting to pack the assignments optimally. The assumption,
+ * enforced in other parts of the code, is that the beginning of
+ * the memory-mapped I/O space will be aligned on the largest
+ * BAR size.
+ */
+static void prepopulate_bars(struct hv_pcibus_device *hbus)
+{
+ resource_size_t high_size = 0;
+ resource_size_t low_size = 0;
+ resource_size_t high_base = 0;
+ resource_size_t low_base = 0;
+ resource_size_t bar_size;
+ struct hv_pci_dev *hpdev;
+ struct list_head *iter;
+ unsigned long flags;
+ u64 bar_val;
+ u32 command;
+ bool high;
+ int i;
+
+ if (hbus->low_mmio_space) {
+ low_size = 1ULL << (63 - __builtin_clzll(hbus->low_mmio_space));
+ low_base = hbus->low_mmio_res->start;
+ }
+
+ if (hbus->high_mmio_space) {
+ high_size = 1ULL <<
+ (63 - __builtin_clzll(hbus->high_mmio_space));
+ high_base = hbus->high_mmio_res->start;
+ }
+
+ spin_lock_irqsave(&hbus->device_list_lock, flags);
+
+ /* Pick addresses for the BARs. */
+ do {
+ list_for_each(iter, &hbus->children) {
+ hpdev = container_of(iter, struct hv_pci_dev,
+ list_entry);
+ for (i = 0; i < 6; i++) {
+ bar_val = hpdev->probed_bar[i];
+ if (bar_val == 0)
+ continue;
+ high = bar_val & PCI_BASE_ADDRESS_MEM_TYPE_64;
+ if (high) {
+ bar_val |=
+ ((u64)hpdev->probed_bar[i + 1]
+ << 32);
+ } else {
+ bar_val |= 0xffffffffULL << 32;
+ }
+ bar_size = get_bar_size(bar_val);
+ if (high) {
+ if (high_size != bar_size) {
+ i++;
+ continue;
+ }
+ _hv_pcifront_write_config(hpdev,
+ PCI_BASE_ADDRESS_0 + (4 * i),
+ 4,
+ (u32)(high_base & 0xffffff00));
+ i++;
+ _hv_pcifront_write_config(hpdev,
+ PCI_BASE_ADDRESS_0 + (4 * i),
+ 4, (u32)(high_base >> 32));
+ high_base += bar_size;
+ } else {
+ if (low_size != bar_size)
+ continue;
+ _hv_pcifront_write_config(hpdev,
+ PCI_BASE_ADDRESS_0 + (4 * i),
+ 4,
+ (u32)(low_base & 0xffffff00));
+ low_base += bar_size;
+ }
+ }
+ if (high_size <= 1 && low_size <= 1) {
+ /* Set the memory enable bit. */
+ _hv_pcifront_read_config(hpdev, PCI_COMMAND, 2,
+ &command);
+ command |= PCI_COMMAND_MEMORY;
+ _hv_pcifront_write_config(hpdev, PCI_COMMAND, 2,
+ command);
+ break;
+ }
+ }
+
+ high_size >>= 1;
+ low_size >>= 1;
+ } while (high_size || low_size);
+
+ spin_unlock_irqrestore(&hbus->device_list_lock, flags);
+}
+
+/*
+ * Assign entries in sysfs pci slot directory.
+ *
+ * Note that this function does not need to lock the children list
+ * because it is called from pci_devices_present_work which
+ * is serialized with hv_eject_device_work because they are on the
+ * same ordered workqueue. Therefore hbus->children list will not change
+ * even when pci_create_slot sleeps.
+ */
+static void hv_pci_assign_slots(struct hv_pcibus_device *hbus)
+{
+ struct hv_pci_dev *hpdev;
+ char name[SLOT_NAME_SIZE];
+ int slot_nr;
+
+ list_for_each_entry(hpdev, &hbus->children, list_entry) {
+ if (hpdev->pci_slot)
+ continue;
+
+ slot_nr = PCI_SLOT(wslot_to_devfn(hpdev->desc.win_slot.slot));
+ snprintf(name, SLOT_NAME_SIZE, "%u", hpdev->desc.ser);
+ hpdev->pci_slot = pci_create_slot(hbus->pci_bus, slot_nr,
+ name, NULL);
+ if (!hpdev->pci_slot)
+ pr_warn("pci_create slot %s failed\n", name);
+ }
+}
+
+/*
+ * Remove entries in sysfs pci slot directory.
+ */
+static void hv_pci_remove_slots(struct hv_pcibus_device *hbus)
+{
+ struct hv_pci_dev *hpdev;
+
+ list_for_each_entry(hpdev, &hbus->children, list_entry) {
+ if (!hpdev->pci_slot)
+ continue;
+ pci_destroy_slot(hpdev->pci_slot);
+ hpdev->pci_slot = NULL;
+ }
+}
+
+/**
+ * create_root_hv_pci_bus() - Expose a new root PCI bus
+ * @hbus: Root PCI bus, as understood by this driver
+ *
+ * Return: 0 on success, -errno on failure
+ */
+static int create_root_hv_pci_bus(struct hv_pcibus_device *hbus)
+{
+ /* Register the device */
+ hbus->pci_bus = pci_create_root_bus(&hbus->hdev->device,
+ 0, /* bus number is always zero */
+ &hv_pcifront_ops,
+ &hbus->sysdata,
+ &hbus->resources_for_children);
+ if (!hbus->pci_bus)
+ return -ENODEV;
+
+ hbus->pci_bus->msi = &hbus->msi_chip;
+ hbus->pci_bus->msi->dev = &hbus->hdev->device;
+
+ pci_lock_rescan_remove();
+ pci_scan_child_bus(hbus->pci_bus);
+ pci_bus_assign_resources(hbus->pci_bus);
+ hv_pci_assign_slots(hbus);
+ pci_bus_add_devices(hbus->pci_bus);
+ pci_unlock_rescan_remove();
+ hbus->state = hv_pcibus_installed;
+ return 0;
+}
+
+struct q_res_req_compl {
+ struct completion host_event;
+ struct hv_pci_dev *hpdev;
+};
+
+/**
+ * q_resource_requirements() - Query Resource Requirements
+ * @context: The completion context.
+ * @resp: The response that came from the host.
+ * @resp_packet_size: The size in bytes of resp.
+ *
+ * This function is invoked on completion of a Query Resource
+ * Requirements packet.
+ */
+static void q_resource_requirements(void *context, struct pci_response *resp,
+ int resp_packet_size)
+{
+ struct q_res_req_compl *completion = context;
+ struct pci_q_res_req_response *q_res_req =
+ (struct pci_q_res_req_response *)resp;
+ int i;
+
+ if (resp->status < 0) {
+ dev_err(&completion->hpdev->hbus->hdev->device,
+ "query resource requirements failed: %x\n",
+ resp->status);
+ } else {
+ for (i = 0; i < 6; i++) {
+ completion->hpdev->probed_bar[i] =
+ q_res_req->probed_bar[i];
+ }
+ }
+
+ complete(&completion->host_event);
+}
+
+static void get_pcichild(struct hv_pci_dev *hpdev,
+ enum hv_pcidev_ref_reason reason)
+{
+ refcount_inc(&hpdev->refs);
+}
+
+static void put_pcichild(struct hv_pci_dev *hpdev,
+ enum hv_pcidev_ref_reason reason)
+{
+ if (refcount_dec_and_test(&hpdev->refs))
+ kfree(hpdev);
+}
+
+/**
+ * new_pcichild_device() - Create a new child device
+ * @hbus: The internal struct tracking this root PCI bus.
+ * @desc: The information supplied so far from the host
+ * about the device.
+ *
+ * This function creates the tracking structure for a new child
+ * device and kicks off the process of figuring out what it is.
+ *
+ * Return: Pointer to the new tracking struct
+ */
+static struct hv_pci_dev *new_pcichild_device(struct hv_pcibus_device *hbus,
+ struct pci_function_description *desc)
+{
+ struct hv_pci_dev *hpdev;
+ struct pci_child_message *res_req;
+ struct q_res_req_compl comp_pkt;
+ struct {
+ struct pci_packet init_packet;
+ u8 buffer[sizeof(struct pci_child_message)];
+ } pkt;
+ unsigned long flags;
+ int ret;
+
+ hpdev = kzalloc(sizeof(*hpdev), GFP_ATOMIC);
+ if (!hpdev)
+ return NULL;
+
+ hpdev->hbus = hbus;
+
+ memset(&pkt, 0, sizeof(pkt));
+ init_completion(&comp_pkt.host_event);
+ comp_pkt.hpdev = hpdev;
+ pkt.init_packet.compl_ctxt = &comp_pkt;
+ pkt.init_packet.completion_func = q_resource_requirements;
+ res_req = (struct pci_child_message *)&pkt.init_packet.message;
+ res_req->message_type.type = PCI_QUERY_RESOURCE_REQUIREMENTS;
+ res_req->wslot.slot = desc->win_slot.slot;
+
+ ret = vmbus_sendpacket(hbus->hdev->channel, res_req,
+ sizeof(struct pci_child_message),
+ (unsigned long)&pkt.init_packet,
+ VM_PKT_DATA_INBAND,
+ VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
+ if (ret)
+ goto error;
+
+ if (wait_for_response(hbus->hdev, &comp_pkt.host_event))
+ goto error;
+
+ hpdev->desc = *desc;
+ refcount_set(&hpdev->refs, 1);
+ get_pcichild(hpdev, hv_pcidev_ref_childlist);
+ spin_lock_irqsave(&hbus->device_list_lock, flags);
+
+ list_add_tail(&hpdev->list_entry, &hbus->children);
+ spin_unlock_irqrestore(&hbus->device_list_lock, flags);
+ return hpdev;
+
+error:
+ kfree(hpdev);
+ return NULL;
+}
+
+/**
+ * get_pcichild_wslot() - Find device from slot
+ * @hbus: Root PCI bus, as understood by this driver
+ * @wslot: Location on the bus
+ *
+ * This function looks up a PCI device and returns the internal
+ * representation of it. It acquires a reference on it, so that
+ * the device won't be deleted while somebody is using it. The
+ * caller is responsible for calling put_pcichild() to release
+ * this reference.
+ *
+ * Return: Internal representation of a PCI device
+ */
+static struct hv_pci_dev *get_pcichild_wslot(struct hv_pcibus_device *hbus,
+ u32 wslot)
+{
+ unsigned long flags;
+ struct hv_pci_dev *iter, *hpdev = NULL;
+
+ spin_lock_irqsave(&hbus->device_list_lock, flags);
+ list_for_each_entry(iter, &hbus->children, list_entry) {
+ if (iter->desc.win_slot.slot == wslot) {
+ hpdev = iter;
+ get_pcichild(hpdev, hv_pcidev_ref_by_slot);
+ break;
+ }
+ }
+ spin_unlock_irqrestore(&hbus->device_list_lock, flags);
+
+ return hpdev;
+}
+
+/**
+ * pci_devices_present_work() - Handle new list of child devices
+ * @work: Work struct embedded in struct hv_dr_work
+ *
+ * "Bus Relations" is the Windows term for "children of this
+ * bus." The terminology is preserved here for people trying to
+ * debug the interaction between Hyper-V and Linux. This
+ * function is called when the parent partition reports a list
+ * of functions that should be observed under this PCI Express
+ * port (bus).
+ *
+ * This function updates the list, and must tolerate being
+ * called multiple times with the same information. The typical
+ * number of child devices is one, with very atypical cases
+ * involving three or four, so the algorithms used here can be
+ * simple and inefficient.
+ *
+ * It must also treat the omission of a previously observed device as
+ * notification that the device no longer exists.
+ *
+ * Note that this function is serialized with hv_eject_device_work(),
+ * because both are pushed to the ordered workqueue hbus->wq.
+ */
+static void pci_devices_present_work(struct work_struct *work)
+{
+ u32 child_no;
+ bool found;
+ struct list_head *iter;
+ struct pci_function_description *new_desc;
+ struct hv_pci_dev *hpdev;
+ struct hv_pcibus_device *hbus;
+ struct list_head removed;
+ struct hv_dr_work *dr_wrk;
+ struct hv_dr_state *dr = NULL;
+ unsigned long flags;
+
+ dr_wrk = container_of(work, struct hv_dr_work, wrk);
+ hbus = dr_wrk->bus;
+ kfree(dr_wrk);
+
+ INIT_LIST_HEAD(&removed);
+
+ /* Pull this off the queue and process it if it was the last one. */
+ spin_lock_irqsave(&hbus->device_list_lock, flags);
+ while (!list_empty(&hbus->dr_list)) {
+ dr = list_first_entry(&hbus->dr_list, struct hv_dr_state,
+ list_entry);
+ list_del(&dr->list_entry);
+
+ /* Throw this away if the list still has stuff in it. */
+ if (!list_empty(&hbus->dr_list)) {
+ kfree(dr);
+ continue;
+ }
+ }
+ spin_unlock_irqrestore(&hbus->device_list_lock, flags);
+
+ if (!dr) {
+ put_hvpcibus(hbus);
+ return;
+ }
+
+ /* First, mark all existing children as reported missing. */
+ spin_lock_irqsave(&hbus->device_list_lock, flags);
+ list_for_each(iter, &hbus->children) {
+ hpdev = container_of(iter, struct hv_pci_dev,
+ list_entry);
+ hpdev->reported_missing = true;
+ }
+ spin_unlock_irqrestore(&hbus->device_list_lock, flags);
+
+ /* Next, add back any reported devices. */
+ for (child_no = 0; child_no < dr->device_count; child_no++) {
+ found = false;
+ new_desc = &dr->func[child_no];
+
+ spin_lock_irqsave(&hbus->device_list_lock, flags);
+ list_for_each(iter, &hbus->children) {
+ hpdev = container_of(iter, struct hv_pci_dev,
+ list_entry);
+ if ((hpdev->desc.win_slot.slot ==
+ new_desc->win_slot.slot) &&
+ (hpdev->desc.v_id == new_desc->v_id) &&
+ (hpdev->desc.d_id == new_desc->d_id) &&
+ (hpdev->desc.ser == new_desc->ser)) {
+ hpdev->reported_missing = false;
+ found = true;
+ }
+ }
+ spin_unlock_irqrestore(&hbus->device_list_lock, flags);
+
+ if (!found) {
+ hpdev = new_pcichild_device(hbus, new_desc);
+ if (!hpdev)
+ dev_err(&hbus->hdev->device,
+ "couldn't record a child device.\n");
+ }
+ }
+
+ /* Move missing children to a list on the stack. */
+ spin_lock_irqsave(&hbus->device_list_lock, flags);
+ do {
+ found = false;
+ list_for_each(iter, &hbus->children) {
+ hpdev = container_of(iter, struct hv_pci_dev,
+ list_entry);
+ if (hpdev->reported_missing) {
+ found = true;
+ put_pcichild(hpdev, hv_pcidev_ref_childlist);
+ list_move_tail(&hpdev->list_entry, &removed);
+ break;
+ }
+ }
+ } while (found);
+ spin_unlock_irqrestore(&hbus->device_list_lock, flags);
+
+ /* Delete everything that should no longer exist. */
+ while (!list_empty(&removed)) {
+ hpdev = list_first_entry(&removed, struct hv_pci_dev,
+ list_entry);
+ list_del(&hpdev->list_entry);
+
+ if (hpdev->pci_slot)
+ pci_destroy_slot(hpdev->pci_slot);
+
+ put_pcichild(hpdev, hv_pcidev_ref_initial);
+ }
+
+ switch (hbus->state) {
+ case hv_pcibus_installed:
+ /*
+ * Tell the core to rescan bus
+ * because there may have been changes.
+ */
+ pci_lock_rescan_remove();
+ pci_scan_child_bus(hbus->pci_bus);
+ hv_pci_assign_slots(hbus);
+ pci_unlock_rescan_remove();
+ break;
+
+ case hv_pcibus_init:
+ case hv_pcibus_probed:
+ survey_child_resources(hbus);
+ break;
+
+ default:
+ break;
+ }
+
+ put_hvpcibus(hbus);
+ kfree(dr);
+}
+
+/**
+ * hv_pci_devices_present() - Handles list of new children
+ * @hbus: Root PCI bus, as understood by this driver
+ * @relations: Packet from host listing children
+ *
+ * This function is invoked whenever a new list of devices for
+ * this bus appears.
+ */
+static void hv_pci_devices_present(struct hv_pcibus_device *hbus,
+ struct pci_bus_relations *relations)
+{
+ struct hv_dr_state *dr;
+ struct hv_dr_work *dr_wrk;
+ unsigned long flags;
+
+ dr_wrk = kzalloc(sizeof(*dr_wrk), GFP_NOWAIT);
+ if (!dr_wrk)
+ return;
+
+ dr = kzalloc(offsetof(struct hv_dr_state, func) +
+ (sizeof(struct pci_function_description) *
+ (relations->device_count)), GFP_NOWAIT);
+ if (!dr) {
+ kfree(dr_wrk);
+ return;
+ }
+
+ INIT_WORK(&dr_wrk->wrk, pci_devices_present_work);
+ dr_wrk->bus = hbus;
+ dr->device_count = relations->device_count;
+ if (dr->device_count != 0) {
+ memcpy(dr->func, relations->func,
+ sizeof(struct pci_function_description) *
+ dr->device_count);
+ }
+
+ spin_lock_irqsave(&hbus->device_list_lock, flags);
+ list_add_tail(&dr->list_entry, &hbus->dr_list);
+ spin_unlock_irqrestore(&hbus->device_list_lock, flags);
+
+ get_hvpcibus(hbus);
+ queue_work(hbus->wq, &dr_wrk->wrk);
+}
+
+/**
+ * hv_eject_device_work() - Asynchronously handles ejection
+ * @work: Work struct embedded in internal device struct
+ *
+ * This function handles ejecting a device. Windows will
+ * attempt to gracefully eject a device, waiting 60 seconds to
+ * hear back from the guest OS that this completed successfully.
+ * If this timer expires, the device will be forcibly removed.
+ */
+static void hv_eject_device_work(struct work_struct *work)
+{
+ struct pci_eject_response *ejct_pkt;
+ struct hv_pcibus_device *hbus;
+ struct hv_pci_dev *hpdev;
+ struct pci_dev *pdev;
+ unsigned long flags;
+ int wslot;
+ struct {
+ struct pci_packet pkt;
+ u8 buffer[sizeof(struct pci_eject_response)];
+ } ctxt;
+
+ hpdev = container_of(work, struct hv_pci_dev, wrk);
+ hbus = hpdev->hbus;
+
+ if (hpdev->state != hv_pcichild_ejecting) {
+ put_pcichild(hpdev, hv_pcidev_ref_pnp);
+ return;
+ }
+
+ /*
+ * Ejection can come before or after the PCI bus has been set up, so
+ * attempt to find it and tear down the bus state, if it exists. This
+ * must be done without constructs like pci_domain_nr(hbus->pci_bus)
+ * because hbus->pci_bus may not exist yet.
+ */
+ wslot = wslot_to_devfn(hpdev->desc.win_slot.slot);
+ pdev = pci_get_domain_bus_and_slot(hbus->sysdata.domain, 0, wslot);
+ if (pdev) {
+ pci_lock_rescan_remove();
+ pci_stop_and_remove_bus_device(pdev);
+ pci_dev_put(pdev);
+ pci_unlock_rescan_remove();
+ }
+
+ spin_lock_irqsave(&hbus->device_list_lock, flags);
+ list_del(&hpdev->list_entry);
+ spin_unlock_irqrestore(&hbus->device_list_lock, flags);
+
+ if (hpdev->pci_slot)
+ pci_destroy_slot(hpdev->pci_slot);
+
+ memset(&ctxt, 0, sizeof(ctxt));
+ ejct_pkt = (struct pci_eject_response *)&ctxt.pkt.message;
+ ejct_pkt->message_type.type = PCI_EJECTION_COMPLETE;
+ ejct_pkt->wslot.slot = hpdev->desc.win_slot.slot;
+ vmbus_sendpacket(hbus->hdev->channel, ejct_pkt,
+ sizeof(*ejct_pkt), (unsigned long)&ctxt.pkt,
+ VM_PKT_DATA_INBAND, 0);
+
+ put_pcichild(hpdev, hv_pcidev_ref_childlist);
+ put_pcichild(hpdev, hv_pcidev_ref_initial);
+ put_pcichild(hpdev, hv_pcidev_ref_pnp);
+
+ /* hpdev has been freed. Do not use it any more. */
+ put_hvpcibus(hbus);
+}
+
+/**
+ * hv_pci_eject_device() - Handles device ejection
+ * @hpdev: Internal device tracking struct
+ *
+ * This function is invoked when an ejection packet arrives. It
+ * just schedules work so that we don't re-enter the packet
+ * delivery code handling the ejection.
+ */
+static void hv_pci_eject_device(struct hv_pci_dev *hpdev)
+{
+ hpdev->state = hv_pcichild_ejecting;
+ get_pcichild(hpdev, hv_pcidev_ref_pnp);
+ INIT_WORK(&hpdev->wrk, hv_eject_device_work);
+ get_hvpcibus(hpdev->hbus);
+ queue_work(hpdev->hbus->wq, &hpdev->wrk);
+}
+
+/**
+ * hv_pci_onchannelcallback() - Handles incoming packets
+ * @context: Internal bus tracking struct
+ *
+ * This function is invoked whenever the host sends a packet to
+ * this channel (which is private to this root PCI bus).
+ */
+static void hv_pci_onchannelcallback(void *context)
+{
+ const int packet_size = 0x100;
+ int ret;
+ struct hv_pcibus_device *hbus = context;
+ u32 bytes_recvd;
+ u64 req_id;
+ struct vmpacket_descriptor *desc;
+ unsigned char *buffer;
+ int bufferlen = packet_size;
+ struct pci_packet *comp_packet;
+ struct pci_response *response;
+ struct pci_incoming_message *new_message;
+ struct pci_bus_relations *bus_rel;
+ struct pci_dev_incoming *dev_message;
+ struct hv_pci_dev *hpdev;
+
+ buffer = kmalloc(bufferlen, GFP_ATOMIC);
+ if (!buffer)
+ return;
+
+ while (1) {
+ ret = vmbus_recvpacket_raw(hbus->hdev->channel, buffer,
+ bufferlen, &bytes_recvd, &req_id);
+
+ if (ret == -ENOBUFS) {
+ kfree(buffer);
+ /* Handle large packet */
+ bufferlen = bytes_recvd;
+ buffer = kmalloc(bytes_recvd, GFP_ATOMIC);
+ if (!buffer)
+ return;
+ continue;
+ }
+
+ /* Zero length indicates there are no more packets. */
+ if (ret || !bytes_recvd)
+ break;
+
+ /*
+ * All incoming packets must be at least as large as a
+ * response.
+ */
+ if (bytes_recvd <= sizeof(struct pci_response))
+ continue;
+ desc = (struct vmpacket_descriptor *)buffer;
+
+ switch (desc->type) {
+ case VM_PKT_COMP:
+
+ /*
+ * The host is trusted, and thus it's safe to interpret
+ * this transaction ID as a pointer.
+ */
+ comp_packet = (struct pci_packet *)req_id;
+ response = (struct pci_response *)buffer;
+ comp_packet->completion_func(comp_packet->compl_ctxt,
+ response,
+ bytes_recvd);
+ break;
+
+ case VM_PKT_DATA_INBAND:
+
+ new_message = (struct pci_incoming_message *)buffer;
+ switch (new_message->message_type.type) {
+ case PCI_BUS_RELATIONS:
+
+ bus_rel = (struct pci_bus_relations *)buffer;
+ if (bytes_recvd <
+ offsetof(struct pci_bus_relations, func) +
+ (sizeof(struct pci_function_description) *
+ (bus_rel->device_count))) {
+ dev_err(&hbus->hdev->device,
+ "bus relations too small\n");
+ break;
+ }
+
+ hv_pci_devices_present(hbus, bus_rel);
+ break;
+
+ case PCI_EJECT:
+
+ dev_message = (struct pci_dev_incoming *)buffer;
+ hpdev = get_pcichild_wslot(hbus,
+ dev_message->wslot.slot);
+ if (hpdev) {
+ hv_pci_eject_device(hpdev);
+ put_pcichild(hpdev,
+ hv_pcidev_ref_by_slot);
+ }
+ break;
+
+ default:
+ dev_warn(&hbus->hdev->device,
+ "Unimplemented protocol message %x\n",
+ new_message->message_type.type);
+ break;
+ }
+ break;
+
+ default:
+ dev_err(&hbus->hdev->device,
+ "unhandled packet type %d, tid %llx len %d\n",
+ desc->type, req_id, bytes_recvd);
+ break;
+ }
+ }
+
+ kfree(buffer);
+}
+
+/**
+ * hv_pci_protocol_negotiation() - Set up protocol
+ * @hdev: VMBus's tracking struct for this root PCI bus
+ *
+ * This driver is intended to support running on Windows 10
+ * (server) and later versions. It will not run on earlier
+ * versions, as they assume that many of the operations which
+ * Linux needs accomplished with a spinlock held were done via
+ * asynchronous messaging via VMBus. Windows 10 increases the
+ * surface area of PCI emulation so that these actions can take
+ * place by suspending a virtual processor for their duration.
+ *
+ * This function negotiates the channel protocol version,
+ * failing if the host doesn't support the necessary protocol
+ * level.
+ */
+static int hv_pci_protocol_negotiation(struct hv_device *hdev)
+{
+ struct pci_version_request *version_req;
+ struct hv_pci_compl comp_pkt;
+ struct pci_packet *pkt;
+ int ret;
+ int i;
+
+ /*
+ * Initiate the handshake with the host and negotiate
+ * a version that the host can support. We start with the
+ * highest version number and go down if the host cannot
+ * support it.
+ */
+ pkt = kzalloc(sizeof(*pkt) + sizeof(*version_req), GFP_KERNEL);
+ if (!pkt)
+ return -ENOMEM;
+
+ init_completion(&comp_pkt.host_event);
+ pkt->completion_func = hv_pci_generic_compl;
+ pkt->compl_ctxt = &comp_pkt;
+ version_req = (struct pci_version_request *)&pkt->message;
+ version_req->message_type.type = PCI_QUERY_PROTOCOL_VERSION;
+
+ for (i = 0; i < ARRAY_SIZE(pci_protocol_versions); i++) {
+ version_req->protocol_version = pci_protocol_versions[i];
+ ret = vmbus_sendpacket(hdev->channel, version_req,
+ sizeof(struct pci_version_request),
+ (unsigned long)pkt, VM_PKT_DATA_INBAND,
+ VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
+ if (!ret)
+ ret = wait_for_response(hdev, &comp_pkt.host_event);
+
+ if (ret) {
+ dev_err(&hdev->device,
+ "PCI Pass-through VSP failed to request version: %d",
+ ret);
+ goto exit;
+ }
+
+ if (comp_pkt.completion_status >= 0) {
+ pci_protocol_version = pci_protocol_versions[i];
+ dev_info(&hdev->device,
+ "PCI VMBus probing: Using version %#x\n",
+ pci_protocol_version);
+ goto exit;
+ }
+
+ if (comp_pkt.completion_status != STATUS_REVISION_MISMATCH) {
+ dev_err(&hdev->device,
+ "PCI Pass-through VSP failed version request: %#x",
+ comp_pkt.completion_status);
+ ret = -EPROTO;
+ goto exit;
+ }
+
+ reinit_completion(&comp_pkt.host_event);
+ }
+
+ dev_err(&hdev->device,
+ "PCI pass-through VSP failed to find supported version");
+ ret = -EPROTO;
+
+exit:
+ kfree(pkt);
+ return ret;
+}
+
+/**
+ * hv_pci_free_bridge_windows() - Release memory regions for the
+ * bus
+ * @hbus: Root PCI bus, as understood by this driver
+ */
+static void hv_pci_free_bridge_windows(struct hv_pcibus_device *hbus)
+{
+ /*
+ * Set the resources back to the way they looked when they
+ * were allocated by setting IORESOURCE_BUSY again.
+ */
+
+ if (hbus->low_mmio_space && hbus->low_mmio_res) {
+ hbus->low_mmio_res->flags |= IORESOURCE_BUSY;
+ vmbus_free_mmio(hbus->low_mmio_res->start,
+ resource_size(hbus->low_mmio_res));
+ }
+
+ if (hbus->high_mmio_space && hbus->high_mmio_res) {
+ hbus->high_mmio_res->flags |= IORESOURCE_BUSY;
+ vmbus_free_mmio(hbus->high_mmio_res->start,
+ resource_size(hbus->high_mmio_res));
+ }
+}
+
+/**
+ * hv_pci_allocate_bridge_windows() - Allocate memory regions
+ * for the bus
+ * @hbus: Root PCI bus, as understood by this driver
+ *
+ * This function calls vmbus_allocate_mmio(), which is itself a
+ * bit of a compromise. Ideally, we might change the pnp layer
+ * in the kernel such that it comprehends either PCI devices
+ * which are "grandchildren of ACPI," with some intermediate bus
+ * node (in this case, VMBus) or change it such that it
+ * understands VMBus. The pnp layer, however, has been declared
+ * deprecated, and not subject to change.
+ *
+ * The workaround, implemented here, is to ask VMBus to allocate
+ * MMIO space for this bus. VMBus itself knows which ranges are
+ * appropriate by looking at its own ACPI objects. Then, after
+ * these ranges are claimed, they're modified to look like they
+ * would have looked if the ACPI and pnp code had allocated
+ * bridge windows. These descriptors have to exist in this form
+ * in order to satisfy the code which will get invoked when the
+ * endpoint PCI function driver calls request_mem_region() or
+ * request_mem_region_exclusive().
+ *
+ * Return: 0 on success, -errno on failure
+ */
+static int hv_pci_allocate_bridge_windows(struct hv_pcibus_device *hbus)
+{
+ resource_size_t align;
+ int ret;
+
+ if (hbus->low_mmio_space) {
+ align = 1ULL << (63 - __builtin_clzll(hbus->low_mmio_space));
+ ret = vmbus_allocate_mmio(&hbus->low_mmio_res, hbus->hdev, 0,
+ (u64)(u32)0xffffffff,
+ hbus->low_mmio_space,
+ align, false);
+ if (ret) {
+ dev_err(&hbus->hdev->device,
+ "Need %#llx of low MMIO space. Consider reconfiguring the VM.\n",
+ hbus->low_mmio_space);
+ return ret;
+ }
+
+ /* Modify this resource to become a bridge window. */
+ hbus->low_mmio_res->flags |= IORESOURCE_WINDOW;
+ hbus->low_mmio_res->flags &= ~IORESOURCE_BUSY;
+ pci_add_resource(&hbus->resources_for_children,
+ hbus->low_mmio_res);
+ }
+
+ if (hbus->high_mmio_space) {
+ align = 1ULL << (63 - __builtin_clzll(hbus->high_mmio_space));
+ ret = vmbus_allocate_mmio(&hbus->high_mmio_res, hbus->hdev,
+ 0x100000000, -1,
+ hbus->high_mmio_space, align,
+ false);
+ if (ret) {
+ dev_err(&hbus->hdev->device,
+ "Need %#llx of high MMIO space. Consider reconfiguring the VM.\n",
+ hbus->high_mmio_space);
+ goto release_low_mmio;
+ }
+
+ /* Modify this resource to become a bridge window. */
+ hbus->high_mmio_res->flags |= IORESOURCE_WINDOW;
+ hbus->high_mmio_res->flags &= ~IORESOURCE_BUSY;
+ pci_add_resource(&hbus->resources_for_children,
+ hbus->high_mmio_res);
+ }
+
+ return 0;
+
+release_low_mmio:
+ if (hbus->low_mmio_res) {
+ vmbus_free_mmio(hbus->low_mmio_res->start,
+ resource_size(hbus->low_mmio_res));
+ }
+
+ return ret;
+}
+
+/**
+ * hv_allocate_config_window() - Find MMIO space for PCI Config
+ * @hbus: Root PCI bus, as understood by this driver
+ *
+ * This function claims memory-mapped I/O space for accessing
+ * configuration space for the functions on this bus.
+ *
+ * Return: 0 on success, -errno on failure
+ */
+static int hv_allocate_config_window(struct hv_pcibus_device *hbus)
+{
+ int ret;
+
+ /*
+ * Set up a region of MMIO space to use for accessing configuration
+ * space.
+ */
+ ret = vmbus_allocate_mmio(&hbus->mem_config, hbus->hdev, 0, -1,
+ PCI_CONFIG_MMIO_LENGTH, 0x1000, false);
+ if (ret)
+ return ret;
+
+ /*
+ * vmbus_allocate_mmio() gets used for allocating both device endpoint
+ * resource claims (those which cannot be overlapped) and the ranges
+ * which are valid for the children of this bus, which are intended
+ * to be overlapped by those children. Set the flag on this claim
+ * meaning that this region can't be overlapped.
+ */
+
+ hbus->mem_config->flags |= IORESOURCE_BUSY;
+
+ return 0;
+}
+
+static void hv_free_config_window(struct hv_pcibus_device *hbus)
+{
+ vmbus_free_mmio(hbus->mem_config->start, PCI_CONFIG_MMIO_LENGTH);
+}
+
+/**
+ * hv_pci_enter_d0() - Bring the "bus" into the D0 power state
+ * @hdev: VMBus's tracking struct for this root PCI bus
+ *
+ * Return: 0 on success, -errno on failure
+ */
+static int hv_pci_enter_d0(struct hv_device *hdev)
+{
+ struct hv_pcibus_device *hbus = hv_get_drvdata(hdev);
+ struct pci_bus_d0_entry *d0_entry;
+ struct hv_pci_compl comp_pkt;
+ struct pci_packet *pkt;
+ int ret;
+
+ /*
+ * Tell the host that the bus is ready to use, and moved into the
+ * powered-on state. This includes telling the host which region
+ * of memory-mapped I/O space has been chosen for configuration space
+ * access.
+ */
+ pkt = kzalloc(sizeof(*pkt) + sizeof(*d0_entry), GFP_KERNEL);
+ if (!pkt)
+ return -ENOMEM;
+
+ init_completion(&comp_pkt.host_event);
+ pkt->completion_func = hv_pci_generic_compl;
+ pkt->compl_ctxt = &comp_pkt;
+ d0_entry = (struct pci_bus_d0_entry *)&pkt->message;
+ d0_entry->message_type.type = PCI_BUS_D0ENTRY;
+ d0_entry->mmio_base = hbus->mem_config->start;
+
+ ret = vmbus_sendpacket(hdev->channel, d0_entry, sizeof(*d0_entry),
+ (unsigned long)pkt, VM_PKT_DATA_INBAND,
+ VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
+ if (!ret)
+ ret = wait_for_response(hdev, &comp_pkt.host_event);
+
+ if (ret)
+ goto exit;
+
+ if (comp_pkt.completion_status < 0) {
+ dev_err(&hdev->device,
+ "PCI Pass-through VSP failed D0 Entry with status %x\n",
+ comp_pkt.completion_status);
+ ret = -EPROTO;
+ goto exit;
+ }
+
+ ret = 0;
+
+exit:
+ kfree(pkt);
+ return ret;
+}
+
+/**
+ * hv_pci_query_relations() - Ask host to send list of child
+ * devices
+ * @hdev: VMBus's tracking struct for this root PCI bus
+ *
+ * Return: 0 on success, -errno on failure
+ */
+static int hv_pci_query_relations(struct hv_device *hdev)
+{
+ struct hv_pcibus_device *hbus = hv_get_drvdata(hdev);
+ struct pci_message message;
+ struct completion comp;
+ int ret;
+
+ /* Ask the host to send along the list of child devices */
+ init_completion(&comp);
+ if (cmpxchg(&hbus->survey_event, NULL, &comp))
+ return -ENOTEMPTY;
+
+ memset(&message, 0, sizeof(message));
+ message.type = PCI_QUERY_BUS_RELATIONS;
+
+ ret = vmbus_sendpacket(hdev->channel, &message, sizeof(message),
+ 0, VM_PKT_DATA_INBAND, 0);
+ if (!ret)
+ ret = wait_for_response(hdev, &comp);
+
+ return ret;
+}
+
+/**
+ * hv_send_resources_allocated() - Report local resource choices
+ * @hdev: VMBus's tracking struct for this root PCI bus
+ *
+ * The host OS is expecting to be sent a request as a message
+ * which contains all the resources that the device will use.
+ * The response contains those same resources, "translated"
+ * which is to say, the values which should be used by the
+ * hardware, when it delivers an interrupt. (MMIO resources are
+ * used in local terms.) This is nice for Windows, and lines up
+ * with the FDO/PDO split, which doesn't exist in Linux. Linux
+ * is deeply expecting to scan an emulated PCI configuration
+ * space. So this message is sent here only to drive the state
+ * machine on the host forward.
+ *
+ * Return: 0 on success, -errno on failure
+ */
+static int hv_send_resources_allocated(struct hv_device *hdev)
+{
+ struct hv_pcibus_device *hbus = hv_get_drvdata(hdev);
+ struct pci_resources_assigned *res_assigned;
+ struct pci_resources_assigned2 *res_assigned2;
+ struct hv_pci_compl comp_pkt;
+ struct hv_pci_dev *hpdev;
+ struct pci_packet *pkt;
+ size_t size_res;
+ u32 wslot;
+ int ret;
+
+ size_res = (pci_protocol_version < PCI_PROTOCOL_VERSION_1_2)
+ ? sizeof(*res_assigned) : sizeof(*res_assigned2);
+
+ pkt = kmalloc(sizeof(*pkt) + size_res, GFP_KERNEL);
+ if (!pkt)
+ return -ENOMEM;
+
+ ret = 0;
+
+ for (wslot = 0; wslot < 256; wslot++) {
+ hpdev = get_pcichild_wslot(hbus, wslot);
+ if (!hpdev)
+ continue;
+
+ memset(pkt, 0, sizeof(*pkt) + size_res);
+ init_completion(&comp_pkt.host_event);
+ pkt->completion_func = hv_pci_generic_compl;
+ pkt->compl_ctxt = &comp_pkt;
+
+ if (pci_protocol_version < PCI_PROTOCOL_VERSION_1_2) {
+ res_assigned =
+ (struct pci_resources_assigned *)&pkt->message;
+ res_assigned->message_type.type =
+ PCI_RESOURCES_ASSIGNED;
+ res_assigned->wslot.slot = hpdev->desc.win_slot.slot;
+ } else {
+ res_assigned2 =
+ (struct pci_resources_assigned2 *)&pkt->message;
+ res_assigned2->message_type.type =
+ PCI_RESOURCES_ASSIGNED2;
+ res_assigned2->wslot.slot = hpdev->desc.win_slot.slot;
+ }
+ put_pcichild(hpdev, hv_pcidev_ref_by_slot);
+
+ ret = vmbus_sendpacket(hdev->channel, &pkt->message,
+ size_res, (unsigned long)pkt,
+ VM_PKT_DATA_INBAND,
+ VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
+ if (!ret)
+ ret = wait_for_response(hdev, &comp_pkt.host_event);
+ if (ret)
+ break;
+
+ if (comp_pkt.completion_status < 0) {
+ ret = -EPROTO;
+ dev_err(&hdev->device,
+ "resource allocated returned 0x%x",
+ comp_pkt.completion_status);
+ break;
+ }
+ }
+
+ kfree(pkt);
+ return ret;
+}
+
+/**
+ * hv_send_resources_released() - Report local resources
+ * released
+ * @hdev: VMBus's tracking struct for this root PCI bus
+ *
+ * Return: 0 on success, -errno on failure
+ */
+static int hv_send_resources_released(struct hv_device *hdev)
+{
+ struct hv_pcibus_device *hbus = hv_get_drvdata(hdev);
+ struct pci_child_message pkt;
+ struct hv_pci_dev *hpdev;
+ u32 wslot;
+ int ret;
+
+ for (wslot = 0; wslot < 256; wslot++) {
+ hpdev = get_pcichild_wslot(hbus, wslot);
+ if (!hpdev)
+ continue;
+
+ memset(&pkt, 0, sizeof(pkt));
+ pkt.message_type.type = PCI_RESOURCES_RELEASED;
+ pkt.wslot.slot = hpdev->desc.win_slot.slot;
+
+ put_pcichild(hpdev, hv_pcidev_ref_by_slot);
+
+ ret = vmbus_sendpacket(hdev->channel, &pkt, sizeof(pkt), 0,
+ VM_PKT_DATA_INBAND, 0);
+ if (ret)
+ return ret;
+ }
+
+ return 0;
+}
+
+static void get_hvpcibus(struct hv_pcibus_device *hbus)
+{
+ atomic_inc(&hbus->remove_lock);
+}
+
+static void put_hvpcibus(struct hv_pcibus_device *hbus)
+{
+ if (atomic_dec_and_test(&hbus->remove_lock))
+ complete(&hbus->remove_event);
+}
+
+/**
+ * hv_pci_probe() - New VMBus channel probe, for a root PCI bus
+ * @hdev: VMBus's tracking struct for this root PCI bus
+ * @dev_id: Identifies the device itself
+ *
+ * Return: 0 on success, -errno on failure
+ */
+static int hv_pci_probe(struct hv_device *hdev,
+ const struct hv_vmbus_device_id *dev_id)
+{
+ struct hv_pcibus_device *hbus;
+ int ret;
+
+ /*
+ * hv_pcibus_device contains the hypercall arguments for retargeting in
+ * hv_irq_unmask(). Those must not cross a page boundary.
+ */
+ BUILD_BUG_ON(sizeof(*hbus) > PAGE_SIZE);
+
+ hbus = (struct hv_pcibus_device *)get_zeroed_page(GFP_KERNEL);
+ if (!hbus)
+ return -ENOMEM;
+ hbus->state = hv_pcibus_init;
+
+ /*
+ * The PCI bus "domain" is what is called "segment" in ACPI and
+ * other specs. Pull it from the instance ID, to get something
+ * unique. Bytes 8 and 9 are what is used in Windows guests, so
+ * do the same thing for consistency. Note that, since this code
+ * only runs in a Hyper-V VM, Hyper-V can (and does) guarantee
+ * that (1) the only domain in use for something that looks like
+ * a physical PCI bus (which is actually emulated by the
+ * hypervisor) is domain 0 and (2) there will be no overlap
+ * between domains derived from these instance IDs in the same
+ * VM.
+ */
+ hbus->sysdata.domain = hdev->dev_instance.b[9] |
+ hdev->dev_instance.b[8] << 8;
+
+ hbus->hdev = hdev;
+ atomic_inc(&hbus->remove_lock);
+ INIT_LIST_HEAD(&hbus->children);
+ INIT_LIST_HEAD(&hbus->dr_list);
+ INIT_LIST_HEAD(&hbus->resources_for_children);
+ spin_lock_init(&hbus->config_lock);
+ spin_lock_init(&hbus->device_list_lock);
+ spin_lock_init(&hbus->retarget_msi_interrupt_lock);
+ init_completion(&hbus->remove_event);
+ hbus->wq = alloc_ordered_workqueue("hv_pci_%x", 0,
+ hbus->sysdata.domain);
+ if (!hbus->wq) {
+ ret = -ENOMEM;
+ goto free_bus;
+ }
+
+ ret = vmbus_open(hdev->channel, pci_ring_size, pci_ring_size, NULL, 0,
+ hv_pci_onchannelcallback, hbus);
+ if (ret)
+ goto destroy_wq;
+
+ hv_set_drvdata(hdev, hbus);
+
+ ret = hv_pci_protocol_negotiation(hdev);
+ if (ret)
+ goto close;
+
+ ret = hv_allocate_config_window(hbus);
+ if (ret)
+ goto close;
+
+ hbus->cfg_addr = ioremap(hbus->mem_config->start,
+ PCI_CONFIG_MMIO_LENGTH);
+ if (!hbus->cfg_addr) {
+ dev_err(&hdev->device,
+ "Unable to map a virtual address for config space\n");
+ ret = -ENOMEM;
+ goto free_config;
+ }
+
+ hbus->sysdata.fwnode = irq_domain_alloc_fwnode(hbus);
+ if (!hbus->sysdata.fwnode) {
+ ret = -ENOMEM;
+ goto unmap;
+ }
+
+ ret = hv_pcie_init_irq_domain(hbus);
+ if (ret)
+ goto free_fwnode;
+
+ ret = hv_pci_query_relations(hdev);
+ if (ret)
+ goto free_irq_domain;
+
+ ret = hv_pci_enter_d0(hdev);
+ if (ret)
+ goto free_irq_domain;
+
+ ret = hv_pci_allocate_bridge_windows(hbus);
+ if (ret)
+ goto free_irq_domain;
+
+ ret = hv_send_resources_allocated(hdev);
+ if (ret)
+ goto free_windows;
+
+ prepopulate_bars(hbus);
+
+ hbus->state = hv_pcibus_probed;
+
+ ret = create_root_hv_pci_bus(hbus);
+ if (ret)
+ goto free_windows;
+
+ return 0;
+
+free_windows:
+ hv_pci_free_bridge_windows(hbus);
+free_irq_domain:
+ irq_domain_remove(hbus->irq_domain);
+free_fwnode:
+ irq_domain_free_fwnode(hbus->sysdata.fwnode);
+unmap:
+ iounmap(hbus->cfg_addr);
+free_config:
+ hv_free_config_window(hbus);
+close:
+ vmbus_close(hdev->channel);
+destroy_wq:
+ destroy_workqueue(hbus->wq);
+free_bus:
+ free_page((unsigned long)hbus);
+ return ret;
+}
+
+static void hv_pci_bus_exit(struct hv_device *hdev)
+{
+ struct hv_pcibus_device *hbus = hv_get_drvdata(hdev);
+ struct {
+ struct pci_packet teardown_packet;
+ u8 buffer[sizeof(struct pci_message)];
+ } pkt;
+ struct pci_bus_relations relations;
+ struct hv_pci_compl comp_pkt;
+ int ret;
+
+ /*
+ * After the host sends the RESCIND_CHANNEL message, it doesn't
+ * access the per-channel ringbuffer any longer.
+ */
+ if (hdev->channel->rescind)
+ return;
+
+ /* Delete any children which might still exist. */
+ memset(&relations, 0, sizeof(relations));
+ hv_pci_devices_present(hbus, &relations);
+
+ ret = hv_send_resources_released(hdev);
+ if (ret)
+ dev_err(&hdev->device,
+ "Couldn't send resources released packet(s)\n");
+
+ memset(&pkt.teardown_packet, 0, sizeof(pkt.teardown_packet));
+ init_completion(&comp_pkt.host_event);
+ pkt.teardown_packet.completion_func = hv_pci_generic_compl;
+ pkt.teardown_packet.compl_ctxt = &comp_pkt;
+ pkt.teardown_packet.message[0].type = PCI_BUS_D0EXIT;
+
+ ret = vmbus_sendpacket(hdev->channel, &pkt.teardown_packet.message,
+ sizeof(struct pci_message),
+ (unsigned long)&pkt.teardown_packet,
+ VM_PKT_DATA_INBAND,
+ VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
+ if (!ret)
+ wait_for_completion_timeout(&comp_pkt.host_event, 10 * HZ);
+}
+
+/**
+ * hv_pci_remove() - Remove routine for this VMBus channel
+ * @hdev: VMBus's tracking struct for this root PCI bus
+ *
+ * Return: 0 on success, -errno on failure
+ */
+static int hv_pci_remove(struct hv_device *hdev)
+{
+ struct hv_pcibus_device *hbus;
+
+ hbus = hv_get_drvdata(hdev);
+ if (hbus->state == hv_pcibus_installed) {
+ /* Remove the bus from PCI's point of view. */
+ pci_lock_rescan_remove();
+ pci_stop_root_bus(hbus->pci_bus);
+ hv_pci_remove_slots(hbus);
+ pci_remove_root_bus(hbus->pci_bus);
+ pci_unlock_rescan_remove();
+ hbus->state = hv_pcibus_removed;
+ }
+
+ hv_pci_bus_exit(hdev);
+
+ vmbus_close(hdev->channel);
+
+ iounmap(hbus->cfg_addr);
+ hv_free_config_window(hbus);
+ pci_free_resource_list(&hbus->resources_for_children);
+ hv_pci_free_bridge_windows(hbus);
+ irq_domain_remove(hbus->irq_domain);
+ irq_domain_free_fwnode(hbus->sysdata.fwnode);
+ put_hvpcibus(hbus);
+ wait_for_completion(&hbus->remove_event);
+ destroy_workqueue(hbus->wq);
+ free_page((unsigned long)hbus);
+ return 0;
+}
+
+static const struct hv_vmbus_device_id hv_pci_id_table[] = {
+ /* PCI Pass-through Class ID */
+ /* 44C4F61D-4444-4400-9D52-802E27EDE19F */
+ { HV_PCIE_GUID, },
+ { },
+};
+
+MODULE_DEVICE_TABLE(vmbus, hv_pci_id_table);
+
+static struct hv_driver hv_pci_drv = {
+ .name = "hv_pci",
+ .id_table = hv_pci_id_table,
+ .probe = hv_pci_probe,
+ .remove = hv_pci_remove,
+};
+
+static void __exit exit_hv_pci_drv(void)
+{
+ vmbus_driver_unregister(&hv_pci_drv);
+}
+
+static int __init init_hv_pci_drv(void)
+{
+ return vmbus_driver_register(&hv_pci_drv);
+}
+
+module_init(init_hv_pci_drv);
+module_exit(exit_hv_pci_drv);
+
+MODULE_DESCRIPTION("Hyper-V PCI");
+MODULE_LICENSE("GPL v2");
diff --git a/src/kernel/linux/v4.14/drivers/pci/host/pci-mvebu.c b/src/kernel/linux/v4.14/drivers/pci/host/pci-mvebu.c
new file mode 100644
index 0000000..12c1c18
--- /dev/null
+++ b/src/kernel/linux/v4.14/drivers/pci/host/pci-mvebu.c
@@ -0,0 +1,1314 @@
+/*
+ * PCIe driver for Marvell Armada 370 and Armada XP SoCs
+ *
+ * Author: Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
+ *
+ * This file is licensed under the terms of the GNU General Public
+ * License version 2. This program is licensed "as is" without any
+ * warranty of any kind, whether express or implied.
+ */
+
+#include <linux/kernel.h>
+#include <linux/pci.h>
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/gpio.h>
+#include <linux/init.h>
+#include <linux/mbus.h>
+#include <linux/msi.h>
+#include <linux/slab.h>
+#include <linux/platform_device.h>
+#include <linux/of_address.h>
+#include <linux/of_irq.h>
+#include <linux/of_gpio.h>
+#include <linux/of_pci.h>
+#include <linux/of_platform.h>
+
+/*
+ * PCIe unit register offsets.
+ */
+#define PCIE_DEV_ID_OFF 0x0000
+#define PCIE_CMD_OFF 0x0004
+#define PCIE_DEV_REV_OFF 0x0008
+#define PCIE_BAR_LO_OFF(n) (0x0010 + ((n) << 3))
+#define PCIE_BAR_HI_OFF(n) (0x0014 + ((n) << 3))
+#define PCIE_CAP_PCIEXP 0x0060
+#define PCIE_HEADER_LOG_4_OFF 0x0128
+#define PCIE_BAR_CTRL_OFF(n) (0x1804 + (((n) - 1) * 4))
+#define PCIE_WIN04_CTRL_OFF(n) (0x1820 + ((n) << 4))
+#define PCIE_WIN04_BASE_OFF(n) (0x1824 + ((n) << 4))
+#define PCIE_WIN04_REMAP_OFF(n) (0x182c + ((n) << 4))
+#define PCIE_WIN5_CTRL_OFF 0x1880
+#define PCIE_WIN5_BASE_OFF 0x1884
+#define PCIE_WIN5_REMAP_OFF 0x188c
+#define PCIE_CONF_ADDR_OFF 0x18f8
+#define PCIE_CONF_ADDR_EN 0x80000000
+#define PCIE_CONF_REG(r) ((((r) & 0xf00) << 16) | ((r) & 0xfc))
+#define PCIE_CONF_BUS(b) (((b) & 0xff) << 16)
+#define PCIE_CONF_DEV(d) (((d) & 0x1f) << 11)
+#define PCIE_CONF_FUNC(f) (((f) & 0x7) << 8)
+#define PCIE_CONF_ADDR(bus, devfn, where) \
+ (PCIE_CONF_BUS(bus) | PCIE_CONF_DEV(PCI_SLOT(devfn)) | \
+ PCIE_CONF_FUNC(PCI_FUNC(devfn)) | PCIE_CONF_REG(where) | \
+ PCIE_CONF_ADDR_EN)
+#define PCIE_CONF_DATA_OFF 0x18fc
+#define PCIE_MASK_OFF 0x1910
+#define PCIE_MASK_ENABLE_INTS 0x0f000000
+#define PCIE_CTRL_OFF 0x1a00
+#define PCIE_CTRL_X1_MODE 0x0001
+#define PCIE_STAT_OFF 0x1a04
+#define PCIE_STAT_BUS 0xff00
+#define PCIE_STAT_DEV 0x1f0000
+#define PCIE_STAT_LINK_DOWN BIT(0)
+#define PCIE_RC_RTSTA 0x1a14
+#define PCIE_DEBUG_CTRL 0x1a60
+#define PCIE_DEBUG_SOFT_RESET BIT(20)
+
+enum {
+ PCISWCAP = PCI_BRIDGE_CONTROL + 2,
+ PCISWCAP_EXP_LIST_ID = PCISWCAP + PCI_CAP_LIST_ID,
+ PCISWCAP_EXP_DEVCAP = PCISWCAP + PCI_EXP_DEVCAP,
+ PCISWCAP_EXP_DEVCTL = PCISWCAP + PCI_EXP_DEVCTL,
+ PCISWCAP_EXP_LNKCAP = PCISWCAP + PCI_EXP_LNKCAP,
+ PCISWCAP_EXP_LNKCTL = PCISWCAP + PCI_EXP_LNKCTL,
+ PCISWCAP_EXP_SLTCAP = PCISWCAP + PCI_EXP_SLTCAP,
+ PCISWCAP_EXP_SLTCTL = PCISWCAP + PCI_EXP_SLTCTL,
+ PCISWCAP_EXP_RTCTL = PCISWCAP + PCI_EXP_RTCTL,
+ PCISWCAP_EXP_RTSTA = PCISWCAP + PCI_EXP_RTSTA,
+ PCISWCAP_EXP_DEVCAP2 = PCISWCAP + PCI_EXP_DEVCAP2,
+ PCISWCAP_EXP_DEVCTL2 = PCISWCAP + PCI_EXP_DEVCTL2,
+ PCISWCAP_EXP_LNKCAP2 = PCISWCAP + PCI_EXP_LNKCAP2,
+ PCISWCAP_EXP_LNKCTL2 = PCISWCAP + PCI_EXP_LNKCTL2,
+ PCISWCAP_EXP_SLTCAP2 = PCISWCAP + PCI_EXP_SLTCAP2,
+ PCISWCAP_EXP_SLTCTL2 = PCISWCAP + PCI_EXP_SLTCTL2,
+};
+
+/* PCI configuration space of a PCI-to-PCI bridge */
+struct mvebu_sw_pci_bridge {
+ u16 vendor;
+ u16 device;
+ u16 command;
+ u16 status;
+ u16 class;
+ u8 interface;
+ u8 revision;
+ u8 bist;
+ u8 header_type;
+ u8 latency_timer;
+ u8 cache_line_size;
+ u32 bar[2];
+ u8 primary_bus;
+ u8 secondary_bus;
+ u8 subordinate_bus;
+ u8 secondary_latency_timer;
+ u8 iobase;
+ u8 iolimit;
+ u16 secondary_status;
+ u16 membase;
+ u16 memlimit;
+ u16 iobaseupper;
+ u16 iolimitupper;
+ u32 romaddr;
+ u8 intline;
+ u8 intpin;
+ u16 bridgectrl;
+
+ /* PCI express capability */
+ u32 pcie_sltcap;
+ u16 pcie_devctl;
+ u16 pcie_rtctl;
+};
+
+struct mvebu_pcie_port;
+
+/* Structure representing all PCIe interfaces */
+struct mvebu_pcie {
+ struct platform_device *pdev;
+ struct mvebu_pcie_port *ports;
+ struct msi_controller *msi;
+ struct resource io;
+ struct resource realio;
+ struct resource mem;
+ struct resource busn;
+ int nports;
+};
+
+struct mvebu_pcie_window {
+ phys_addr_t base;
+ phys_addr_t remap;
+ size_t size;
+};
+
+/* Structure representing one PCIe interface */
+struct mvebu_pcie_port {
+ char *name;
+ void __iomem *base;
+ u32 port;
+ u32 lane;
+ int devfn;
+ unsigned int mem_target;
+ unsigned int mem_attr;
+ unsigned int io_target;
+ unsigned int io_attr;
+ struct clk *clk;
+ struct gpio_desc *reset_gpio;
+ char *reset_name;
+ struct mvebu_sw_pci_bridge bridge;
+ struct device_node *dn;
+ struct mvebu_pcie *pcie;
+ struct mvebu_pcie_window memwin;
+ struct mvebu_pcie_window iowin;
+ u32 saved_pcie_stat;
+};
+
+static inline void mvebu_writel(struct mvebu_pcie_port *port, u32 val, u32 reg)
+{
+ writel(val, port->base + reg);
+}
+
+static inline u32 mvebu_readl(struct mvebu_pcie_port *port, u32 reg)
+{
+ return readl(port->base + reg);
+}
+
+static inline bool mvebu_has_ioport(struct mvebu_pcie_port *port)
+{
+ return port->io_target != -1 && port->io_attr != -1;
+}
+
+static bool mvebu_pcie_link_up(struct mvebu_pcie_port *port)
+{
+ return !(mvebu_readl(port, PCIE_STAT_OFF) & PCIE_STAT_LINK_DOWN);
+}
+
+static void mvebu_pcie_set_local_bus_nr(struct mvebu_pcie_port *port, int nr)
+{
+ u32 stat;
+
+ stat = mvebu_readl(port, PCIE_STAT_OFF);
+ stat &= ~PCIE_STAT_BUS;
+ stat |= nr << 8;
+ mvebu_writel(port, stat, PCIE_STAT_OFF);
+}
+
+static void mvebu_pcie_set_local_dev_nr(struct mvebu_pcie_port *port, int nr)
+{
+ u32 stat;
+
+ stat = mvebu_readl(port, PCIE_STAT_OFF);
+ stat &= ~PCIE_STAT_DEV;
+ stat |= nr << 16;
+ mvebu_writel(port, stat, PCIE_STAT_OFF);
+}
+
+/*
+ * Setup PCIE BARs and Address Decode Wins:
+ * BAR[0,2] -> disabled, BAR[1] -> covers all DRAM banks
+ * WIN[0-3] -> DRAM bank[0-3]
+ */
+static void mvebu_pcie_setup_wins(struct mvebu_pcie_port *port)
+{
+ const struct mbus_dram_target_info *dram;
+ u32 size;
+ int i;
+
+ dram = mv_mbus_dram_info();
+
+ /* First, disable and clear BARs and windows. */
+ for (i = 1; i < 3; i++) {
+ mvebu_writel(port, 0, PCIE_BAR_CTRL_OFF(i));
+ mvebu_writel(port, 0, PCIE_BAR_LO_OFF(i));
+ mvebu_writel(port, 0, PCIE_BAR_HI_OFF(i));
+ }
+
+ for (i = 0; i < 5; i++) {
+ mvebu_writel(port, 0, PCIE_WIN04_CTRL_OFF(i));
+ mvebu_writel(port, 0, PCIE_WIN04_BASE_OFF(i));
+ mvebu_writel(port, 0, PCIE_WIN04_REMAP_OFF(i));
+ }
+
+ mvebu_writel(port, 0, PCIE_WIN5_CTRL_OFF);
+ mvebu_writel(port, 0, PCIE_WIN5_BASE_OFF);
+ mvebu_writel(port, 0, PCIE_WIN5_REMAP_OFF);
+
+ /* Setup windows for DDR banks. Count total DDR size on the fly. */
+ size = 0;
+ for (i = 0; i < dram->num_cs; i++) {
+ const struct mbus_dram_window *cs = dram->cs + i;
+
+ mvebu_writel(port, cs->base & 0xffff0000,
+ PCIE_WIN04_BASE_OFF(i));
+ mvebu_writel(port, 0, PCIE_WIN04_REMAP_OFF(i));
+ mvebu_writel(port,
+ ((cs->size - 1) & 0xffff0000) |
+ (cs->mbus_attr << 8) |
+ (dram->mbus_dram_target_id << 4) | 1,
+ PCIE_WIN04_CTRL_OFF(i));
+
+ size += cs->size;
+ }
+
+ /* Round up 'size' to the nearest power of two. */
+ if ((size & (size - 1)) != 0)
+ size = 1 << fls(size);
+
+ /* Setup BAR[1] to all DRAM banks. */
+ mvebu_writel(port, dram->cs[0].base, PCIE_BAR_LO_OFF(1));
+ mvebu_writel(port, 0, PCIE_BAR_HI_OFF(1));
+ mvebu_writel(port, ((size - 1) & 0xffff0000) | 1,
+ PCIE_BAR_CTRL_OFF(1));
+}
+
+static void mvebu_pcie_setup_hw(struct mvebu_pcie_port *port)
+{
+ u32 cmd, mask;
+
+ /* Point PCIe unit MBUS decode windows to DRAM space. */
+ mvebu_pcie_setup_wins(port);
+
+ /* Master + slave enable. */
+ cmd = mvebu_readl(port, PCIE_CMD_OFF);
+ cmd |= PCI_COMMAND_IO;
+ cmd |= PCI_COMMAND_MEMORY;
+ cmd |= PCI_COMMAND_MASTER;
+ mvebu_writel(port, cmd, PCIE_CMD_OFF);
+
+ /* Enable interrupt lines A-D. */
+ mask = mvebu_readl(port, PCIE_MASK_OFF);
+ mask |= PCIE_MASK_ENABLE_INTS;
+ mvebu_writel(port, mask, PCIE_MASK_OFF);
+}
+
+static int mvebu_pcie_hw_rd_conf(struct mvebu_pcie_port *port,
+ struct pci_bus *bus,
+ u32 devfn, int where, int size, u32 *val)
+{
+ void __iomem *conf_data = port->base + PCIE_CONF_DATA_OFF;
+
+ mvebu_writel(port, PCIE_CONF_ADDR(bus->number, devfn, where),
+ PCIE_CONF_ADDR_OFF);
+
+ switch (size) {
+ case 1:
+ *val = readb_relaxed(conf_data + (where & 3));
+ break;
+ case 2:
+ *val = readw_relaxed(conf_data + (where & 2));
+ break;
+ case 4:
+ *val = readl_relaxed(conf_data);
+ break;
+ }
+
+ return PCIBIOS_SUCCESSFUL;
+}
+
+static int mvebu_pcie_hw_wr_conf(struct mvebu_pcie_port *port,
+ struct pci_bus *bus,
+ u32 devfn, int where, int size, u32 val)
+{
+ void __iomem *conf_data = port->base + PCIE_CONF_DATA_OFF;
+
+ mvebu_writel(port, PCIE_CONF_ADDR(bus->number, devfn, where),
+ PCIE_CONF_ADDR_OFF);
+
+ switch (size) {
+ case 1:
+ writeb(val, conf_data + (where & 3));
+ break;
+ case 2:
+ writew(val, conf_data + (where & 2));
+ break;
+ case 4:
+ writel(val, conf_data);
+ break;
+ default:
+ return PCIBIOS_BAD_REGISTER_NUMBER;
+ }
+
+ return PCIBIOS_SUCCESSFUL;
+}
+
+/*
+ * Remove windows, starting from the largest ones to the smallest
+ * ones.
+ */
+static void mvebu_pcie_del_windows(struct mvebu_pcie_port *port,
+ phys_addr_t base, size_t size)
+{
+ while (size) {
+ size_t sz = 1 << (fls(size) - 1);
+
+ mvebu_mbus_del_window(base, sz);
+ base += sz;
+ size -= sz;
+ }
+}
+
+/*
+ * MBus windows can only have a power of two size, but PCI BARs do not
+ * have this constraint. Therefore, we have to split the PCI BAR into
+ * areas each having a power of two size. We start from the largest
+ * one (i.e highest order bit set in the size).
+ */
+static void mvebu_pcie_add_windows(struct mvebu_pcie_port *port,
+ unsigned int target, unsigned int attribute,
+ phys_addr_t base, size_t size,
+ phys_addr_t remap)
+{
+ size_t size_mapped = 0;
+
+ while (size) {
+ size_t sz = 1 << (fls(size) - 1);
+ int ret;
+
+ ret = mvebu_mbus_add_window_remap_by_id(target, attribute, base,
+ sz, remap);
+ if (ret) {
+ phys_addr_t end = base + sz - 1;
+
+ dev_err(&port->pcie->pdev->dev,
+ "Could not create MBus window at [mem %pa-%pa]: %d\n",
+ &base, &end, ret);
+ mvebu_pcie_del_windows(port, base - size_mapped,
+ size_mapped);
+ return;
+ }
+
+ size -= sz;
+ size_mapped += sz;
+ base += sz;
+ if (remap != MVEBU_MBUS_NO_REMAP)
+ remap += sz;
+ }
+}
+
+static void mvebu_pcie_set_window(struct mvebu_pcie_port *port,
+ unsigned int target, unsigned int attribute,
+ const struct mvebu_pcie_window *desired,
+ struct mvebu_pcie_window *cur)
+{
+ if (desired->base == cur->base && desired->remap == cur->remap &&
+ desired->size == cur->size)
+ return;
+
+ if (cur->size != 0) {
+ mvebu_pcie_del_windows(port, cur->base, cur->size);
+ cur->size = 0;
+ cur->base = 0;
+
+ /*
+ * If something tries to change the window while it is enabled
+ * the change will not be done atomically. That would be
+ * difficult to do in the general case.
+ */
+ }
+
+ if (desired->size == 0)
+ return;
+
+ mvebu_pcie_add_windows(port, target, attribute, desired->base,
+ desired->size, desired->remap);
+ *cur = *desired;
+}
+
+static void mvebu_pcie_handle_iobase_change(struct mvebu_pcie_port *port)
+{
+ struct mvebu_pcie_window desired = {};
+
+ /* Are the new iobase/iolimit values invalid? */
+ if (port->bridge.iolimit < port->bridge.iobase ||
+ port->bridge.iolimitupper < port->bridge.iobaseupper ||
+ !(port->bridge.command & PCI_COMMAND_IO)) {
+ mvebu_pcie_set_window(port, port->io_target, port->io_attr,
+ &desired, &port->iowin);
+ return;
+ }
+
+ if (!mvebu_has_ioport(port)) {
+ dev_WARN(&port->pcie->pdev->dev,
+ "Attempt to set IO when IO is disabled\n");
+ return;
+ }
+
+ /*
+ * We read the PCI-to-PCI bridge emulated registers, and
+ * calculate the base address and size of the address decoding
+ * window to setup, according to the PCI-to-PCI bridge
+ * specifications. iobase is the bus address, port->iowin_base
+ * is the CPU address.
+ */
+ desired.remap = ((port->bridge.iobase & 0xF0) << 8) |
+ (port->bridge.iobaseupper << 16);
+ desired.base = port->pcie->io.start + desired.remap;
+ desired.size = ((0xFFF | ((port->bridge.iolimit & 0xF0) << 8) |
+ (port->bridge.iolimitupper << 16)) -
+ desired.remap) +
+ 1;
+
+ mvebu_pcie_set_window(port, port->io_target, port->io_attr, &desired,
+ &port->iowin);
+}
+
+static void mvebu_pcie_handle_membase_change(struct mvebu_pcie_port *port)
+{
+ struct mvebu_pcie_window desired = {.remap = MVEBU_MBUS_NO_REMAP};
+
+ /* Are the new membase/memlimit values invalid? */
+ if (port->bridge.memlimit < port->bridge.membase ||
+ !(port->bridge.command & PCI_COMMAND_MEMORY)) {
+ mvebu_pcie_set_window(port, port->mem_target, port->mem_attr,
+ &desired, &port->memwin);
+ return;
+ }
+
+ /*
+ * We read the PCI-to-PCI bridge emulated registers, and
+ * calculate the base address and size of the address decoding
+ * window to setup, according to the PCI-to-PCI bridge
+ * specifications.
+ */
+ desired.base = ((port->bridge.membase & 0xFFF0) << 16);
+ desired.size = (((port->bridge.memlimit & 0xFFF0) << 16) | 0xFFFFF) -
+ desired.base + 1;
+
+ mvebu_pcie_set_window(port, port->mem_target, port->mem_attr, &desired,
+ &port->memwin);
+}
+
+/*
+ * Initialize the configuration space of the PCI-to-PCI bridge
+ * associated with the given PCIe interface.
+ */
+static void mvebu_sw_pci_bridge_init(struct mvebu_pcie_port *port)
+{
+ struct mvebu_sw_pci_bridge *bridge = &port->bridge;
+
+ memset(bridge, 0, sizeof(struct mvebu_sw_pci_bridge));
+
+ bridge->class = PCI_CLASS_BRIDGE_PCI;
+ bridge->vendor = PCI_VENDOR_ID_MARVELL;
+ bridge->device = mvebu_readl(port, PCIE_DEV_ID_OFF) >> 16;
+ bridge->revision = mvebu_readl(port, PCIE_DEV_REV_OFF) & 0xff;
+ bridge->header_type = PCI_HEADER_TYPE_BRIDGE;
+ bridge->cache_line_size = 0x10;
+
+ /* We support 32 bits I/O addressing */
+ bridge->iobase = PCI_IO_RANGE_TYPE_32;
+ bridge->iolimit = PCI_IO_RANGE_TYPE_32;
+
+ /* Add capabilities */
+ bridge->status = PCI_STATUS_CAP_LIST;
+}
+
+/*
+ * Read the configuration space of the PCI-to-PCI bridge associated to
+ * the given PCIe interface.
+ */
+static int mvebu_sw_pci_bridge_read(struct mvebu_pcie_port *port,
+ unsigned int where, int size, u32 *value)
+{
+ struct mvebu_sw_pci_bridge *bridge = &port->bridge;
+
+ switch (where & ~3) {
+ case PCI_VENDOR_ID:
+ *value = bridge->device << 16 | bridge->vendor;
+ break;
+
+ case PCI_COMMAND:
+ *value = bridge->command | bridge->status << 16;
+ break;
+
+ case PCI_CLASS_REVISION:
+ *value = bridge->class << 16 | bridge->interface << 8 |
+ bridge->revision;
+ break;
+
+ case PCI_CACHE_LINE_SIZE:
+ *value = bridge->bist << 24 | bridge->header_type << 16 |
+ bridge->latency_timer << 8 | bridge->cache_line_size;
+ break;
+
+ case PCI_BASE_ADDRESS_0 ... PCI_BASE_ADDRESS_1:
+ *value = bridge->bar[((where & ~3) - PCI_BASE_ADDRESS_0) / 4];
+ break;
+
+ case PCI_PRIMARY_BUS:
+ *value = (bridge->secondary_latency_timer << 24 |
+ bridge->subordinate_bus << 16 |
+ bridge->secondary_bus << 8 |
+ bridge->primary_bus);
+ break;
+
+ case PCI_IO_BASE:
+ if (!mvebu_has_ioport(port))
+ *value = bridge->secondary_status << 16;
+ else
+ *value = (bridge->secondary_status << 16 |
+ bridge->iolimit << 8 |
+ bridge->iobase);
+ break;
+
+ case PCI_MEMORY_BASE:
+ *value = (bridge->memlimit << 16 | bridge->membase);
+ break;
+
+ case PCI_PREF_MEMORY_BASE:
+ *value = 0;
+ break;
+
+ case PCI_IO_BASE_UPPER16:
+ *value = (bridge->iolimitupper << 16 | bridge->iobaseupper);
+ break;
+
+ case PCI_CAPABILITY_LIST:
+ *value = PCISWCAP;
+ break;
+
+ case PCI_ROM_ADDRESS1:
+ *value = 0;
+ break;
+
+ case PCI_INTERRUPT_LINE:
+ /* LINE PIN MIN_GNT MAX_LAT */
+ *value = 0;
+ break;
+
+ case PCISWCAP_EXP_LIST_ID:
+ /* Set PCIe v2, root port, slot support */
+ *value = (PCI_EXP_TYPE_ROOT_PORT << 4 | 2 |
+ PCI_EXP_FLAGS_SLOT) << 16 | PCI_CAP_ID_EXP;
+ break;
+
+ case PCISWCAP_EXP_DEVCAP:
+ *value = mvebu_readl(port, PCIE_CAP_PCIEXP + PCI_EXP_DEVCAP);
+ break;
+
+ case PCISWCAP_EXP_DEVCTL:
+ *value = mvebu_readl(port, PCIE_CAP_PCIEXP + PCI_EXP_DEVCTL) &
+ ~(PCI_EXP_DEVCTL_URRE | PCI_EXP_DEVCTL_FERE |
+ PCI_EXP_DEVCTL_NFERE | PCI_EXP_DEVCTL_CERE);
+ *value |= bridge->pcie_devctl;
+ break;
+
+ case PCISWCAP_EXP_LNKCAP:
+ /*
+ * PCIe requires the clock power management capability to be
+ * hard-wired to zero for downstream ports
+ */
+ *value = mvebu_readl(port, PCIE_CAP_PCIEXP + PCI_EXP_LNKCAP) &
+ ~PCI_EXP_LNKCAP_CLKPM;
+ break;
+
+ case PCISWCAP_EXP_LNKCTL:
+ *value = mvebu_readl(port, PCIE_CAP_PCIEXP + PCI_EXP_LNKCTL);
+ break;
+
+ case PCISWCAP_EXP_SLTCAP:
+ *value = bridge->pcie_sltcap;
+ break;
+
+ case PCISWCAP_EXP_SLTCTL:
+ *value = PCI_EXP_SLTSTA_PDS << 16;
+ break;
+
+ case PCISWCAP_EXP_RTCTL:
+ *value = bridge->pcie_rtctl;
+ break;
+
+ case PCISWCAP_EXP_RTSTA:
+ *value = mvebu_readl(port, PCIE_RC_RTSTA);
+ break;
+
+ /* PCIe requires the v2 fields to be hard-wired to zero */
+ case PCISWCAP_EXP_DEVCAP2:
+ case PCISWCAP_EXP_DEVCTL2:
+ case PCISWCAP_EXP_LNKCAP2:
+ case PCISWCAP_EXP_LNKCTL2:
+ case PCISWCAP_EXP_SLTCAP2:
+ case PCISWCAP_EXP_SLTCTL2:
+ default:
+ /*
+ * PCI defines configuration read accesses to reserved or
+ * unimplemented registers to read as zero and complete
+ * normally.
+ */
+ *value = 0;
+ return PCIBIOS_SUCCESSFUL;
+ }
+
+ if (size == 2)
+ *value = (*value >> (8 * (where & 3))) & 0xffff;
+ else if (size == 1)
+ *value = (*value >> (8 * (where & 3))) & 0xff;
+
+ return PCIBIOS_SUCCESSFUL;
+}
+
+/* Write to the PCI-to-PCI bridge configuration space */
+static int mvebu_sw_pci_bridge_write(struct mvebu_pcie_port *port,
+ unsigned int where, int size, u32 value)
+{
+ struct mvebu_sw_pci_bridge *bridge = &port->bridge;
+ u32 mask, reg;
+ int err;
+
+ if (size == 4)
+ mask = 0x0;
+ else if (size == 2)
+ mask = ~(0xffff << ((where & 3) * 8));
+ else if (size == 1)
+ mask = ~(0xff << ((where & 3) * 8));
+ else
+ return PCIBIOS_BAD_REGISTER_NUMBER;
+
+ err = mvebu_sw_pci_bridge_read(port, where & ~3, 4, ®);
+ if (err)
+ return err;
+
+ value = (reg & mask) | value << ((where & 3) * 8);
+
+ switch (where & ~3) {
+ case PCI_COMMAND:
+ {
+ u32 old = bridge->command;
+
+ if (!mvebu_has_ioport(port))
+ value &= ~PCI_COMMAND_IO;
+
+ bridge->command = value & 0xffff;
+ if ((old ^ bridge->command) & PCI_COMMAND_IO)
+ mvebu_pcie_handle_iobase_change(port);
+ if ((old ^ bridge->command) & PCI_COMMAND_MEMORY)
+ mvebu_pcie_handle_membase_change(port);
+ break;
+ }
+
+ case PCI_BASE_ADDRESS_0 ... PCI_BASE_ADDRESS_1:
+ bridge->bar[((where & ~3) - PCI_BASE_ADDRESS_0) / 4] = value;
+ break;
+
+ case PCI_IO_BASE:
+ /*
+ * We also keep bit 1 set, it is a read-only bit that
+ * indicates we support 32 bits addressing for the
+ * I/O
+ */
+ bridge->iobase = (value & 0xff) | PCI_IO_RANGE_TYPE_32;
+ bridge->iolimit = ((value >> 8) & 0xff) | PCI_IO_RANGE_TYPE_32;
+ mvebu_pcie_handle_iobase_change(port);
+ break;
+
+ case PCI_MEMORY_BASE:
+ bridge->membase = value & 0xffff;
+ bridge->memlimit = value >> 16;
+ mvebu_pcie_handle_membase_change(port);
+ break;
+
+ case PCI_IO_BASE_UPPER16:
+ bridge->iobaseupper = value & 0xffff;
+ bridge->iolimitupper = value >> 16;
+ mvebu_pcie_handle_iobase_change(port);
+ break;
+
+ case PCI_PRIMARY_BUS:
+ bridge->primary_bus = value & 0xff;
+ bridge->secondary_bus = (value >> 8) & 0xff;
+ bridge->subordinate_bus = (value >> 16) & 0xff;
+ bridge->secondary_latency_timer = (value >> 24) & 0xff;
+ mvebu_pcie_set_local_bus_nr(port, bridge->secondary_bus);
+ break;
+
+ case PCISWCAP_EXP_DEVCTL:
+ /*
+ * Armada370 data says these bits must always
+ * be zero when in root complex mode.
+ */
+ value &= ~(PCI_EXP_DEVCTL_URRE | PCI_EXP_DEVCTL_FERE |
+ PCI_EXP_DEVCTL_NFERE | PCI_EXP_DEVCTL_CERE);
+
+ /*
+ * If the mask is 0xffff0000, then we only want to write
+ * the device control register, rather than clearing the
+ * RW1C bits in the device status register. Mask out the
+ * status register bits.
+ */
+ if (mask == 0xffff0000)
+ value &= 0xffff;
+
+ mvebu_writel(port, value, PCIE_CAP_PCIEXP + PCI_EXP_DEVCTL);
+ break;
+
+ case PCISWCAP_EXP_LNKCTL:
+ /*
+ * If we don't support CLKREQ, we must ensure that the
+ * CLKREQ enable bit always reads zero. Since we haven't
+ * had this capability, and it's dependent on board wiring,
+ * disable it for the time being.
+ */
+ value &= ~PCI_EXP_LNKCTL_CLKREQ_EN;
+
+ /*
+ * If the mask is 0xffff0000, then we only want to write
+ * the link control register, rather than clearing the
+ * RW1C bits in the link status register. Mask out the
+ * RW1C status register bits.
+ */
+ if (mask == 0xffff0000)
+ value &= ~((PCI_EXP_LNKSTA_LABS |
+ PCI_EXP_LNKSTA_LBMS) << 16);
+
+ mvebu_writel(port, value, PCIE_CAP_PCIEXP + PCI_EXP_LNKCTL);
+ break;
+
+ case PCISWCAP_EXP_RTSTA:
+ mvebu_writel(port, value, PCIE_RC_RTSTA);
+ break;
+
+ default:
+ break;
+ }
+
+ return PCIBIOS_SUCCESSFUL;
+}
+
+static inline struct mvebu_pcie *sys_to_pcie(struct pci_sys_data *sys)
+{
+ return sys->private_data;
+}
+
+static struct mvebu_pcie_port *mvebu_pcie_find_port(struct mvebu_pcie *pcie,
+ struct pci_bus *bus,
+ int devfn)
+{
+ int i;
+
+ for (i = 0; i < pcie->nports; i++) {
+ struct mvebu_pcie_port *port = &pcie->ports[i];
+
+ if (bus->number == 0 && port->devfn == devfn)
+ return port;
+ if (bus->number != 0 &&
+ bus->number >= port->bridge.secondary_bus &&
+ bus->number <= port->bridge.subordinate_bus)
+ return port;
+ }
+
+ return NULL;
+}
+
+/* PCI configuration space write function */
+static int mvebu_pcie_wr_conf(struct pci_bus *bus, u32 devfn,
+ int where, int size, u32 val)
+{
+ struct mvebu_pcie *pcie = sys_to_pcie(bus->sysdata);
+ struct mvebu_pcie_port *port;
+ int ret;
+
+ port = mvebu_pcie_find_port(pcie, bus, devfn);
+ if (!port)
+ return PCIBIOS_DEVICE_NOT_FOUND;
+
+ /* Access the emulated PCI-to-PCI bridge */
+ if (bus->number == 0)
+ return mvebu_sw_pci_bridge_write(port, where, size, val);
+
+ if (!mvebu_pcie_link_up(port))
+ return PCIBIOS_DEVICE_NOT_FOUND;
+
+ /* Access the real PCIe interface */
+ ret = mvebu_pcie_hw_wr_conf(port, bus, devfn,
+ where, size, val);
+
+ return ret;
+}
+
+/* PCI configuration space read function */
+static int mvebu_pcie_rd_conf(struct pci_bus *bus, u32 devfn, int where,
+ int size, u32 *val)
+{
+ struct mvebu_pcie *pcie = sys_to_pcie(bus->sysdata);
+ struct mvebu_pcie_port *port;
+ int ret;
+
+ port = mvebu_pcie_find_port(pcie, bus, devfn);
+ if (!port) {
+ *val = 0xffffffff;
+ return PCIBIOS_DEVICE_NOT_FOUND;
+ }
+
+ /* Access the emulated PCI-to-PCI bridge */
+ if (bus->number == 0)
+ return mvebu_sw_pci_bridge_read(port, where, size, val);
+
+ if (!mvebu_pcie_link_up(port)) {
+ *val = 0xffffffff;
+ return PCIBIOS_DEVICE_NOT_FOUND;
+ }
+
+ /* Access the real PCIe interface */
+ ret = mvebu_pcie_hw_rd_conf(port, bus, devfn,
+ where, size, val);
+
+ return ret;
+}
+
+static struct pci_ops mvebu_pcie_ops = {
+ .read = mvebu_pcie_rd_conf,
+ .write = mvebu_pcie_wr_conf,
+};
+
+static int mvebu_pcie_setup(int nr, struct pci_sys_data *sys)
+{
+ struct mvebu_pcie *pcie = sys_to_pcie(sys);
+ int err, i;
+
+ pcie->mem.name = "PCI MEM";
+ pcie->realio.name = "PCI I/O";
+
+ if (resource_size(&pcie->realio) != 0)
+ pci_add_resource_offset(&sys->resources, &pcie->realio,
+ sys->io_offset);
+
+ pci_add_resource_offset(&sys->resources, &pcie->mem, sys->mem_offset);
+ pci_add_resource(&sys->resources, &pcie->busn);
+
+ err = devm_request_pci_bus_resources(&pcie->pdev->dev, &sys->resources);
+ if (err)
+ return 0;
+
+ for (i = 0; i < pcie->nports; i++) {
+ struct mvebu_pcie_port *port = &pcie->ports[i];
+
+ if (!port->base)
+ continue;
+ mvebu_pcie_setup_hw(port);
+ }
+
+ return 1;
+}
+
+static resource_size_t mvebu_pcie_align_resource(struct pci_dev *dev,
+ const struct resource *res,
+ resource_size_t start,
+ resource_size_t size,
+ resource_size_t align)
+{
+ if (dev->bus->number != 0)
+ return start;
+
+ /*
+ * On the PCI-to-PCI bridge side, the I/O windows must have at
+ * least a 64 KB size and the memory windows must have at
+ * least a 1 MB size. Moreover, MBus windows need to have a
+ * base address aligned on their size, and their size must be
+ * a power of two. This means that if the BAR doesn't have a
+ * power of two size, several MBus windows will actually be
+ * created. We need to ensure that the biggest MBus window
+ * (which will be the first one) is aligned on its size, which
+ * explains the rounddown_pow_of_two() being done here.
+ */
+ if (res->flags & IORESOURCE_IO)
+ return round_up(start, max_t(resource_size_t, SZ_64K,
+ rounddown_pow_of_two(size)));
+ else if (res->flags & IORESOURCE_MEM)
+ return round_up(start, max_t(resource_size_t, SZ_1M,
+ rounddown_pow_of_two(size)));
+ else
+ return start;
+}
+
+static void mvebu_pcie_enable(struct mvebu_pcie *pcie)
+{
+ struct hw_pci hw;
+
+ memset(&hw, 0, sizeof(hw));
+
+#ifdef CONFIG_PCI_MSI
+ hw.msi_ctrl = pcie->msi;
+#endif
+
+ hw.nr_controllers = 1;
+ hw.private_data = (void **)&pcie;
+ hw.setup = mvebu_pcie_setup;
+ hw.map_irq = of_irq_parse_and_map_pci;
+ hw.ops = &mvebu_pcie_ops;
+ hw.align_resource = mvebu_pcie_align_resource;
+
+ pci_common_init_dev(&pcie->pdev->dev, &hw);
+}
+
+/*
+ * Looks up the list of register addresses encoded into the reg =
+ * <...> property for one that matches the given port/lane. Once
+ * found, maps it.
+ */
+static void __iomem *mvebu_pcie_map_registers(struct platform_device *pdev,
+ struct device_node *np,
+ struct mvebu_pcie_port *port)
+{
+ struct resource regs;
+ int ret = 0;
+
+ ret = of_address_to_resource(np, 0, ®s);
+ if (ret)
+ return ERR_PTR(ret);
+
+ return devm_ioremap_resource(&pdev->dev, ®s);
+}
+
+#define DT_FLAGS_TO_TYPE(flags) (((flags) >> 24) & 0x03)
+#define DT_TYPE_IO 0x1
+#define DT_TYPE_MEM32 0x2
+#define DT_CPUADDR_TO_TARGET(cpuaddr) (((cpuaddr) >> 56) & 0xFF)
+#define DT_CPUADDR_TO_ATTR(cpuaddr) (((cpuaddr) >> 48) & 0xFF)
+
+static int mvebu_get_tgt_attr(struct device_node *np, int devfn,
+ unsigned long type,
+ unsigned int *tgt,
+ unsigned int *attr)
+{
+ const int na = 3, ns = 2;
+ const __be32 *range;
+ int rlen, nranges, rangesz, pna, i;
+
+ *tgt = -1;
+ *attr = -1;
+
+ range = of_get_property(np, "ranges", &rlen);
+ if (!range)
+ return -EINVAL;
+
+ pna = of_n_addr_cells(np);
+ rangesz = pna + na + ns;
+ nranges = rlen / sizeof(__be32) / rangesz;
+
+ for (i = 0; i < nranges; i++, range += rangesz) {
+ u32 flags = of_read_number(range, 1);
+ u32 slot = of_read_number(range + 1, 1);
+ u64 cpuaddr = of_read_number(range + na, pna);
+ unsigned long rtype;
+
+ if (DT_FLAGS_TO_TYPE(flags) == DT_TYPE_IO)
+ rtype = IORESOURCE_IO;
+ else if (DT_FLAGS_TO_TYPE(flags) == DT_TYPE_MEM32)
+ rtype = IORESOURCE_MEM;
+ else
+ continue;
+
+ if (slot == PCI_SLOT(devfn) && type == rtype) {
+ *tgt = DT_CPUADDR_TO_TARGET(cpuaddr);
+ *attr = DT_CPUADDR_TO_ATTR(cpuaddr);
+ return 0;
+ }
+ }
+
+ return -ENOENT;
+}
+
+#ifdef CONFIG_PM_SLEEP
+static int mvebu_pcie_suspend(struct device *dev)
+{
+ struct mvebu_pcie *pcie;
+ int i;
+
+ pcie = dev_get_drvdata(dev);
+ for (i = 0; i < pcie->nports; i++) {
+ struct mvebu_pcie_port *port = pcie->ports + i;
+ port->saved_pcie_stat = mvebu_readl(port, PCIE_STAT_OFF);
+ }
+
+ return 0;
+}
+
+static int mvebu_pcie_resume(struct device *dev)
+{
+ struct mvebu_pcie *pcie;
+ int i;
+
+ pcie = dev_get_drvdata(dev);
+ for (i = 0; i < pcie->nports; i++) {
+ struct mvebu_pcie_port *port = pcie->ports + i;
+ mvebu_writel(port, port->saved_pcie_stat, PCIE_STAT_OFF);
+ mvebu_pcie_setup_hw(port);
+ }
+
+ return 0;
+}
+#endif
+
+static void mvebu_pcie_port_clk_put(void *data)
+{
+ struct mvebu_pcie_port *port = data;
+
+ clk_put(port->clk);
+}
+
+static int mvebu_pcie_parse_port(struct mvebu_pcie *pcie,
+ struct mvebu_pcie_port *port, struct device_node *child)
+{
+ struct device *dev = &pcie->pdev->dev;
+ enum of_gpio_flags flags;
+ int reset_gpio, ret;
+
+ port->pcie = pcie;
+
+ if (of_property_read_u32(child, "marvell,pcie-port", &port->port)) {
+ dev_warn(dev, "ignoring %pOF, missing pcie-port property\n",
+ child);
+ goto skip;
+ }
+
+ if (of_property_read_u32(child, "marvell,pcie-lane", &port->lane))
+ port->lane = 0;
+
+ port->name = devm_kasprintf(dev, GFP_KERNEL, "pcie%d.%d", port->port,
+ port->lane);
+ if (!port->name) {
+ ret = -ENOMEM;
+ goto err;
+ }
+
+ port->devfn = of_pci_get_devfn(child);
+ if (port->devfn < 0)
+ goto skip;
+
+ ret = mvebu_get_tgt_attr(dev->of_node, port->devfn, IORESOURCE_MEM,
+ &port->mem_target, &port->mem_attr);
+ if (ret < 0) {
+ dev_err(dev, "%s: cannot get tgt/attr for mem window\n",
+ port->name);
+ goto skip;
+ }
+
+ if (resource_size(&pcie->io) != 0) {
+ mvebu_get_tgt_attr(dev->of_node, port->devfn, IORESOURCE_IO,
+ &port->io_target, &port->io_attr);
+ } else {
+ port->io_target = -1;
+ port->io_attr = -1;
+ }
+
+ reset_gpio = of_get_named_gpio_flags(child, "reset-gpios", 0, &flags);
+ if (reset_gpio == -EPROBE_DEFER) {
+ ret = reset_gpio;
+ goto err;
+ }
+
+ if (gpio_is_valid(reset_gpio)) {
+ unsigned long gpio_flags;
+
+ port->reset_name = devm_kasprintf(dev, GFP_KERNEL, "%s-reset",
+ port->name);
+ if (!port->reset_name) {
+ ret = -ENOMEM;
+ goto err;
+ }
+
+ if (flags & OF_GPIO_ACTIVE_LOW) {
+ dev_info(dev, "%pOF: reset gpio is active low\n",
+ child);
+ gpio_flags = GPIOF_ACTIVE_LOW |
+ GPIOF_OUT_INIT_LOW;
+ } else {
+ gpio_flags = GPIOF_OUT_INIT_HIGH;
+ }
+
+ ret = devm_gpio_request_one(dev, reset_gpio, gpio_flags,
+ port->reset_name);
+ if (ret) {
+ if (ret == -EPROBE_DEFER)
+ goto err;
+ goto skip;
+ }
+
+ port->reset_gpio = gpio_to_desc(reset_gpio);
+ }
+
+ port->clk = of_clk_get_by_name(child, NULL);
+ if (IS_ERR(port->clk)) {
+ dev_err(dev, "%s: cannot get clock\n", port->name);
+ goto skip;
+ }
+
+ ret = devm_add_action(dev, mvebu_pcie_port_clk_put, port);
+ if (ret < 0) {
+ clk_put(port->clk);
+ goto err;
+ }
+
+ return 1;
+
+skip:
+ ret = 0;
+
+ /* In the case of skipping, we need to free these */
+ devm_kfree(dev, port->reset_name);
+ port->reset_name = NULL;
+ devm_kfree(dev, port->name);
+ port->name = NULL;
+
+err:
+ return ret;
+}
+
+/*
+ * Power up a PCIe port. PCIe requires the refclk to be stable for 100µs
+ * prior to releasing PERST. See table 2-4 in section 2.6.2 AC Specifications
+ * of the PCI Express Card Electromechanical Specification, 1.1.
+ */
+static int mvebu_pcie_powerup(struct mvebu_pcie_port *port)
+{
+ int ret;
+
+ ret = clk_prepare_enable(port->clk);
+ if (ret < 0)
+ return ret;
+
+ if (port->reset_gpio) {
+ u32 reset_udelay = PCI_PM_D3COLD_WAIT * 1000;
+
+ of_property_read_u32(port->dn, "reset-delay-us",
+ &reset_udelay);
+
+ udelay(100);
+
+ gpiod_set_value_cansleep(port->reset_gpio, 0);
+ msleep(reset_udelay / 1000);
+ }
+
+ return 0;
+}
+
+/*
+ * Power down a PCIe port. Strictly, PCIe requires us to place the card
+ * in D3hot state before asserting PERST#.
+ */
+static void mvebu_pcie_powerdown(struct mvebu_pcie_port *port)
+{
+ gpiod_set_value_cansleep(port->reset_gpio, 1);
+
+ clk_disable_unprepare(port->clk);
+}
+
+static int mvebu_pcie_probe(struct platform_device *pdev)
+{
+ struct device *dev = &pdev->dev;
+ struct mvebu_pcie *pcie;
+ struct device_node *np = dev->of_node;
+ struct device_node *child;
+ int num, i, ret;
+
+ pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL);
+ if (!pcie)
+ return -ENOMEM;
+
+ pcie->pdev = pdev;
+ platform_set_drvdata(pdev, pcie);
+
+ /* Get the PCIe memory and I/O aperture */
+ mvebu_mbus_get_pcie_mem_aperture(&pcie->mem);
+ if (resource_size(&pcie->mem) == 0) {
+ dev_err(dev, "invalid memory aperture size\n");
+ return -EINVAL;
+ }
+
+ mvebu_mbus_get_pcie_io_aperture(&pcie->io);
+
+ if (resource_size(&pcie->io) != 0) {
+ pcie->realio.flags = pcie->io.flags;
+ pcie->realio.start = PCIBIOS_MIN_IO;
+ pcie->realio.end = min_t(resource_size_t,
+ IO_SPACE_LIMIT,
+ resource_size(&pcie->io) - 1);
+ } else
+ pcie->realio = pcie->io;
+
+ /* Get the bus range */
+ ret = of_pci_parse_bus_range(np, &pcie->busn);
+ if (ret) {
+ dev_err(dev, "failed to parse bus-range property: %d\n", ret);
+ return ret;
+ }
+
+ num = of_get_available_child_count(np);
+
+ pcie->ports = devm_kcalloc(dev, num, sizeof(*pcie->ports), GFP_KERNEL);
+ if (!pcie->ports)
+ return -ENOMEM;
+
+ i = 0;
+ for_each_available_child_of_node(np, child) {
+ struct mvebu_pcie_port *port = &pcie->ports[i];
+
+ ret = mvebu_pcie_parse_port(pcie, port, child);
+ if (ret < 0) {
+ of_node_put(child);
+ return ret;
+ } else if (ret == 0) {
+ continue;
+ }
+
+ port->dn = child;
+ i++;
+ }
+ pcie->nports = i;
+
+ for (i = 0; i < pcie->nports; i++) {
+ struct mvebu_pcie_port *port = &pcie->ports[i];
+
+ child = port->dn;
+ if (!child)
+ continue;
+
+ ret = mvebu_pcie_powerup(port);
+ if (ret < 0)
+ continue;
+
+ port->base = mvebu_pcie_map_registers(pdev, child, port);
+ if (IS_ERR(port->base)) {
+ dev_err(dev, "%s: cannot map registers\n", port->name);
+ port->base = NULL;
+ mvebu_pcie_powerdown(port);
+ continue;
+ }
+
+ mvebu_pcie_set_local_dev_nr(port, 1);
+ mvebu_sw_pci_bridge_init(port);
+ }
+
+ pcie->nports = i;
+
+ for (i = 0; i < (IO_SPACE_LIMIT - SZ_64K); i += SZ_64K)
+ pci_ioremap_io(i, pcie->io.start + i);
+
+ mvebu_pcie_enable(pcie);
+
+ platform_set_drvdata(pdev, pcie);
+
+ return 0;
+}
+
+static const struct of_device_id mvebu_pcie_of_match_table[] = {
+ { .compatible = "marvell,armada-xp-pcie", },
+ { .compatible = "marvell,armada-370-pcie", },
+ { .compatible = "marvell,dove-pcie", },
+ { .compatible = "marvell,kirkwood-pcie", },
+ {},
+};
+
+static const struct dev_pm_ops mvebu_pcie_pm_ops = {
+ SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(mvebu_pcie_suspend, mvebu_pcie_resume)
+};
+
+static struct platform_driver mvebu_pcie_driver = {
+ .driver = {
+ .name = "mvebu-pcie",
+ .of_match_table = mvebu_pcie_of_match_table,
+ /* driver unloading/unbinding currently not supported */
+ .suppress_bind_attrs = true,
+ .pm = &mvebu_pcie_pm_ops,
+ },
+ .probe = mvebu_pcie_probe,
+};
+builtin_platform_driver(mvebu_pcie_driver);
diff --git a/src/kernel/linux/v4.14/drivers/pci/host/pci-rcar-gen2.c b/src/kernel/linux/v4.14/drivers/pci/host/pci-rcar-gen2.c
new file mode 100644
index 0000000..6f87968
--- /dev/null
+++ b/src/kernel/linux/v4.14/drivers/pci/host/pci-rcar-gen2.c
@@ -0,0 +1,448 @@
+/*
+ * pci-rcar-gen2: internal PCI bus support
+ *
+ * Copyright (C) 2013 Renesas Solutions Corp.
+ * Copyright (C) 2013 Cogent Embedded, Inc.
+ *
+ * Author: Valentine Barshak <valentine.barshak@cogentembedded.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/delay.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/of_address.h>
+#include <linux/of_pci.h>
+#include <linux/pci.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/sizes.h>
+#include <linux/slab.h>
+
+/* AHB-PCI Bridge PCI communication registers */
+#define RCAR_AHBPCI_PCICOM_OFFSET 0x800
+
+#define RCAR_PCIAHB_WIN1_CTR_REG (RCAR_AHBPCI_PCICOM_OFFSET + 0x00)
+#define RCAR_PCIAHB_WIN2_CTR_REG (RCAR_AHBPCI_PCICOM_OFFSET + 0x04)
+#define RCAR_PCIAHB_PREFETCH0 0x0
+#define RCAR_PCIAHB_PREFETCH4 0x1
+#define RCAR_PCIAHB_PREFETCH8 0x2
+#define RCAR_PCIAHB_PREFETCH16 0x3
+
+#define RCAR_AHBPCI_WIN1_CTR_REG (RCAR_AHBPCI_PCICOM_OFFSET + 0x10)
+#define RCAR_AHBPCI_WIN2_CTR_REG (RCAR_AHBPCI_PCICOM_OFFSET + 0x14)
+#define RCAR_AHBPCI_WIN_CTR_MEM (3 << 1)
+#define RCAR_AHBPCI_WIN_CTR_CFG (5 << 1)
+#define RCAR_AHBPCI_WIN1_HOST (1 << 30)
+#define RCAR_AHBPCI_WIN1_DEVICE (1 << 31)
+
+#define RCAR_PCI_INT_ENABLE_REG (RCAR_AHBPCI_PCICOM_OFFSET + 0x20)
+#define RCAR_PCI_INT_STATUS_REG (RCAR_AHBPCI_PCICOM_OFFSET + 0x24)
+#define RCAR_PCI_INT_SIGTABORT (1 << 0)
+#define RCAR_PCI_INT_SIGRETABORT (1 << 1)
+#define RCAR_PCI_INT_REMABORT (1 << 2)
+#define RCAR_PCI_INT_PERR (1 << 3)
+#define RCAR_PCI_INT_SIGSERR (1 << 4)
+#define RCAR_PCI_INT_RESERR (1 << 5)
+#define RCAR_PCI_INT_WIN1ERR (1 << 12)
+#define RCAR_PCI_INT_WIN2ERR (1 << 13)
+#define RCAR_PCI_INT_A (1 << 16)
+#define RCAR_PCI_INT_B (1 << 17)
+#define RCAR_PCI_INT_PME (1 << 19)
+#define RCAR_PCI_INT_ALLERRORS (RCAR_PCI_INT_SIGTABORT | \
+ RCAR_PCI_INT_SIGRETABORT | \
+ RCAR_PCI_INT_SIGRETABORT | \
+ RCAR_PCI_INT_REMABORT | \
+ RCAR_PCI_INT_PERR | \
+ RCAR_PCI_INT_SIGSERR | \
+ RCAR_PCI_INT_RESERR | \
+ RCAR_PCI_INT_WIN1ERR | \
+ RCAR_PCI_INT_WIN2ERR)
+
+#define RCAR_AHB_BUS_CTR_REG (RCAR_AHBPCI_PCICOM_OFFSET + 0x30)
+#define RCAR_AHB_BUS_MMODE_HTRANS (1 << 0)
+#define RCAR_AHB_BUS_MMODE_BYTE_BURST (1 << 1)
+#define RCAR_AHB_BUS_MMODE_WR_INCR (1 << 2)
+#define RCAR_AHB_BUS_MMODE_HBUS_REQ (1 << 7)
+#define RCAR_AHB_BUS_SMODE_READYCTR (1 << 17)
+#define RCAR_AHB_BUS_MODE (RCAR_AHB_BUS_MMODE_HTRANS | \
+ RCAR_AHB_BUS_MMODE_BYTE_BURST | \
+ RCAR_AHB_BUS_MMODE_WR_INCR | \
+ RCAR_AHB_BUS_MMODE_HBUS_REQ | \
+ RCAR_AHB_BUS_SMODE_READYCTR)
+
+#define RCAR_USBCTR_REG (RCAR_AHBPCI_PCICOM_OFFSET + 0x34)
+#define RCAR_USBCTR_USBH_RST (1 << 0)
+#define RCAR_USBCTR_PCICLK_MASK (1 << 1)
+#define RCAR_USBCTR_PLL_RST (1 << 2)
+#define RCAR_USBCTR_DIRPD (1 << 8)
+#define RCAR_USBCTR_PCIAHB_WIN2_EN (1 << 9)
+#define RCAR_USBCTR_PCIAHB_WIN1_256M (0 << 10)
+#define RCAR_USBCTR_PCIAHB_WIN1_512M (1 << 10)
+#define RCAR_USBCTR_PCIAHB_WIN1_1G (2 << 10)
+#define RCAR_USBCTR_PCIAHB_WIN1_2G (3 << 10)
+#define RCAR_USBCTR_PCIAHB_WIN1_MASK (3 << 10)
+
+#define RCAR_PCI_ARBITER_CTR_REG (RCAR_AHBPCI_PCICOM_OFFSET + 0x40)
+#define RCAR_PCI_ARBITER_PCIREQ0 (1 << 0)
+#define RCAR_PCI_ARBITER_PCIREQ1 (1 << 1)
+#define RCAR_PCI_ARBITER_PCIBP_MODE (1 << 12)
+
+#define RCAR_PCI_UNIT_REV_REG (RCAR_AHBPCI_PCICOM_OFFSET + 0x48)
+
+struct rcar_pci_priv {
+ struct device *dev;
+ void __iomem *reg;
+ struct resource mem_res;
+ struct resource *cfg_res;
+ unsigned busnr;
+ int irq;
+ unsigned long window_size;
+ unsigned long window_addr;
+ unsigned long window_pci;
+};
+
+/* PCI configuration space operations */
+static void __iomem *rcar_pci_cfg_base(struct pci_bus *bus, unsigned int devfn,
+ int where)
+{
+ struct pci_sys_data *sys = bus->sysdata;
+ struct rcar_pci_priv *priv = sys->private_data;
+ int slot, val;
+
+ if (sys->busnr != bus->number || PCI_FUNC(devfn))
+ return NULL;
+
+ /* Only one EHCI/OHCI device built-in */
+ slot = PCI_SLOT(devfn);
+ if (slot > 2)
+ return NULL;
+
+ /* bridge logic only has registers to 0x40 */
+ if (slot == 0x0 && where >= 0x40)
+ return NULL;
+
+ val = slot ? RCAR_AHBPCI_WIN1_DEVICE | RCAR_AHBPCI_WIN_CTR_CFG :
+ RCAR_AHBPCI_WIN1_HOST | RCAR_AHBPCI_WIN_CTR_CFG;
+
+ iowrite32(val, priv->reg + RCAR_AHBPCI_WIN1_CTR_REG);
+ return priv->reg + (slot >> 1) * 0x100 + where;
+}
+
+/* PCI interrupt mapping */
+static int rcar_pci_map_irq(const struct pci_dev *dev, u8 slot, u8 pin)
+{
+ struct pci_sys_data *sys = dev->bus->sysdata;
+ struct rcar_pci_priv *priv = sys->private_data;
+ int irq;
+
+ irq = of_irq_parse_and_map_pci(dev, slot, pin);
+ if (!irq)
+ irq = priv->irq;
+
+ return irq;
+}
+
+#ifdef CONFIG_PCI_DEBUG
+/* if debug enabled, then attach an error handler irq to the bridge */
+
+static irqreturn_t rcar_pci_err_irq(int irq, void *pw)
+{
+ struct rcar_pci_priv *priv = pw;
+ struct device *dev = priv->dev;
+ u32 status = ioread32(priv->reg + RCAR_PCI_INT_STATUS_REG);
+
+ if (status & RCAR_PCI_INT_ALLERRORS) {
+ dev_err(dev, "error irq: status %08x\n", status);
+
+ /* clear the error(s) */
+ iowrite32(status & RCAR_PCI_INT_ALLERRORS,
+ priv->reg + RCAR_PCI_INT_STATUS_REG);
+ return IRQ_HANDLED;
+ }
+
+ return IRQ_NONE;
+}
+
+static void rcar_pci_setup_errirq(struct rcar_pci_priv *priv)
+{
+ struct device *dev = priv->dev;
+ int ret;
+ u32 val;
+
+ ret = devm_request_irq(dev, priv->irq, rcar_pci_err_irq,
+ IRQF_SHARED, "error irq", priv);
+ if (ret) {
+ dev_err(dev, "cannot claim IRQ for error handling\n");
+ return;
+ }
+
+ val = ioread32(priv->reg + RCAR_PCI_INT_ENABLE_REG);
+ val |= RCAR_PCI_INT_ALLERRORS;
+ iowrite32(val, priv->reg + RCAR_PCI_INT_ENABLE_REG);
+}
+#else
+static inline void rcar_pci_setup_errirq(struct rcar_pci_priv *priv) { }
+#endif
+
+/* PCI host controller setup */
+static int rcar_pci_setup(int nr, struct pci_sys_data *sys)
+{
+ struct rcar_pci_priv *priv = sys->private_data;
+ struct device *dev = priv->dev;
+ void __iomem *reg = priv->reg;
+ u32 val;
+ int ret;
+
+ pm_runtime_enable(dev);
+ pm_runtime_get_sync(dev);
+
+ val = ioread32(reg + RCAR_PCI_UNIT_REV_REG);
+ dev_info(dev, "PCI: bus%u revision %x\n", sys->busnr, val);
+
+ /* Disable Direct Power Down State and assert reset */
+ val = ioread32(reg + RCAR_USBCTR_REG) & ~RCAR_USBCTR_DIRPD;
+ val |= RCAR_USBCTR_USBH_RST | RCAR_USBCTR_PLL_RST;
+ iowrite32(val, reg + RCAR_USBCTR_REG);
+ udelay(4);
+
+ /* De-assert reset and reset PCIAHB window1 size */
+ val &= ~(RCAR_USBCTR_PCIAHB_WIN1_MASK | RCAR_USBCTR_PCICLK_MASK |
+ RCAR_USBCTR_USBH_RST | RCAR_USBCTR_PLL_RST);
+
+ /* Setup PCIAHB window1 size */
+ switch (priv->window_size) {
+ case SZ_2G:
+ val |= RCAR_USBCTR_PCIAHB_WIN1_2G;
+ break;
+ case SZ_1G:
+ val |= RCAR_USBCTR_PCIAHB_WIN1_1G;
+ break;
+ case SZ_512M:
+ val |= RCAR_USBCTR_PCIAHB_WIN1_512M;
+ break;
+ default:
+ pr_warn("unknown window size %ld - defaulting to 256M\n",
+ priv->window_size);
+ priv->window_size = SZ_256M;
+ /* fall-through */
+ case SZ_256M:
+ val |= RCAR_USBCTR_PCIAHB_WIN1_256M;
+ break;
+ }
+ iowrite32(val, reg + RCAR_USBCTR_REG);
+
+ /* Configure AHB master and slave modes */
+ iowrite32(RCAR_AHB_BUS_MODE, reg + RCAR_AHB_BUS_CTR_REG);
+
+ /* Configure PCI arbiter */
+ val = ioread32(reg + RCAR_PCI_ARBITER_CTR_REG);
+ val |= RCAR_PCI_ARBITER_PCIREQ0 | RCAR_PCI_ARBITER_PCIREQ1 |
+ RCAR_PCI_ARBITER_PCIBP_MODE;
+ iowrite32(val, reg + RCAR_PCI_ARBITER_CTR_REG);
+
+ /* PCI-AHB mapping */
+ iowrite32(priv->window_addr | RCAR_PCIAHB_PREFETCH16,
+ reg + RCAR_PCIAHB_WIN1_CTR_REG);
+
+ /* AHB-PCI mapping: OHCI/EHCI registers */
+ val = priv->mem_res.start | RCAR_AHBPCI_WIN_CTR_MEM;
+ iowrite32(val, reg + RCAR_AHBPCI_WIN2_CTR_REG);
+
+ /* Enable AHB-PCI bridge PCI configuration access */
+ iowrite32(RCAR_AHBPCI_WIN1_HOST | RCAR_AHBPCI_WIN_CTR_CFG,
+ reg + RCAR_AHBPCI_WIN1_CTR_REG);
+ /* Set PCI-AHB Window1 address */
+ iowrite32(priv->window_pci | PCI_BASE_ADDRESS_MEM_PREFETCH,
+ reg + PCI_BASE_ADDRESS_1);
+ /* Set AHB-PCI bridge PCI communication area address */
+ val = priv->cfg_res->start + RCAR_AHBPCI_PCICOM_OFFSET;
+ iowrite32(val, reg + PCI_BASE_ADDRESS_0);
+
+ val = ioread32(reg + PCI_COMMAND);
+ val |= PCI_COMMAND_SERR | PCI_COMMAND_PARITY |
+ PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER;
+ iowrite32(val, reg + PCI_COMMAND);
+
+ /* Enable PCI interrupts */
+ iowrite32(RCAR_PCI_INT_A | RCAR_PCI_INT_B | RCAR_PCI_INT_PME,
+ reg + RCAR_PCI_INT_ENABLE_REG);
+
+ if (priv->irq > 0)
+ rcar_pci_setup_errirq(priv);
+
+ /* Add PCI resources */
+ pci_add_resource(&sys->resources, &priv->mem_res);
+ ret = devm_request_pci_bus_resources(dev, &sys->resources);
+ if (ret < 0)
+ return ret;
+
+ /* Setup bus number based on platform device id / of bus-range */
+ sys->busnr = priv->busnr;
+ return 1;
+}
+
+static struct pci_ops rcar_pci_ops = {
+ .map_bus = rcar_pci_cfg_base,
+ .read = pci_generic_config_read,
+ .write = pci_generic_config_write,
+};
+
+static int pci_dma_range_parser_init(struct of_pci_range_parser *parser,
+ struct device_node *node)
+{
+ const int na = 3, ns = 2;
+ int rlen;
+
+ parser->node = node;
+ parser->pna = of_n_addr_cells(node);
+ parser->np = parser->pna + na + ns;
+
+ parser->range = of_get_property(node, "dma-ranges", &rlen);
+ if (!parser->range)
+ return -ENOENT;
+
+ parser->end = parser->range + rlen / sizeof(__be32);
+ return 0;
+}
+
+static int rcar_pci_parse_map_dma_ranges(struct rcar_pci_priv *pci,
+ struct device_node *np)
+{
+ struct device *dev = pci->dev;
+ struct of_pci_range range;
+ struct of_pci_range_parser parser;
+ int index = 0;
+
+ /* Failure to parse is ok as we fall back to defaults */
+ if (pci_dma_range_parser_init(&parser, np))
+ return 0;
+
+ /* Get the dma-ranges from DT */
+ for_each_of_pci_range(&parser, &range) {
+ /* Hardware only allows one inbound 32-bit range */
+ if (index)
+ return -EINVAL;
+
+ pci->window_addr = (unsigned long)range.cpu_addr;
+ pci->window_pci = (unsigned long)range.pci_addr;
+ pci->window_size = (unsigned long)range.size;
+
+ /* Catch HW limitations */
+ if (!(range.flags & IORESOURCE_PREFETCH)) {
+ dev_err(dev, "window must be prefetchable\n");
+ return -EINVAL;
+ }
+ if (pci->window_addr) {
+ u32 lowaddr = 1 << (ffs(pci->window_addr) - 1);
+
+ if (lowaddr < pci->window_size) {
+ dev_err(dev, "invalid window size/addr\n");
+ return -EINVAL;
+ }
+ }
+ index++;
+ }
+
+ return 0;
+}
+
+static int rcar_pci_probe(struct platform_device *pdev)
+{
+ struct device *dev = &pdev->dev;
+ struct resource *cfg_res, *mem_res;
+ struct rcar_pci_priv *priv;
+ void __iomem *reg;
+ struct hw_pci hw;
+ void *hw_private[1];
+
+ cfg_res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ reg = devm_ioremap_resource(dev, cfg_res);
+ if (IS_ERR(reg))
+ return PTR_ERR(reg);
+
+ mem_res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
+ if (!mem_res || !mem_res->start)
+ return -ENODEV;
+
+ if (mem_res->start & 0xFFFF)
+ return -EINVAL;
+
+ priv = devm_kzalloc(dev, sizeof(struct rcar_pci_priv), GFP_KERNEL);
+ if (!priv)
+ return -ENOMEM;
+
+ priv->mem_res = *mem_res;
+ priv->cfg_res = cfg_res;
+
+ priv->irq = platform_get_irq(pdev, 0);
+ priv->reg = reg;
+ priv->dev = dev;
+
+ if (priv->irq < 0) {
+ dev_err(dev, "no valid irq found\n");
+ return priv->irq;
+ }
+
+ /* default window addr and size if not specified in DT */
+ priv->window_addr = 0x40000000;
+ priv->window_pci = 0x40000000;
+ priv->window_size = SZ_1G;
+
+ if (dev->of_node) {
+ struct resource busnr;
+ int ret;
+
+ ret = of_pci_parse_bus_range(dev->of_node, &busnr);
+ if (ret < 0) {
+ dev_err(dev, "failed to parse bus-range\n");
+ return ret;
+ }
+
+ priv->busnr = busnr.start;
+ if (busnr.end != busnr.start)
+ dev_warn(dev, "only one bus number supported\n");
+
+ ret = rcar_pci_parse_map_dma_ranges(priv, dev->of_node);
+ if (ret < 0) {
+ dev_err(dev, "failed to parse dma-range\n");
+ return ret;
+ }
+ } else {
+ priv->busnr = pdev->id;
+ }
+
+ hw_private[0] = priv;
+ memset(&hw, 0, sizeof(hw));
+ hw.nr_controllers = ARRAY_SIZE(hw_private);
+ hw.io_optional = 1;
+ hw.private_data = hw_private;
+ hw.map_irq = rcar_pci_map_irq;
+ hw.ops = &rcar_pci_ops;
+ hw.setup = rcar_pci_setup;
+ pci_common_init_dev(dev, &hw);
+ return 0;
+}
+
+static const struct of_device_id rcar_pci_of_match[] = {
+ { .compatible = "renesas,pci-r8a7790", },
+ { .compatible = "renesas,pci-r8a7791", },
+ { .compatible = "renesas,pci-r8a7794", },
+ { .compatible = "renesas,pci-rcar-gen2", },
+ { },
+};
+
+static struct platform_driver rcar_pci_driver = {
+ .driver = {
+ .name = "pci-rcar-gen2",
+ .suppress_bind_attrs = true,
+ .of_match_table = rcar_pci_of_match,
+ },
+ .probe = rcar_pci_probe,
+};
+builtin_platform_driver(rcar_pci_driver);
diff --git a/src/kernel/linux/v4.14/drivers/pci/host/pci-tegra.c b/src/kernel/linux/v4.14/drivers/pci/host/pci-tegra.c
new file mode 100644
index 0000000..5bf874f
--- /dev/null
+++ b/src/kernel/linux/v4.14/drivers/pci/host/pci-tegra.c
@@ -0,0 +1,2337 @@
+/*
+ * PCIe host controller driver for Tegra SoCs
+ *
+ * Copyright (c) 2010, CompuLab, Ltd.
+ * Author: Mike Rapoport <mike@compulab.co.il>
+ *
+ * Based on NVIDIA PCIe driver
+ * Copyright (c) 2008-2009, NVIDIA Corporation.
+ *
+ * Bits taken from arch/arm/mach-dove/pcie.c
+ *
+ * Author: Thierry Reding <treding@nvidia.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+#include <linux/clk.h>
+#include <linux/debugfs.h>
+#include <linux/delay.h>
+#include <linux/export.h>
+#include <linux/interrupt.h>
+#include <linux/irq.h>
+#include <linux/irqdomain.h>
+#include <linux/kernel.h>
+#include <linux/init.h>
+#include <linux/msi.h>
+#include <linux/of_address.h>
+#include <linux/of_pci.h>
+#include <linux/of_platform.h>
+#include <linux/pci.h>
+#include <linux/phy/phy.h>
+#include <linux/platform_device.h>
+#include <linux/reset.h>
+#include <linux/sizes.h>
+#include <linux/slab.h>
+#include <linux/vmalloc.h>
+#include <linux/regulator/consumer.h>
+
+#include <soc/tegra/cpuidle.h>
+#include <soc/tegra/pmc.h>
+
+#define INT_PCI_MSI_NR (8 * 32)
+
+/* register definitions */
+
+#define AFI_AXI_BAR0_SZ 0x00
+#define AFI_AXI_BAR1_SZ 0x04
+#define AFI_AXI_BAR2_SZ 0x08
+#define AFI_AXI_BAR3_SZ 0x0c
+#define AFI_AXI_BAR4_SZ 0x10
+#define AFI_AXI_BAR5_SZ 0x14
+
+#define AFI_AXI_BAR0_START 0x18
+#define AFI_AXI_BAR1_START 0x1c
+#define AFI_AXI_BAR2_START 0x20
+#define AFI_AXI_BAR3_START 0x24
+#define AFI_AXI_BAR4_START 0x28
+#define AFI_AXI_BAR5_START 0x2c
+
+#define AFI_FPCI_BAR0 0x30
+#define AFI_FPCI_BAR1 0x34
+#define AFI_FPCI_BAR2 0x38
+#define AFI_FPCI_BAR3 0x3c
+#define AFI_FPCI_BAR4 0x40
+#define AFI_FPCI_BAR5 0x44
+
+#define AFI_CACHE_BAR0_SZ 0x48
+#define AFI_CACHE_BAR0_ST 0x4c
+#define AFI_CACHE_BAR1_SZ 0x50
+#define AFI_CACHE_BAR1_ST 0x54
+
+#define AFI_MSI_BAR_SZ 0x60
+#define AFI_MSI_FPCI_BAR_ST 0x64
+#define AFI_MSI_AXI_BAR_ST 0x68
+
+#define AFI_MSI_VEC0 0x6c
+#define AFI_MSI_VEC1 0x70
+#define AFI_MSI_VEC2 0x74
+#define AFI_MSI_VEC3 0x78
+#define AFI_MSI_VEC4 0x7c
+#define AFI_MSI_VEC5 0x80
+#define AFI_MSI_VEC6 0x84
+#define AFI_MSI_VEC7 0x88
+
+#define AFI_MSI_EN_VEC0 0x8c
+#define AFI_MSI_EN_VEC1 0x90
+#define AFI_MSI_EN_VEC2 0x94
+#define AFI_MSI_EN_VEC3 0x98
+#define AFI_MSI_EN_VEC4 0x9c
+#define AFI_MSI_EN_VEC5 0xa0
+#define AFI_MSI_EN_VEC6 0xa4
+#define AFI_MSI_EN_VEC7 0xa8
+
+#define AFI_CONFIGURATION 0xac
+#define AFI_CONFIGURATION_EN_FPCI (1 << 0)
+
+#define AFI_FPCI_ERROR_MASKS 0xb0
+
+#define AFI_INTR_MASK 0xb4
+#define AFI_INTR_MASK_INT_MASK (1 << 0)
+#define AFI_INTR_MASK_MSI_MASK (1 << 8)
+
+#define AFI_INTR_CODE 0xb8
+#define AFI_INTR_CODE_MASK 0xf
+#define AFI_INTR_INI_SLAVE_ERROR 1
+#define AFI_INTR_INI_DECODE_ERROR 2
+#define AFI_INTR_TARGET_ABORT 3
+#define AFI_INTR_MASTER_ABORT 4
+#define AFI_INTR_INVALID_WRITE 5
+#define AFI_INTR_LEGACY 6
+#define AFI_INTR_FPCI_DECODE_ERROR 7
+#define AFI_INTR_AXI_DECODE_ERROR 8
+#define AFI_INTR_FPCI_TIMEOUT 9
+#define AFI_INTR_PE_PRSNT_SENSE 10
+#define AFI_INTR_PE_CLKREQ_SENSE 11
+#define AFI_INTR_CLKCLAMP_SENSE 12
+#define AFI_INTR_RDY4PD_SENSE 13
+#define AFI_INTR_P2P_ERROR 14
+
+#define AFI_INTR_SIGNATURE 0xbc
+#define AFI_UPPER_FPCI_ADDRESS 0xc0
+#define AFI_SM_INTR_ENABLE 0xc4
+#define AFI_SM_INTR_INTA_ASSERT (1 << 0)
+#define AFI_SM_INTR_INTB_ASSERT (1 << 1)
+#define AFI_SM_INTR_INTC_ASSERT (1 << 2)
+#define AFI_SM_INTR_INTD_ASSERT (1 << 3)
+#define AFI_SM_INTR_INTA_DEASSERT (1 << 4)
+#define AFI_SM_INTR_INTB_DEASSERT (1 << 5)
+#define AFI_SM_INTR_INTC_DEASSERT (1 << 6)
+#define AFI_SM_INTR_INTD_DEASSERT (1 << 7)
+
+#define AFI_AFI_INTR_ENABLE 0xc8
+#define AFI_INTR_EN_INI_SLVERR (1 << 0)
+#define AFI_INTR_EN_INI_DECERR (1 << 1)
+#define AFI_INTR_EN_TGT_SLVERR (1 << 2)
+#define AFI_INTR_EN_TGT_DECERR (1 << 3)
+#define AFI_INTR_EN_TGT_WRERR (1 << 4)
+#define AFI_INTR_EN_DFPCI_DECERR (1 << 5)
+#define AFI_INTR_EN_AXI_DECERR (1 << 6)
+#define AFI_INTR_EN_FPCI_TIMEOUT (1 << 7)
+#define AFI_INTR_EN_PRSNT_SENSE (1 << 8)
+
+#define AFI_PCIE_CONFIG 0x0f8
+#define AFI_PCIE_CONFIG_PCIE_DISABLE(x) (1 << ((x) + 1))
+#define AFI_PCIE_CONFIG_PCIE_DISABLE_ALL 0xe
+#define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_MASK (0xf << 20)
+#define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_SINGLE (0x0 << 20)
+#define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_420 (0x0 << 20)
+#define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_X2_X1 (0x0 << 20)
+#define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_DUAL (0x1 << 20)
+#define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_222 (0x1 << 20)
+#define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_X4_X1 (0x1 << 20)
+#define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_411 (0x2 << 20)
+
+#define AFI_FUSE 0x104
+#define AFI_FUSE_PCIE_T0_GEN2_DIS (1 << 2)
+
+#define AFI_PEX0_CTRL 0x110
+#define AFI_PEX1_CTRL 0x118
+#define AFI_PEX2_CTRL 0x128
+#define AFI_PEX_CTRL_RST (1 << 0)
+#define AFI_PEX_CTRL_CLKREQ_EN (1 << 1)
+#define AFI_PEX_CTRL_REFCLK_EN (1 << 3)
+#define AFI_PEX_CTRL_OVERRIDE_EN (1 << 4)
+
+#define AFI_PLLE_CONTROL 0x160
+#define AFI_PLLE_CONTROL_BYPASS_PADS2PLLE_CONTROL (1 << 9)
+#define AFI_PLLE_CONTROL_PADS2PLLE_CONTROL_EN (1 << 1)
+
+#define AFI_PEXBIAS_CTRL_0 0x168
+
+#define RP_VEND_XP 0x00000f00
+#define RP_VEND_XP_DL_UP (1 << 30)
+
+#define RP_VEND_CTL2 0x00000fa8
+#define RP_VEND_CTL2_PCA_ENABLE (1 << 7)
+
+#define RP_PRIV_MISC 0x00000fe0
+#define RP_PRIV_MISC_PRSNT_MAP_EP_PRSNT (0xe << 0)
+#define RP_PRIV_MISC_PRSNT_MAP_EP_ABSNT (0xf << 0)
+
+#define RP_LINK_CONTROL_STATUS 0x00000090
+#define RP_LINK_CONTROL_STATUS_DL_LINK_ACTIVE 0x20000000
+#define RP_LINK_CONTROL_STATUS_LINKSTAT_MASK 0x3fff0000
+
+#define PADS_CTL_SEL 0x0000009c
+
+#define PADS_CTL 0x000000a0
+#define PADS_CTL_IDDQ_1L (1 << 0)
+#define PADS_CTL_TX_DATA_EN_1L (1 << 6)
+#define PADS_CTL_RX_DATA_EN_1L (1 << 10)
+
+#define PADS_PLL_CTL_TEGRA20 0x000000b8
+#define PADS_PLL_CTL_TEGRA30 0x000000b4
+#define PADS_PLL_CTL_RST_B4SM (1 << 1)
+#define PADS_PLL_CTL_LOCKDET (1 << 8)
+#define PADS_PLL_CTL_REFCLK_MASK (0x3 << 16)
+#define PADS_PLL_CTL_REFCLK_INTERNAL_CML (0 << 16)
+#define PADS_PLL_CTL_REFCLK_INTERNAL_CMOS (1 << 16)
+#define PADS_PLL_CTL_REFCLK_EXTERNAL (2 << 16)
+#define PADS_PLL_CTL_TXCLKREF_MASK (0x1 << 20)
+#define PADS_PLL_CTL_TXCLKREF_DIV10 (0 << 20)
+#define PADS_PLL_CTL_TXCLKREF_DIV5 (1 << 20)
+#define PADS_PLL_CTL_TXCLKREF_BUF_EN (1 << 22)
+
+#define PADS_REFCLK_CFG0 0x000000c8
+#define PADS_REFCLK_CFG1 0x000000cc
+#define PADS_REFCLK_BIAS 0x000000d0
+
+/*
+ * Fields in PADS_REFCLK_CFG*. Those registers form an array of 16-bit
+ * entries, one entry per PCIe port. These field definitions and desired
+ * values aren't in the TRM, but do come from NVIDIA.
+ */
+#define PADS_REFCLK_CFG_TERM_SHIFT 2 /* 6:2 */
+#define PADS_REFCLK_CFG_E_TERM_SHIFT 7
+#define PADS_REFCLK_CFG_PREDI_SHIFT 8 /* 11:8 */
+#define PADS_REFCLK_CFG_DRVI_SHIFT 12 /* 15:12 */
+
+struct tegra_msi {
+ struct msi_controller chip;
+ DECLARE_BITMAP(used, INT_PCI_MSI_NR);
+ struct irq_domain *domain;
+ unsigned long pages;
+ struct mutex lock;
+ u64 phys;
+ int irq;
+};
+
+/* used to differentiate between Tegra SoC generations */
+struct tegra_pcie_soc {
+ unsigned int num_ports;
+ unsigned int msi_base_shift;
+ u32 pads_pll_ctl;
+ u32 tx_ref_sel;
+ u32 pads_refclk_cfg0;
+ u32 pads_refclk_cfg1;
+ bool has_pex_clkreq_en;
+ bool has_pex_bias_ctrl;
+ bool has_intr_prsnt_sense;
+ bool has_cml_clk;
+ bool has_gen2;
+ bool force_pca_enable;
+};
+
+static inline struct tegra_msi *to_tegra_msi(struct msi_controller *chip)
+{
+ return container_of(chip, struct tegra_msi, chip);
+}
+
+struct tegra_pcie {
+ struct device *dev;
+
+ void __iomem *pads;
+ void __iomem *afi;
+ int irq;
+
+ struct list_head buses;
+ struct resource *cs;
+
+ struct resource io;
+ struct resource pio;
+ struct resource mem;
+ struct resource prefetch;
+ struct resource busn;
+
+ struct {
+ resource_size_t mem;
+ resource_size_t io;
+ } offset;
+
+ struct clk *pex_clk;
+ struct clk *afi_clk;
+ struct clk *pll_e;
+ struct clk *cml_clk;
+
+ struct reset_control *pex_rst;
+ struct reset_control *afi_rst;
+ struct reset_control *pcie_xrst;
+
+ bool legacy_phy;
+ struct phy *phy;
+
+ struct tegra_msi msi;
+
+ struct list_head ports;
+ u32 xbar_config;
+
+ struct regulator_bulk_data *supplies;
+ unsigned int num_supplies;
+
+ const struct tegra_pcie_soc *soc;
+ struct dentry *debugfs;
+};
+
+struct tegra_pcie_port {
+ struct tegra_pcie *pcie;
+ struct device_node *np;
+ struct list_head list;
+ struct resource regs;
+ void __iomem *base;
+ unsigned int index;
+ unsigned int lanes;
+
+ struct phy **phys;
+};
+
+struct tegra_pcie_bus {
+ struct vm_struct *area;
+ struct list_head list;
+ unsigned int nr;
+};
+
+static inline void afi_writel(struct tegra_pcie *pcie, u32 value,
+ unsigned long offset)
+{
+ writel(value, pcie->afi + offset);
+}
+
+static inline u32 afi_readl(struct tegra_pcie *pcie, unsigned long offset)
+{
+ return readl(pcie->afi + offset);
+}
+
+static inline void pads_writel(struct tegra_pcie *pcie, u32 value,
+ unsigned long offset)
+{
+ writel(value, pcie->pads + offset);
+}
+
+static inline u32 pads_readl(struct tegra_pcie *pcie, unsigned long offset)
+{
+ return readl(pcie->pads + offset);
+}
+
+/*
+ * The configuration space mapping on Tegra is somewhat similar to the ECAM
+ * defined by PCIe. However it deviates a bit in how the 4 bits for extended
+ * register accesses are mapped:
+ *
+ * [27:24] extended register number
+ * [23:16] bus number
+ * [15:11] device number
+ * [10: 8] function number
+ * [ 7: 0] register number
+ *
+ * Mapping the whole extended configuration space would require 256 MiB of
+ * virtual address space, only a small part of which will actually be used.
+ * To work around this, a 1 MiB of virtual addresses are allocated per bus
+ * when the bus is first accessed. When the physical range is mapped, the
+ * the bus number bits are hidden so that the extended register number bits
+ * appear as bits [19:16]. Therefore the virtual mapping looks like this:
+ *
+ * [19:16] extended register number
+ * [15:11] device number
+ * [10: 8] function number
+ * [ 7: 0] register number
+ *
+ * This is achieved by stitching together 16 chunks of 64 KiB of physical
+ * address space via the MMU.
+ */
+static unsigned long tegra_pcie_conf_offset(unsigned int devfn, int where)
+{
+ return ((where & 0xf00) << 8) | (PCI_SLOT(devfn) << 11) |
+ (PCI_FUNC(devfn) << 8) | (where & 0xfc);
+}
+
+static struct tegra_pcie_bus *tegra_pcie_bus_alloc(struct tegra_pcie *pcie,
+ unsigned int busnr)
+{
+ struct device *dev = pcie->dev;
+ pgprot_t prot = pgprot_noncached(PAGE_KERNEL);
+ phys_addr_t cs = pcie->cs->start;
+ struct tegra_pcie_bus *bus;
+ unsigned int i;
+ int err;
+
+ bus = kzalloc(sizeof(*bus), GFP_KERNEL);
+ if (!bus)
+ return ERR_PTR(-ENOMEM);
+
+ INIT_LIST_HEAD(&bus->list);
+ bus->nr = busnr;
+
+ /* allocate 1 MiB of virtual addresses */
+ bus->area = get_vm_area(SZ_1M, VM_IOREMAP);
+ if (!bus->area) {
+ err = -ENOMEM;
+ goto free;
+ }
+
+ /* map each of the 16 chunks of 64 KiB each */
+ for (i = 0; i < 16; i++) {
+ unsigned long virt = (unsigned long)bus->area->addr +
+ i * SZ_64K;
+ phys_addr_t phys = cs + i * SZ_16M + busnr * SZ_64K;
+
+ err = ioremap_page_range(virt, virt + SZ_64K, phys, prot);
+ if (err < 0) {
+ dev_err(dev, "ioremap_page_range() failed: %d\n", err);
+ goto unmap;
+ }
+ }
+
+ return bus;
+
+unmap:
+ vunmap(bus->area->addr);
+free:
+ kfree(bus);
+ return ERR_PTR(err);
+}
+
+static int tegra_pcie_add_bus(struct pci_bus *bus)
+{
+ struct pci_host_bridge *host = pci_find_host_bridge(bus);
+ struct tegra_pcie *pcie = pci_host_bridge_priv(host);
+ struct tegra_pcie_bus *b;
+
+ b = tegra_pcie_bus_alloc(pcie, bus->number);
+ if (IS_ERR(b))
+ return PTR_ERR(b);
+
+ list_add_tail(&b->list, &pcie->buses);
+
+ return 0;
+}
+
+static void tegra_pcie_remove_bus(struct pci_bus *child)
+{
+ struct pci_host_bridge *host = pci_find_host_bridge(child);
+ struct tegra_pcie *pcie = pci_host_bridge_priv(host);
+ struct tegra_pcie_bus *bus, *tmp;
+
+ list_for_each_entry_safe(bus, tmp, &pcie->buses, list) {
+ if (bus->nr == child->number) {
+ vunmap(bus->area->addr);
+ list_del(&bus->list);
+ kfree(bus);
+ break;
+ }
+ }
+}
+
+static void __iomem *tegra_pcie_map_bus(struct pci_bus *bus,
+ unsigned int devfn,
+ int where)
+{
+ struct pci_host_bridge *host = pci_find_host_bridge(bus);
+ struct tegra_pcie *pcie = pci_host_bridge_priv(host);
+ struct device *dev = pcie->dev;
+ void __iomem *addr = NULL;
+
+ if (bus->number == 0) {
+ unsigned int slot = PCI_SLOT(devfn);
+ struct tegra_pcie_port *port;
+
+ list_for_each_entry(port, &pcie->ports, list) {
+ if (port->index + 1 == slot) {
+ addr = port->base + (where & ~3);
+ break;
+ }
+ }
+ } else {
+ struct tegra_pcie_bus *b;
+
+ list_for_each_entry(b, &pcie->buses, list)
+ if (b->nr == bus->number)
+ addr = (void __iomem *)b->area->addr;
+
+ if (!addr) {
+ dev_err(dev, "failed to map cfg. space for bus %u\n",
+ bus->number);
+ return NULL;
+ }
+
+ addr += tegra_pcie_conf_offset(devfn, where);
+ }
+
+ return addr;
+}
+
+static struct pci_ops tegra_pcie_ops = {
+ .add_bus = tegra_pcie_add_bus,
+ .remove_bus = tegra_pcie_remove_bus,
+ .map_bus = tegra_pcie_map_bus,
+ .read = pci_generic_config_read32,
+ .write = pci_generic_config_write32,
+};
+
+static unsigned long tegra_pcie_port_get_pex_ctrl(struct tegra_pcie_port *port)
+{
+ unsigned long ret = 0;
+
+ switch (port->index) {
+ case 0:
+ ret = AFI_PEX0_CTRL;
+ break;
+
+ case 1:
+ ret = AFI_PEX1_CTRL;
+ break;
+
+ case 2:
+ ret = AFI_PEX2_CTRL;
+ break;
+ }
+
+ return ret;
+}
+
+static void tegra_pcie_port_reset(struct tegra_pcie_port *port)
+{
+ unsigned long ctrl = tegra_pcie_port_get_pex_ctrl(port);
+ unsigned long value;
+
+ /* pulse reset signal */
+ value = afi_readl(port->pcie, ctrl);
+ value &= ~AFI_PEX_CTRL_RST;
+ afi_writel(port->pcie, value, ctrl);
+
+ usleep_range(1000, 2000);
+
+ value = afi_readl(port->pcie, ctrl);
+ value |= AFI_PEX_CTRL_RST;
+ afi_writel(port->pcie, value, ctrl);
+}
+
+static void tegra_pcie_port_enable(struct tegra_pcie_port *port)
+{
+ unsigned long ctrl = tegra_pcie_port_get_pex_ctrl(port);
+ const struct tegra_pcie_soc *soc = port->pcie->soc;
+ unsigned long value;
+
+ /* enable reference clock */
+ value = afi_readl(port->pcie, ctrl);
+ value |= AFI_PEX_CTRL_REFCLK_EN;
+
+ if (soc->has_pex_clkreq_en)
+ value |= AFI_PEX_CTRL_CLKREQ_EN;
+
+ value |= AFI_PEX_CTRL_OVERRIDE_EN;
+
+ afi_writel(port->pcie, value, ctrl);
+
+ tegra_pcie_port_reset(port);
+
+ if (soc->force_pca_enable) {
+ value = readl(port->base + RP_VEND_CTL2);
+ value |= RP_VEND_CTL2_PCA_ENABLE;
+ writel(value, port->base + RP_VEND_CTL2);
+ }
+}
+
+static void tegra_pcie_port_disable(struct tegra_pcie_port *port)
+{
+ unsigned long ctrl = tegra_pcie_port_get_pex_ctrl(port);
+ const struct tegra_pcie_soc *soc = port->pcie->soc;
+ unsigned long value;
+
+ /* assert port reset */
+ value = afi_readl(port->pcie, ctrl);
+ value &= ~AFI_PEX_CTRL_RST;
+ afi_writel(port->pcie, value, ctrl);
+
+ /* disable reference clock */
+ value = afi_readl(port->pcie, ctrl);
+
+ if (soc->has_pex_clkreq_en)
+ value &= ~AFI_PEX_CTRL_CLKREQ_EN;
+
+ value &= ~AFI_PEX_CTRL_REFCLK_EN;
+ afi_writel(port->pcie, value, ctrl);
+}
+
+static void tegra_pcie_port_free(struct tegra_pcie_port *port)
+{
+ struct tegra_pcie *pcie = port->pcie;
+ struct device *dev = pcie->dev;
+
+ devm_iounmap(dev, port->base);
+ devm_release_mem_region(dev, port->regs.start,
+ resource_size(&port->regs));
+ list_del(&port->list);
+ devm_kfree(dev, port);
+}
+
+/* Tegra PCIE root complex wrongly reports device class */
+static void tegra_pcie_fixup_class(struct pci_dev *dev)
+{
+ dev->class = PCI_CLASS_BRIDGE_PCI << 8;
+}
+DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_NVIDIA, 0x0bf0, tegra_pcie_fixup_class);
+DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_NVIDIA, 0x0bf1, tegra_pcie_fixup_class);
+DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_NVIDIA, 0x0e1c, tegra_pcie_fixup_class);
+DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_NVIDIA, 0x0e1d, tegra_pcie_fixup_class);
+
+/* Tegra20 and Tegra30 PCIE requires relaxed ordering */
+static void tegra_pcie_relax_enable(struct pci_dev *dev)
+{
+ pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_RELAX_EN);
+}
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_NVIDIA, 0x0bf0, tegra_pcie_relax_enable);
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_NVIDIA, 0x0bf1, tegra_pcie_relax_enable);
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_NVIDIA, 0x0e1c, tegra_pcie_relax_enable);
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_NVIDIA, 0x0e1d, tegra_pcie_relax_enable);
+
+static int tegra_pcie_request_resources(struct tegra_pcie *pcie)
+{
+ struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie);
+ struct list_head *windows = &host->windows;
+ struct device *dev = pcie->dev;
+ int err;
+
+ pci_add_resource_offset(windows, &pcie->pio, pcie->offset.io);
+ pci_add_resource_offset(windows, &pcie->mem, pcie->offset.mem);
+ pci_add_resource_offset(windows, &pcie->prefetch, pcie->offset.mem);
+ pci_add_resource(windows, &pcie->busn);
+
+ err = devm_request_pci_bus_resources(dev, windows);
+ if (err < 0)
+ return err;
+
+ pci_remap_iospace(&pcie->pio, pcie->io.start);
+
+ return 0;
+}
+
+static int tegra_pcie_map_irq(const struct pci_dev *pdev, u8 slot, u8 pin)
+{
+ struct pci_host_bridge *host = pci_find_host_bridge(pdev->bus);
+ struct tegra_pcie *pcie = pci_host_bridge_priv(host);
+ int irq;
+
+ tegra_cpuidle_pcie_irqs_in_use();
+
+ irq = of_irq_parse_and_map_pci(pdev, slot, pin);
+ if (!irq)
+ irq = pcie->irq;
+
+ return irq;
+}
+
+static irqreturn_t tegra_pcie_isr(int irq, void *arg)
+{
+ const char *err_msg[] = {
+ "Unknown",
+ "AXI slave error",
+ "AXI decode error",
+ "Target abort",
+ "Master abort",
+ "Invalid write",
+ "Legacy interrupt",
+ "Response decoding error",
+ "AXI response decoding error",
+ "Transaction timeout",
+ "Slot present pin change",
+ "Slot clock request change",
+ "TMS clock ramp change",
+ "TMS ready for power down",
+ "Peer2Peer error",
+ };
+ struct tegra_pcie *pcie = arg;
+ struct device *dev = pcie->dev;
+ u32 code, signature;
+
+ code = afi_readl(pcie, AFI_INTR_CODE) & AFI_INTR_CODE_MASK;
+ signature = afi_readl(pcie, AFI_INTR_SIGNATURE);
+ afi_writel(pcie, 0, AFI_INTR_CODE);
+
+ if (code == AFI_INTR_LEGACY)
+ return IRQ_NONE;
+
+ if (code >= ARRAY_SIZE(err_msg))
+ code = 0;
+
+ /*
+ * do not pollute kernel log with master abort reports since they
+ * happen a lot during enumeration
+ */
+ if (code == AFI_INTR_MASTER_ABORT)
+ dev_dbg(dev, "%s, signature: %08x\n", err_msg[code], signature);
+ else
+ dev_err(dev, "%s, signature: %08x\n", err_msg[code], signature);
+
+ if (code == AFI_INTR_TARGET_ABORT || code == AFI_INTR_MASTER_ABORT ||
+ code == AFI_INTR_FPCI_DECODE_ERROR) {
+ u32 fpci = afi_readl(pcie, AFI_UPPER_FPCI_ADDRESS) & 0xff;
+ u64 address = (u64)fpci << 32 | (signature & 0xfffffffc);
+
+ if (code == AFI_INTR_MASTER_ABORT)
+ dev_dbg(dev, " FPCI address: %10llx\n", address);
+ else
+ dev_err(dev, " FPCI address: %10llx\n", address);
+ }
+
+ return IRQ_HANDLED;
+}
+
+/*
+ * FPCI map is as follows:
+ * - 0xfdfc000000: I/O space
+ * - 0xfdfe000000: type 0 configuration space
+ * - 0xfdff000000: type 1 configuration space
+ * - 0xfe00000000: type 0 extended configuration space
+ * - 0xfe10000000: type 1 extended configuration space
+ */
+static void tegra_pcie_setup_translations(struct tegra_pcie *pcie)
+{
+ u32 fpci_bar, size, axi_address;
+
+ /* Bar 0: type 1 extended configuration space */
+ fpci_bar = 0xfe100000;
+ size = resource_size(pcie->cs);
+ axi_address = pcie->cs->start;
+ afi_writel(pcie, axi_address, AFI_AXI_BAR0_START);
+ afi_writel(pcie, size >> 12, AFI_AXI_BAR0_SZ);
+ afi_writel(pcie, fpci_bar, AFI_FPCI_BAR0);
+
+ /* Bar 1: downstream IO bar */
+ fpci_bar = 0xfdfc0000;
+ size = resource_size(&pcie->io);
+ axi_address = pcie->io.start;
+ afi_writel(pcie, axi_address, AFI_AXI_BAR1_START);
+ afi_writel(pcie, size >> 12, AFI_AXI_BAR1_SZ);
+ afi_writel(pcie, fpci_bar, AFI_FPCI_BAR1);
+
+ /* Bar 2: prefetchable memory BAR */
+ fpci_bar = (((pcie->prefetch.start >> 12) & 0x0fffffff) << 4) | 0x1;
+ size = resource_size(&pcie->prefetch);
+ axi_address = pcie->prefetch.start;
+ afi_writel(pcie, axi_address, AFI_AXI_BAR2_START);
+ afi_writel(pcie, size >> 12, AFI_AXI_BAR2_SZ);
+ afi_writel(pcie, fpci_bar, AFI_FPCI_BAR2);
+
+ /* Bar 3: non prefetchable memory BAR */
+ fpci_bar = (((pcie->mem.start >> 12) & 0x0fffffff) << 4) | 0x1;
+ size = resource_size(&pcie->mem);
+ axi_address = pcie->mem.start;
+ afi_writel(pcie, axi_address, AFI_AXI_BAR3_START);
+ afi_writel(pcie, size >> 12, AFI_AXI_BAR3_SZ);
+ afi_writel(pcie, fpci_bar, AFI_FPCI_BAR3);
+
+ /* NULL out the remaining BARs as they are not used */
+ afi_writel(pcie, 0, AFI_AXI_BAR4_START);
+ afi_writel(pcie, 0, AFI_AXI_BAR4_SZ);
+ afi_writel(pcie, 0, AFI_FPCI_BAR4);
+
+ afi_writel(pcie, 0, AFI_AXI_BAR5_START);
+ afi_writel(pcie, 0, AFI_AXI_BAR5_SZ);
+ afi_writel(pcie, 0, AFI_FPCI_BAR5);
+
+ /* map all upstream transactions as uncached */
+ afi_writel(pcie, 0, AFI_CACHE_BAR0_ST);
+ afi_writel(pcie, 0, AFI_CACHE_BAR0_SZ);
+ afi_writel(pcie, 0, AFI_CACHE_BAR1_ST);
+ afi_writel(pcie, 0, AFI_CACHE_BAR1_SZ);
+
+ /* MSI translations are setup only when needed */
+ afi_writel(pcie, 0, AFI_MSI_FPCI_BAR_ST);
+ afi_writel(pcie, 0, AFI_MSI_BAR_SZ);
+ afi_writel(pcie, 0, AFI_MSI_AXI_BAR_ST);
+ afi_writel(pcie, 0, AFI_MSI_BAR_SZ);
+}
+
+static int tegra_pcie_pll_wait(struct tegra_pcie *pcie, unsigned long timeout)
+{
+ const struct tegra_pcie_soc *soc = pcie->soc;
+ u32 value;
+
+ timeout = jiffies + msecs_to_jiffies(timeout);
+
+ while (time_before(jiffies, timeout)) {
+ value = pads_readl(pcie, soc->pads_pll_ctl);
+ if (value & PADS_PLL_CTL_LOCKDET)
+ return 0;
+ }
+
+ return -ETIMEDOUT;
+}
+
+static int tegra_pcie_phy_enable(struct tegra_pcie *pcie)
+{
+ struct device *dev = pcie->dev;
+ const struct tegra_pcie_soc *soc = pcie->soc;
+ u32 value;
+ int err;
+
+ /* initialize internal PHY, enable up to 16 PCIE lanes */
+ pads_writel(pcie, 0x0, PADS_CTL_SEL);
+
+ /* override IDDQ to 1 on all 4 lanes */
+ value = pads_readl(pcie, PADS_CTL);
+ value |= PADS_CTL_IDDQ_1L;
+ pads_writel(pcie, value, PADS_CTL);
+
+ /*
+ * Set up PHY PLL inputs select PLLE output as refclock,
+ * set TX ref sel to div10 (not div5).
+ */
+ value = pads_readl(pcie, soc->pads_pll_ctl);
+ value &= ~(PADS_PLL_CTL_REFCLK_MASK | PADS_PLL_CTL_TXCLKREF_MASK);
+ value |= PADS_PLL_CTL_REFCLK_INTERNAL_CML | soc->tx_ref_sel;
+ pads_writel(pcie, value, soc->pads_pll_ctl);
+
+ /* reset PLL */
+ value = pads_readl(pcie, soc->pads_pll_ctl);
+ value &= ~PADS_PLL_CTL_RST_B4SM;
+ pads_writel(pcie, value, soc->pads_pll_ctl);
+
+ usleep_range(20, 100);
+
+ /* take PLL out of reset */
+ value = pads_readl(pcie, soc->pads_pll_ctl);
+ value |= PADS_PLL_CTL_RST_B4SM;
+ pads_writel(pcie, value, soc->pads_pll_ctl);
+
+ /* wait for the PLL to lock */
+ err = tegra_pcie_pll_wait(pcie, 500);
+ if (err < 0) {
+ dev_err(dev, "PLL failed to lock: %d\n", err);
+ return err;
+ }
+
+ /* turn off IDDQ override */
+ value = pads_readl(pcie, PADS_CTL);
+ value &= ~PADS_CTL_IDDQ_1L;
+ pads_writel(pcie, value, PADS_CTL);
+
+ /* enable TX/RX data */
+ value = pads_readl(pcie, PADS_CTL);
+ value |= PADS_CTL_TX_DATA_EN_1L | PADS_CTL_RX_DATA_EN_1L;
+ pads_writel(pcie, value, PADS_CTL);
+
+ return 0;
+}
+
+static int tegra_pcie_phy_disable(struct tegra_pcie *pcie)
+{
+ const struct tegra_pcie_soc *soc = pcie->soc;
+ u32 value;
+
+ /* disable TX/RX data */
+ value = pads_readl(pcie, PADS_CTL);
+ value &= ~(PADS_CTL_TX_DATA_EN_1L | PADS_CTL_RX_DATA_EN_1L);
+ pads_writel(pcie, value, PADS_CTL);
+
+ /* override IDDQ */
+ value = pads_readl(pcie, PADS_CTL);
+ value |= PADS_CTL_IDDQ_1L;
+ pads_writel(pcie, value, PADS_CTL);
+
+ /* reset PLL */
+ value = pads_readl(pcie, soc->pads_pll_ctl);
+ value &= ~PADS_PLL_CTL_RST_B4SM;
+ pads_writel(pcie, value, soc->pads_pll_ctl);
+
+ usleep_range(20, 100);
+
+ return 0;
+}
+
+static int tegra_pcie_port_phy_power_on(struct tegra_pcie_port *port)
+{
+ struct device *dev = port->pcie->dev;
+ unsigned int i;
+ int err;
+
+ for (i = 0; i < port->lanes; i++) {
+ err = phy_power_on(port->phys[i]);
+ if (err < 0) {
+ dev_err(dev, "failed to power on PHY#%u: %d\n", i, err);
+ return err;
+ }
+ }
+
+ return 0;
+}
+
+static int tegra_pcie_port_phy_power_off(struct tegra_pcie_port *port)
+{
+ struct device *dev = port->pcie->dev;
+ unsigned int i;
+ int err;
+
+ for (i = 0; i < port->lanes; i++) {
+ err = phy_power_off(port->phys[i]);
+ if (err < 0) {
+ dev_err(dev, "failed to power off PHY#%u: %d\n", i,
+ err);
+ return err;
+ }
+ }
+
+ return 0;
+}
+
+static int tegra_pcie_phy_power_on(struct tegra_pcie *pcie)
+{
+ struct device *dev = pcie->dev;
+ const struct tegra_pcie_soc *soc = pcie->soc;
+ struct tegra_pcie_port *port;
+ int err;
+
+ if (pcie->legacy_phy) {
+ if (pcie->phy)
+ err = phy_power_on(pcie->phy);
+ else
+ err = tegra_pcie_phy_enable(pcie);
+
+ if (err < 0)
+ dev_err(dev, "failed to power on PHY: %d\n", err);
+
+ return err;
+ }
+
+ list_for_each_entry(port, &pcie->ports, list) {
+ err = tegra_pcie_port_phy_power_on(port);
+ if (err < 0) {
+ dev_err(dev,
+ "failed to power on PCIe port %u PHY: %d\n",
+ port->index, err);
+ return err;
+ }
+ }
+
+ /* Configure the reference clock driver */
+ pads_writel(pcie, soc->pads_refclk_cfg0, PADS_REFCLK_CFG0);
+
+ if (soc->num_ports > 2)
+ pads_writel(pcie, soc->pads_refclk_cfg1, PADS_REFCLK_CFG1);
+
+ return 0;
+}
+
+static int tegra_pcie_phy_power_off(struct tegra_pcie *pcie)
+{
+ struct device *dev = pcie->dev;
+ struct tegra_pcie_port *port;
+ int err;
+
+ if (pcie->legacy_phy) {
+ if (pcie->phy)
+ err = phy_power_off(pcie->phy);
+ else
+ err = tegra_pcie_phy_disable(pcie);
+
+ if (err < 0)
+ dev_err(dev, "failed to power off PHY: %d\n", err);
+
+ return err;
+ }
+
+ list_for_each_entry(port, &pcie->ports, list) {
+ err = tegra_pcie_port_phy_power_off(port);
+ if (err < 0) {
+ dev_err(dev,
+ "failed to power off PCIe port %u PHY: %d\n",
+ port->index, err);
+ return err;
+ }
+ }
+
+ return 0;
+}
+
+static int tegra_pcie_enable_controller(struct tegra_pcie *pcie)
+{
+ struct device *dev = pcie->dev;
+ const struct tegra_pcie_soc *soc = pcie->soc;
+ struct tegra_pcie_port *port;
+ unsigned long value;
+ int err;
+
+ /* enable PLL power down */
+ if (pcie->phy) {
+ value = afi_readl(pcie, AFI_PLLE_CONTROL);
+ value &= ~AFI_PLLE_CONTROL_BYPASS_PADS2PLLE_CONTROL;
+ value |= AFI_PLLE_CONTROL_PADS2PLLE_CONTROL_EN;
+ afi_writel(pcie, value, AFI_PLLE_CONTROL);
+ }
+
+ /* power down PCIe slot clock bias pad */
+ if (soc->has_pex_bias_ctrl)
+ afi_writel(pcie, 0, AFI_PEXBIAS_CTRL_0);
+
+ /* configure mode and disable all ports */
+ value = afi_readl(pcie, AFI_PCIE_CONFIG);
+ value &= ~AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_MASK;
+ value |= AFI_PCIE_CONFIG_PCIE_DISABLE_ALL | pcie->xbar_config;
+
+ list_for_each_entry(port, &pcie->ports, list)
+ value &= ~AFI_PCIE_CONFIG_PCIE_DISABLE(port->index);
+
+ afi_writel(pcie, value, AFI_PCIE_CONFIG);
+
+ if (soc->has_gen2) {
+ value = afi_readl(pcie, AFI_FUSE);
+ value &= ~AFI_FUSE_PCIE_T0_GEN2_DIS;
+ afi_writel(pcie, value, AFI_FUSE);
+ } else {
+ value = afi_readl(pcie, AFI_FUSE);
+ value |= AFI_FUSE_PCIE_T0_GEN2_DIS;
+ afi_writel(pcie, value, AFI_FUSE);
+ }
+
+ err = tegra_pcie_phy_power_on(pcie);
+ if (err < 0) {
+ dev_err(dev, "failed to power on PHY(s): %d\n", err);
+ return err;
+ }
+
+ /* take the PCIe interface module out of reset */
+ reset_control_deassert(pcie->pcie_xrst);
+
+ /* finally enable PCIe */
+ value = afi_readl(pcie, AFI_CONFIGURATION);
+ value |= AFI_CONFIGURATION_EN_FPCI;
+ afi_writel(pcie, value, AFI_CONFIGURATION);
+
+ value = AFI_INTR_EN_INI_SLVERR | AFI_INTR_EN_INI_DECERR |
+ AFI_INTR_EN_TGT_SLVERR | AFI_INTR_EN_TGT_DECERR |
+ AFI_INTR_EN_TGT_WRERR | AFI_INTR_EN_DFPCI_DECERR;
+
+ if (soc->has_intr_prsnt_sense)
+ value |= AFI_INTR_EN_PRSNT_SENSE;
+
+ afi_writel(pcie, value, AFI_AFI_INTR_ENABLE);
+ afi_writel(pcie, 0xffffffff, AFI_SM_INTR_ENABLE);
+
+ /* don't enable MSI for now, only when needed */
+ afi_writel(pcie, AFI_INTR_MASK_INT_MASK, AFI_INTR_MASK);
+
+ /* disable all exceptions */
+ afi_writel(pcie, 0, AFI_FPCI_ERROR_MASKS);
+
+ return 0;
+}
+
+static void tegra_pcie_power_off(struct tegra_pcie *pcie)
+{
+ struct device *dev = pcie->dev;
+ int err;
+
+ /* TODO: disable and unprepare clocks? */
+
+ err = tegra_pcie_phy_power_off(pcie);
+ if (err < 0)
+ dev_err(dev, "failed to power off PHY(s): %d\n", err);
+
+ reset_control_assert(pcie->pcie_xrst);
+ reset_control_assert(pcie->afi_rst);
+ reset_control_assert(pcie->pex_rst);
+
+ tegra_powergate_power_off(TEGRA_POWERGATE_PCIE);
+
+ err = regulator_bulk_disable(pcie->num_supplies, pcie->supplies);
+ if (err < 0)
+ dev_warn(dev, "failed to disable regulators: %d\n", err);
+}
+
+static int tegra_pcie_power_on(struct tegra_pcie *pcie)
+{
+ struct device *dev = pcie->dev;
+ const struct tegra_pcie_soc *soc = pcie->soc;
+ int err;
+
+ reset_control_assert(pcie->pcie_xrst);
+ reset_control_assert(pcie->afi_rst);
+ reset_control_assert(pcie->pex_rst);
+
+ tegra_powergate_power_off(TEGRA_POWERGATE_PCIE);
+
+ /* enable regulators */
+ err = regulator_bulk_enable(pcie->num_supplies, pcie->supplies);
+ if (err < 0)
+ dev_err(dev, "failed to enable regulators: %d\n", err);
+
+ err = tegra_powergate_sequence_power_up(TEGRA_POWERGATE_PCIE,
+ pcie->pex_clk,
+ pcie->pex_rst);
+ if (err) {
+ dev_err(dev, "powerup sequence failed: %d\n", err);
+ return err;
+ }
+
+ reset_control_deassert(pcie->afi_rst);
+
+ err = clk_prepare_enable(pcie->afi_clk);
+ if (err < 0) {
+ dev_err(dev, "failed to enable AFI clock: %d\n", err);
+ return err;
+ }
+
+ if (soc->has_cml_clk) {
+ err = clk_prepare_enable(pcie->cml_clk);
+ if (err < 0) {
+ dev_err(dev, "failed to enable CML clock: %d\n", err);
+ return err;
+ }
+ }
+
+ err = clk_prepare_enable(pcie->pll_e);
+ if (err < 0) {
+ dev_err(dev, "failed to enable PLLE clock: %d\n", err);
+ return err;
+ }
+
+ return 0;
+}
+
+static int tegra_pcie_clocks_get(struct tegra_pcie *pcie)
+{
+ struct device *dev = pcie->dev;
+ const struct tegra_pcie_soc *soc = pcie->soc;
+
+ pcie->pex_clk = devm_clk_get(dev, "pex");
+ if (IS_ERR(pcie->pex_clk))
+ return PTR_ERR(pcie->pex_clk);
+
+ pcie->afi_clk = devm_clk_get(dev, "afi");
+ if (IS_ERR(pcie->afi_clk))
+ return PTR_ERR(pcie->afi_clk);
+
+ pcie->pll_e = devm_clk_get(dev, "pll_e");
+ if (IS_ERR(pcie->pll_e))
+ return PTR_ERR(pcie->pll_e);
+
+ if (soc->has_cml_clk) {
+ pcie->cml_clk = devm_clk_get(dev, "cml");
+ if (IS_ERR(pcie->cml_clk))
+ return PTR_ERR(pcie->cml_clk);
+ }
+
+ return 0;
+}
+
+static int tegra_pcie_resets_get(struct tegra_pcie *pcie)
+{
+ struct device *dev = pcie->dev;
+
+ pcie->pex_rst = devm_reset_control_get_exclusive(dev, "pex");
+ if (IS_ERR(pcie->pex_rst))
+ return PTR_ERR(pcie->pex_rst);
+
+ pcie->afi_rst = devm_reset_control_get_exclusive(dev, "afi");
+ if (IS_ERR(pcie->afi_rst))
+ return PTR_ERR(pcie->afi_rst);
+
+ pcie->pcie_xrst = devm_reset_control_get_exclusive(dev, "pcie_x");
+ if (IS_ERR(pcie->pcie_xrst))
+ return PTR_ERR(pcie->pcie_xrst);
+
+ return 0;
+}
+
+static int tegra_pcie_phys_get_legacy(struct tegra_pcie *pcie)
+{
+ struct device *dev = pcie->dev;
+ int err;
+
+ pcie->phy = devm_phy_optional_get(dev, "pcie");
+ if (IS_ERR(pcie->phy)) {
+ err = PTR_ERR(pcie->phy);
+ dev_err(dev, "failed to get PHY: %d\n", err);
+ return err;
+ }
+
+ err = phy_init(pcie->phy);
+ if (err < 0) {
+ dev_err(dev, "failed to initialize PHY: %d\n", err);
+ return err;
+ }
+
+ pcie->legacy_phy = true;
+
+ return 0;
+}
+
+static struct phy *devm_of_phy_optional_get_index(struct device *dev,
+ struct device_node *np,
+ const char *consumer,
+ unsigned int index)
+{
+ struct phy *phy;
+ char *name;
+
+ name = kasprintf(GFP_KERNEL, "%s-%u", consumer, index);
+ if (!name)
+ return ERR_PTR(-ENOMEM);
+
+ phy = devm_of_phy_get(dev, np, name);
+ kfree(name);
+
+ if (IS_ERR(phy) && PTR_ERR(phy) == -ENODEV)
+ phy = NULL;
+
+ return phy;
+}
+
+static int tegra_pcie_port_get_phys(struct tegra_pcie_port *port)
+{
+ struct device *dev = port->pcie->dev;
+ struct phy *phy;
+ unsigned int i;
+ int err;
+
+ port->phys = devm_kcalloc(dev, sizeof(phy), port->lanes, GFP_KERNEL);
+ if (!port->phys)
+ return -ENOMEM;
+
+ for (i = 0; i < port->lanes; i++) {
+ phy = devm_of_phy_optional_get_index(dev, port->np, "pcie", i);
+ if (IS_ERR(phy)) {
+ dev_err(dev, "failed to get PHY#%u: %ld\n", i,
+ PTR_ERR(phy));
+ return PTR_ERR(phy);
+ }
+
+ err = phy_init(phy);
+ if (err < 0) {
+ dev_err(dev, "failed to initialize PHY#%u: %d\n", i,
+ err);
+ return err;
+ }
+
+ port->phys[i] = phy;
+ }
+
+ return 0;
+}
+
+static int tegra_pcie_phys_get(struct tegra_pcie *pcie)
+{
+ const struct tegra_pcie_soc *soc = pcie->soc;
+ struct device_node *np = pcie->dev->of_node;
+ struct tegra_pcie_port *port;
+ int err;
+
+ if (!soc->has_gen2 || of_find_property(np, "phys", NULL) != NULL)
+ return tegra_pcie_phys_get_legacy(pcie);
+
+ list_for_each_entry(port, &pcie->ports, list) {
+ err = tegra_pcie_port_get_phys(port);
+ if (err < 0)
+ return err;
+ }
+
+ return 0;
+}
+
+static int tegra_pcie_get_resources(struct tegra_pcie *pcie)
+{
+ struct device *dev = pcie->dev;
+ struct platform_device *pdev = to_platform_device(dev);
+ struct resource *pads, *afi, *res;
+ int err;
+
+ err = tegra_pcie_clocks_get(pcie);
+ if (err) {
+ dev_err(dev, "failed to get clocks: %d\n", err);
+ return err;
+ }
+
+ err = tegra_pcie_resets_get(pcie);
+ if (err) {
+ dev_err(dev, "failed to get resets: %d\n", err);
+ return err;
+ }
+
+ err = tegra_pcie_phys_get(pcie);
+ if (err < 0) {
+ dev_err(dev, "failed to get PHYs: %d\n", err);
+ return err;
+ }
+
+ err = tegra_pcie_power_on(pcie);
+ if (err) {
+ dev_err(dev, "failed to power up: %d\n", err);
+ return err;
+ }
+
+ pads = platform_get_resource_byname(pdev, IORESOURCE_MEM, "pads");
+ pcie->pads = devm_ioremap_resource(dev, pads);
+ if (IS_ERR(pcie->pads)) {
+ err = PTR_ERR(pcie->pads);
+ goto poweroff;
+ }
+
+ afi = platform_get_resource_byname(pdev, IORESOURCE_MEM, "afi");
+ pcie->afi = devm_ioremap_resource(dev, afi);
+ if (IS_ERR(pcie->afi)) {
+ err = PTR_ERR(pcie->afi);
+ goto poweroff;
+ }
+
+ /* request configuration space, but remap later, on demand */
+ res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "cs");
+ if (!res) {
+ err = -EADDRNOTAVAIL;
+ goto poweroff;
+ }
+
+ pcie->cs = devm_request_mem_region(dev, res->start,
+ resource_size(res), res->name);
+ if (!pcie->cs) {
+ err = -EADDRNOTAVAIL;
+ goto poweroff;
+ }
+
+ /* request interrupt */
+ err = platform_get_irq_byname(pdev, "intr");
+ if (err < 0) {
+ dev_err(dev, "failed to get IRQ: %d\n", err);
+ goto poweroff;
+ }
+
+ pcie->irq = err;
+
+ err = request_irq(pcie->irq, tegra_pcie_isr, IRQF_SHARED, "PCIE", pcie);
+ if (err) {
+ dev_err(dev, "failed to register IRQ: %d\n", err);
+ goto poweroff;
+ }
+
+ return 0;
+
+poweroff:
+ tegra_pcie_power_off(pcie);
+ return err;
+}
+
+static int tegra_pcie_put_resources(struct tegra_pcie *pcie)
+{
+ struct device *dev = pcie->dev;
+ int err;
+
+ if (pcie->irq > 0)
+ free_irq(pcie->irq, pcie);
+
+ tegra_pcie_power_off(pcie);
+
+ err = phy_exit(pcie->phy);
+ if (err < 0)
+ dev_err(dev, "failed to teardown PHY: %d\n", err);
+
+ return 0;
+}
+
+static int tegra_msi_alloc(struct tegra_msi *chip)
+{
+ int msi;
+
+ mutex_lock(&chip->lock);
+
+ msi = find_first_zero_bit(chip->used, INT_PCI_MSI_NR);
+ if (msi < INT_PCI_MSI_NR)
+ set_bit(msi, chip->used);
+ else
+ msi = -ENOSPC;
+
+ mutex_unlock(&chip->lock);
+
+ return msi;
+}
+
+static void tegra_msi_free(struct tegra_msi *chip, unsigned long irq)
+{
+ struct device *dev = chip->chip.dev;
+
+ mutex_lock(&chip->lock);
+
+ if (!test_bit(irq, chip->used))
+ dev_err(dev, "trying to free unused MSI#%lu\n", irq);
+ else
+ clear_bit(irq, chip->used);
+
+ mutex_unlock(&chip->lock);
+}
+
+static irqreturn_t tegra_pcie_msi_irq(int irq, void *data)
+{
+ struct tegra_pcie *pcie = data;
+ struct device *dev = pcie->dev;
+ struct tegra_msi *msi = &pcie->msi;
+ unsigned int i, processed = 0;
+
+ for (i = 0; i < 8; i++) {
+ unsigned long reg = afi_readl(pcie, AFI_MSI_VEC0 + i * 4);
+
+ while (reg) {
+ unsigned int offset = find_first_bit(®, 32);
+ unsigned int index = i * 32 + offset;
+ unsigned int irq;
+
+ /* clear the interrupt */
+ afi_writel(pcie, 1 << offset, AFI_MSI_VEC0 + i * 4);
+
+ irq = irq_find_mapping(msi->domain, index);
+ if (irq) {
+ if (test_bit(index, msi->used))
+ generic_handle_irq(irq);
+ else
+ dev_info(dev, "unhandled MSI\n");
+ } else {
+ /*
+ * that's weird who triggered this?
+ * just clear it
+ */
+ dev_info(dev, "unexpected MSI\n");
+ }
+
+ /* see if there's any more pending in this vector */
+ reg = afi_readl(pcie, AFI_MSI_VEC0 + i * 4);
+
+ processed++;
+ }
+ }
+
+ return processed > 0 ? IRQ_HANDLED : IRQ_NONE;
+}
+
+static int tegra_msi_setup_irq(struct msi_controller *chip,
+ struct pci_dev *pdev, struct msi_desc *desc)
+{
+ struct tegra_msi *msi = to_tegra_msi(chip);
+ struct msi_msg msg;
+ unsigned int irq;
+ int hwirq;
+
+ hwirq = tegra_msi_alloc(msi);
+ if (hwirq < 0)
+ return hwirq;
+
+ irq = irq_create_mapping(msi->domain, hwirq);
+ if (!irq) {
+ tegra_msi_free(msi, hwirq);
+ return -EINVAL;
+ }
+
+ irq_set_msi_desc(irq, desc);
+
+ msg.address_lo = lower_32_bits(msi->phys);
+ msg.address_hi = upper_32_bits(msi->phys);
+ msg.data = hwirq;
+
+ pci_write_msi_msg(irq, &msg);
+
+ return 0;
+}
+
+static void tegra_msi_teardown_irq(struct msi_controller *chip,
+ unsigned int irq)
+{
+ struct tegra_msi *msi = to_tegra_msi(chip);
+ struct irq_data *d = irq_get_irq_data(irq);
+ irq_hw_number_t hwirq = irqd_to_hwirq(d);
+
+ irq_dispose_mapping(irq);
+ tegra_msi_free(msi, hwirq);
+}
+
+static struct irq_chip tegra_msi_irq_chip = {
+ .name = "Tegra PCIe MSI",
+ .irq_enable = pci_msi_unmask_irq,
+ .irq_disable = pci_msi_mask_irq,
+ .irq_mask = pci_msi_mask_irq,
+ .irq_unmask = pci_msi_unmask_irq,
+};
+
+static int tegra_msi_map(struct irq_domain *domain, unsigned int irq,
+ irq_hw_number_t hwirq)
+{
+ irq_set_chip_and_handler(irq, &tegra_msi_irq_chip, handle_simple_irq);
+ irq_set_chip_data(irq, domain->host_data);
+
+ tegra_cpuidle_pcie_irqs_in_use();
+
+ return 0;
+}
+
+static const struct irq_domain_ops msi_domain_ops = {
+ .map = tegra_msi_map,
+};
+
+static int tegra_pcie_enable_msi(struct tegra_pcie *pcie)
+{
+ struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie);
+ struct platform_device *pdev = to_platform_device(pcie->dev);
+ const struct tegra_pcie_soc *soc = pcie->soc;
+ struct tegra_msi *msi = &pcie->msi;
+ struct device *dev = pcie->dev;
+ int err;
+ u32 reg;
+
+ mutex_init(&msi->lock);
+
+ msi->chip.dev = dev;
+ msi->chip.setup_irq = tegra_msi_setup_irq;
+ msi->chip.teardown_irq = tegra_msi_teardown_irq;
+
+ msi->domain = irq_domain_add_linear(dev->of_node, INT_PCI_MSI_NR,
+ &msi_domain_ops, &msi->chip);
+ if (!msi->domain) {
+ dev_err(dev, "failed to create IRQ domain\n");
+ return -ENOMEM;
+ }
+
+ err = platform_get_irq_byname(pdev, "msi");
+ if (err < 0) {
+ dev_err(dev, "failed to get IRQ: %d\n", err);
+ goto err;
+ }
+
+ msi->irq = err;
+
+ err = request_irq(msi->irq, tegra_pcie_msi_irq, IRQF_NO_THREAD,
+ tegra_msi_irq_chip.name, pcie);
+ if (err < 0) {
+ dev_err(dev, "failed to request IRQ: %d\n", err);
+ goto err;
+ }
+
+ /* setup AFI/FPCI range */
+ msi->pages = __get_free_pages(GFP_KERNEL, 0);
+ msi->phys = virt_to_phys((void *)msi->pages);
+
+ afi_writel(pcie, msi->phys >> soc->msi_base_shift, AFI_MSI_FPCI_BAR_ST);
+ afi_writel(pcie, msi->phys, AFI_MSI_AXI_BAR_ST);
+ /* this register is in 4K increments */
+ afi_writel(pcie, 1, AFI_MSI_BAR_SZ);
+
+ /* enable all MSI vectors */
+ afi_writel(pcie, 0xffffffff, AFI_MSI_EN_VEC0);
+ afi_writel(pcie, 0xffffffff, AFI_MSI_EN_VEC1);
+ afi_writel(pcie, 0xffffffff, AFI_MSI_EN_VEC2);
+ afi_writel(pcie, 0xffffffff, AFI_MSI_EN_VEC3);
+ afi_writel(pcie, 0xffffffff, AFI_MSI_EN_VEC4);
+ afi_writel(pcie, 0xffffffff, AFI_MSI_EN_VEC5);
+ afi_writel(pcie, 0xffffffff, AFI_MSI_EN_VEC6);
+ afi_writel(pcie, 0xffffffff, AFI_MSI_EN_VEC7);
+
+ /* and unmask the MSI interrupt */
+ reg = afi_readl(pcie, AFI_INTR_MASK);
+ reg |= AFI_INTR_MASK_MSI_MASK;
+ afi_writel(pcie, reg, AFI_INTR_MASK);
+
+ host->msi = &msi->chip;
+
+ return 0;
+
+err:
+ irq_domain_remove(msi->domain);
+ return err;
+}
+
+static int tegra_pcie_disable_msi(struct tegra_pcie *pcie)
+{
+ struct tegra_msi *msi = &pcie->msi;
+ unsigned int i, irq;
+ u32 value;
+
+ /* mask the MSI interrupt */
+ value = afi_readl(pcie, AFI_INTR_MASK);
+ value &= ~AFI_INTR_MASK_MSI_MASK;
+ afi_writel(pcie, value, AFI_INTR_MASK);
+
+ /* disable all MSI vectors */
+ afi_writel(pcie, 0, AFI_MSI_EN_VEC0);
+ afi_writel(pcie, 0, AFI_MSI_EN_VEC1);
+ afi_writel(pcie, 0, AFI_MSI_EN_VEC2);
+ afi_writel(pcie, 0, AFI_MSI_EN_VEC3);
+ afi_writel(pcie, 0, AFI_MSI_EN_VEC4);
+ afi_writel(pcie, 0, AFI_MSI_EN_VEC5);
+ afi_writel(pcie, 0, AFI_MSI_EN_VEC6);
+ afi_writel(pcie, 0, AFI_MSI_EN_VEC7);
+
+ free_pages(msi->pages, 0);
+
+ if (msi->irq > 0)
+ free_irq(msi->irq, pcie);
+
+ for (i = 0; i < INT_PCI_MSI_NR; i++) {
+ irq = irq_find_mapping(msi->domain, i);
+ if (irq > 0)
+ irq_dispose_mapping(irq);
+ }
+
+ irq_domain_remove(msi->domain);
+
+ return 0;
+}
+
+static int tegra_pcie_get_xbar_config(struct tegra_pcie *pcie, u32 lanes,
+ u32 *xbar)
+{
+ struct device *dev = pcie->dev;
+ struct device_node *np = dev->of_node;
+
+ if (of_device_is_compatible(np, "nvidia,tegra124-pcie") ||
+ of_device_is_compatible(np, "nvidia,tegra210-pcie")) {
+ switch (lanes) {
+ case 0x0000104:
+ dev_info(dev, "4x1, 1x1 configuration\n");
+ *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_X4_X1;
+ return 0;
+
+ case 0x0000102:
+ dev_info(dev, "2x1, 1x1 configuration\n");
+ *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_X2_X1;
+ return 0;
+ }
+ } else if (of_device_is_compatible(np, "nvidia,tegra30-pcie")) {
+ switch (lanes) {
+ case 0x00000204:
+ dev_info(dev, "4x1, 2x1 configuration\n");
+ *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_420;
+ return 0;
+
+ case 0x00020202:
+ dev_info(dev, "2x3 configuration\n");
+ *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_222;
+ return 0;
+
+ case 0x00010104:
+ dev_info(dev, "4x1, 1x2 configuration\n");
+ *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_411;
+ return 0;
+ }
+ } else if (of_device_is_compatible(np, "nvidia,tegra20-pcie")) {
+ switch (lanes) {
+ case 0x00000004:
+ dev_info(dev, "single-mode configuration\n");
+ *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_SINGLE;
+ return 0;
+
+ case 0x00000202:
+ dev_info(dev, "dual-mode configuration\n");
+ *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_DUAL;
+ return 0;
+ }
+ }
+
+ return -EINVAL;
+}
+
+/*
+ * Check whether a given set of supplies is available in a device tree node.
+ * This is used to check whether the new or the legacy device tree bindings
+ * should be used.
+ */
+static bool of_regulator_bulk_available(struct device_node *np,
+ struct regulator_bulk_data *supplies,
+ unsigned int num_supplies)
+{
+ char property[32];
+ unsigned int i;
+
+ for (i = 0; i < num_supplies; i++) {
+ snprintf(property, 32, "%s-supply", supplies[i].supply);
+
+ if (of_find_property(np, property, NULL) == NULL)
+ return false;
+ }
+
+ return true;
+}
+
+/*
+ * Old versions of the device tree binding for this device used a set of power
+ * supplies that didn't match the hardware inputs. This happened to work for a
+ * number of cases but is not future proof. However to preserve backwards-
+ * compatibility with old device trees, this function will try to use the old
+ * set of supplies.
+ */
+static int tegra_pcie_get_legacy_regulators(struct tegra_pcie *pcie)
+{
+ struct device *dev = pcie->dev;
+ struct device_node *np = dev->of_node;
+
+ if (of_device_is_compatible(np, "nvidia,tegra30-pcie"))
+ pcie->num_supplies = 3;
+ else if (of_device_is_compatible(np, "nvidia,tegra20-pcie"))
+ pcie->num_supplies = 2;
+
+ if (pcie->num_supplies == 0) {
+ dev_err(dev, "device %pOF not supported in legacy mode\n", np);
+ return -ENODEV;
+ }
+
+ pcie->supplies = devm_kcalloc(dev, pcie->num_supplies,
+ sizeof(*pcie->supplies),
+ GFP_KERNEL);
+ if (!pcie->supplies)
+ return -ENOMEM;
+
+ pcie->supplies[0].supply = "pex-clk";
+ pcie->supplies[1].supply = "vdd";
+
+ if (pcie->num_supplies > 2)
+ pcie->supplies[2].supply = "avdd";
+
+ return devm_regulator_bulk_get(dev, pcie->num_supplies, pcie->supplies);
+}
+
+/*
+ * Obtains the list of regulators required for a particular generation of the
+ * IP block.
+ *
+ * This would've been nice to do simply by providing static tables for use
+ * with the regulator_bulk_*() API, but unfortunately Tegra30 is a bit quirky
+ * in that it has two pairs or AVDD_PEX and VDD_PEX supplies (PEXA and PEXB)
+ * and either seems to be optional depending on which ports are being used.
+ */
+static int tegra_pcie_get_regulators(struct tegra_pcie *pcie, u32 lane_mask)
+{
+ struct device *dev = pcie->dev;
+ struct device_node *np = dev->of_node;
+ unsigned int i = 0;
+
+ if (of_device_is_compatible(np, "nvidia,tegra210-pcie")) {
+ pcie->num_supplies = 6;
+
+ pcie->supplies = devm_kcalloc(pcie->dev, pcie->num_supplies,
+ sizeof(*pcie->supplies),
+ GFP_KERNEL);
+ if (!pcie->supplies)
+ return -ENOMEM;
+
+ pcie->supplies[i++].supply = "avdd-pll-uerefe";
+ pcie->supplies[i++].supply = "hvddio-pex";
+ pcie->supplies[i++].supply = "dvddio-pex";
+ pcie->supplies[i++].supply = "dvdd-pex-pll";
+ pcie->supplies[i++].supply = "hvdd-pex-pll-e";
+ pcie->supplies[i++].supply = "vddio-pex-ctl";
+ } else if (of_device_is_compatible(np, "nvidia,tegra124-pcie")) {
+ pcie->num_supplies = 7;
+
+ pcie->supplies = devm_kcalloc(dev, pcie->num_supplies,
+ sizeof(*pcie->supplies),
+ GFP_KERNEL);
+ if (!pcie->supplies)
+ return -ENOMEM;
+
+ pcie->supplies[i++].supply = "avddio-pex";
+ pcie->supplies[i++].supply = "dvddio-pex";
+ pcie->supplies[i++].supply = "avdd-pex-pll";
+ pcie->supplies[i++].supply = "hvdd-pex";
+ pcie->supplies[i++].supply = "hvdd-pex-pll-e";
+ pcie->supplies[i++].supply = "vddio-pex-ctl";
+ pcie->supplies[i++].supply = "avdd-pll-erefe";
+ } else if (of_device_is_compatible(np, "nvidia,tegra30-pcie")) {
+ bool need_pexa = false, need_pexb = false;
+
+ /* VDD_PEXA and AVDD_PEXA supply lanes 0 to 3 */
+ if (lane_mask & 0x0f)
+ need_pexa = true;
+
+ /* VDD_PEXB and AVDD_PEXB supply lanes 4 to 5 */
+ if (lane_mask & 0x30)
+ need_pexb = true;
+
+ pcie->num_supplies = 4 + (need_pexa ? 2 : 0) +
+ (need_pexb ? 2 : 0);
+
+ pcie->supplies = devm_kcalloc(dev, pcie->num_supplies,
+ sizeof(*pcie->supplies),
+ GFP_KERNEL);
+ if (!pcie->supplies)
+ return -ENOMEM;
+
+ pcie->supplies[i++].supply = "avdd-pex-pll";
+ pcie->supplies[i++].supply = "hvdd-pex";
+ pcie->supplies[i++].supply = "vddio-pex-ctl";
+ pcie->supplies[i++].supply = "avdd-plle";
+
+ if (need_pexa) {
+ pcie->supplies[i++].supply = "avdd-pexa";
+ pcie->supplies[i++].supply = "vdd-pexa";
+ }
+
+ if (need_pexb) {
+ pcie->supplies[i++].supply = "avdd-pexb";
+ pcie->supplies[i++].supply = "vdd-pexb";
+ }
+ } else if (of_device_is_compatible(np, "nvidia,tegra20-pcie")) {
+ pcie->num_supplies = 5;
+
+ pcie->supplies = devm_kcalloc(dev, pcie->num_supplies,
+ sizeof(*pcie->supplies),
+ GFP_KERNEL);
+ if (!pcie->supplies)
+ return -ENOMEM;
+
+ pcie->supplies[0].supply = "avdd-pex";
+ pcie->supplies[1].supply = "vdd-pex";
+ pcie->supplies[2].supply = "avdd-pex-pll";
+ pcie->supplies[3].supply = "avdd-plle";
+ pcie->supplies[4].supply = "vddio-pex-clk";
+ }
+
+ if (of_regulator_bulk_available(dev->of_node, pcie->supplies,
+ pcie->num_supplies))
+ return devm_regulator_bulk_get(dev, pcie->num_supplies,
+ pcie->supplies);
+
+ /*
+ * If not all regulators are available for this new scheme, assume
+ * that the device tree complies with an older version of the device
+ * tree binding.
+ */
+ dev_info(dev, "using legacy DT binding for power supplies\n");
+
+ devm_kfree(dev, pcie->supplies);
+ pcie->num_supplies = 0;
+
+ return tegra_pcie_get_legacy_regulators(pcie);
+}
+
+static int tegra_pcie_parse_dt(struct tegra_pcie *pcie)
+{
+ struct device *dev = pcie->dev;
+ struct device_node *np = dev->of_node, *port;
+ const struct tegra_pcie_soc *soc = pcie->soc;
+ struct of_pci_range_parser parser;
+ struct of_pci_range range;
+ u32 lanes = 0, mask = 0;
+ unsigned int lane = 0;
+ struct resource res;
+ int err;
+
+ if (of_pci_range_parser_init(&parser, np)) {
+ dev_err(dev, "missing \"ranges\" property\n");
+ return -EINVAL;
+ }
+
+ for_each_of_pci_range(&parser, &range) {
+ err = of_pci_range_to_resource(&range, np, &res);
+ if (err < 0)
+ return err;
+
+ switch (res.flags & IORESOURCE_TYPE_BITS) {
+ case IORESOURCE_IO:
+ /* Track the bus -> CPU I/O mapping offset. */
+ pcie->offset.io = res.start - range.pci_addr;
+
+ memcpy(&pcie->pio, &res, sizeof(res));
+ pcie->pio.name = np->full_name;
+
+ /*
+ * The Tegra PCIe host bridge uses this to program the
+ * mapping of the I/O space to the physical address,
+ * so we override the .start and .end fields here that
+ * of_pci_range_to_resource() converted to I/O space.
+ * We also set the IORESOURCE_MEM type to clarify that
+ * the resource is in the physical memory space.
+ */
+ pcie->io.start = range.cpu_addr;
+ pcie->io.end = range.cpu_addr + range.size - 1;
+ pcie->io.flags = IORESOURCE_MEM;
+ pcie->io.name = "I/O";
+
+ memcpy(&res, &pcie->io, sizeof(res));
+ break;
+
+ case IORESOURCE_MEM:
+ /*
+ * Track the bus -> CPU memory mapping offset. This
+ * assumes that the prefetchable and non-prefetchable
+ * regions will be the last of type IORESOURCE_MEM in
+ * the ranges property.
+ * */
+ pcie->offset.mem = res.start - range.pci_addr;
+
+ if (res.flags & IORESOURCE_PREFETCH) {
+ memcpy(&pcie->prefetch, &res, sizeof(res));
+ pcie->prefetch.name = "prefetchable";
+ } else {
+ memcpy(&pcie->mem, &res, sizeof(res));
+ pcie->mem.name = "non-prefetchable";
+ }
+ break;
+ }
+ }
+
+ err = of_pci_parse_bus_range(np, &pcie->busn);
+ if (err < 0) {
+ dev_err(dev, "failed to parse ranges property: %d\n", err);
+ pcie->busn.name = np->name;
+ pcie->busn.start = 0;
+ pcie->busn.end = 0xff;
+ pcie->busn.flags = IORESOURCE_BUS;
+ }
+
+ /* parse root ports */
+ for_each_child_of_node(np, port) {
+ struct tegra_pcie_port *rp;
+ unsigned int index;
+ u32 value;
+
+ err = of_pci_get_devfn(port);
+ if (err < 0) {
+ dev_err(dev, "failed to parse address: %d\n", err);
+ goto err_node_put;
+ }
+
+ index = PCI_SLOT(err);
+
+ if (index < 1 || index > soc->num_ports) {
+ dev_err(dev, "invalid port number: %d\n", index);
+ err = -EINVAL;
+ goto err_node_put;
+ }
+
+ index--;
+
+ err = of_property_read_u32(port, "nvidia,num-lanes", &value);
+ if (err < 0) {
+ dev_err(dev, "failed to parse # of lanes: %d\n",
+ err);
+ goto err_node_put;
+ }
+
+ if (value > 16) {
+ dev_err(dev, "invalid # of lanes: %u\n", value);
+ err = -EINVAL;
+ goto err_node_put;
+ }
+
+ lanes |= value << (index << 3);
+
+ if (!of_device_is_available(port)) {
+ lane += value;
+ continue;
+ }
+
+ mask |= ((1 << value) - 1) << lane;
+ lane += value;
+
+ rp = devm_kzalloc(dev, sizeof(*rp), GFP_KERNEL);
+ if (!rp) {
+ err = -ENOMEM;
+ goto err_node_put;
+ }
+
+ err = of_address_to_resource(port, 0, &rp->regs);
+ if (err < 0) {
+ dev_err(dev, "failed to parse address: %d\n", err);
+ goto err_node_put;
+ }
+
+ INIT_LIST_HEAD(&rp->list);
+ rp->index = index;
+ rp->lanes = value;
+ rp->pcie = pcie;
+ rp->np = port;
+
+ rp->base = devm_pci_remap_cfg_resource(dev, &rp->regs);
+ if (IS_ERR(rp->base))
+ return PTR_ERR(rp->base);
+
+ list_add_tail(&rp->list, &pcie->ports);
+ }
+
+ err = tegra_pcie_get_xbar_config(pcie, lanes, &pcie->xbar_config);
+ if (err < 0) {
+ dev_err(dev, "invalid lane configuration\n");
+ return err;
+ }
+
+ err = tegra_pcie_get_regulators(pcie, mask);
+ if (err < 0)
+ return err;
+
+ return 0;
+
+err_node_put:
+ of_node_put(port);
+ return err;
+}
+
+/*
+ * FIXME: If there are no PCIe cards attached, then calling this function
+ * can result in the increase of the bootup time as there are big timeout
+ * loops.
+ */
+#define TEGRA_PCIE_LINKUP_TIMEOUT 200 /* up to 1.2 seconds */
+static bool tegra_pcie_port_check_link(struct tegra_pcie_port *port)
+{
+ struct device *dev = port->pcie->dev;
+ unsigned int retries = 3;
+ unsigned long value;
+
+ /* override presence detection */
+ value = readl(port->base + RP_PRIV_MISC);
+ value &= ~RP_PRIV_MISC_PRSNT_MAP_EP_ABSNT;
+ value |= RP_PRIV_MISC_PRSNT_MAP_EP_PRSNT;
+ writel(value, port->base + RP_PRIV_MISC);
+
+ do {
+ unsigned int timeout = TEGRA_PCIE_LINKUP_TIMEOUT;
+
+ do {
+ value = readl(port->base + RP_VEND_XP);
+
+ if (value & RP_VEND_XP_DL_UP)
+ break;
+
+ usleep_range(1000, 2000);
+ } while (--timeout);
+
+ if (!timeout) {
+ dev_err(dev, "link %u down, retrying\n", port->index);
+ goto retry;
+ }
+
+ timeout = TEGRA_PCIE_LINKUP_TIMEOUT;
+
+ do {
+ value = readl(port->base + RP_LINK_CONTROL_STATUS);
+
+ if (value & RP_LINK_CONTROL_STATUS_DL_LINK_ACTIVE)
+ return true;
+
+ usleep_range(1000, 2000);
+ } while (--timeout);
+
+retry:
+ tegra_pcie_port_reset(port);
+ } while (--retries);
+
+ return false;
+}
+
+static void tegra_pcie_enable_ports(struct tegra_pcie *pcie)
+{
+ struct device *dev = pcie->dev;
+ struct tegra_pcie_port *port, *tmp;
+
+ list_for_each_entry_safe(port, tmp, &pcie->ports, list) {
+ dev_info(dev, "probing port %u, using %u lanes\n",
+ port->index, port->lanes);
+
+ tegra_pcie_port_enable(port);
+
+ if (tegra_pcie_port_check_link(port))
+ continue;
+
+ dev_info(dev, "link %u down, ignoring\n", port->index);
+
+ tegra_pcie_port_disable(port);
+ tegra_pcie_port_free(port);
+ }
+}
+
+static const struct tegra_pcie_soc tegra20_pcie = {
+ .num_ports = 2,
+ .msi_base_shift = 0,
+ .pads_pll_ctl = PADS_PLL_CTL_TEGRA20,
+ .tx_ref_sel = PADS_PLL_CTL_TXCLKREF_DIV10,
+ .pads_refclk_cfg0 = 0xfa5cfa5c,
+ .has_pex_clkreq_en = false,
+ .has_pex_bias_ctrl = false,
+ .has_intr_prsnt_sense = false,
+ .has_cml_clk = false,
+ .has_gen2 = false,
+ .force_pca_enable = false,
+};
+
+static const struct tegra_pcie_soc tegra30_pcie = {
+ .num_ports = 3,
+ .msi_base_shift = 8,
+ .pads_pll_ctl = PADS_PLL_CTL_TEGRA30,
+ .tx_ref_sel = PADS_PLL_CTL_TXCLKREF_BUF_EN,
+ .pads_refclk_cfg0 = 0xfa5cfa5c,
+ .pads_refclk_cfg1 = 0xfa5cfa5c,
+ .has_pex_clkreq_en = true,
+ .has_pex_bias_ctrl = true,
+ .has_intr_prsnt_sense = true,
+ .has_cml_clk = true,
+ .has_gen2 = false,
+ .force_pca_enable = false,
+};
+
+static const struct tegra_pcie_soc tegra124_pcie = {
+ .num_ports = 2,
+ .msi_base_shift = 8,
+ .pads_pll_ctl = PADS_PLL_CTL_TEGRA30,
+ .tx_ref_sel = PADS_PLL_CTL_TXCLKREF_BUF_EN,
+ .pads_refclk_cfg0 = 0x44ac44ac,
+ .has_pex_clkreq_en = true,
+ .has_pex_bias_ctrl = true,
+ .has_intr_prsnt_sense = true,
+ .has_cml_clk = true,
+ .has_gen2 = true,
+ .force_pca_enable = false,
+};
+
+static const struct tegra_pcie_soc tegra210_pcie = {
+ .num_ports = 2,
+ .msi_base_shift = 8,
+ .pads_pll_ctl = PADS_PLL_CTL_TEGRA30,
+ .tx_ref_sel = PADS_PLL_CTL_TXCLKREF_BUF_EN,
+ .pads_refclk_cfg0 = 0x90b890b8,
+ .has_pex_clkreq_en = true,
+ .has_pex_bias_ctrl = true,
+ .has_intr_prsnt_sense = true,
+ .has_cml_clk = true,
+ .has_gen2 = true,
+ .force_pca_enable = true,
+};
+
+static const struct of_device_id tegra_pcie_of_match[] = {
+ { .compatible = "nvidia,tegra210-pcie", .data = &tegra210_pcie },
+ { .compatible = "nvidia,tegra124-pcie", .data = &tegra124_pcie },
+ { .compatible = "nvidia,tegra30-pcie", .data = &tegra30_pcie },
+ { .compatible = "nvidia,tegra20-pcie", .data = &tegra20_pcie },
+ { },
+};
+
+static void *tegra_pcie_ports_seq_start(struct seq_file *s, loff_t *pos)
+{
+ struct tegra_pcie *pcie = s->private;
+
+ if (list_empty(&pcie->ports))
+ return NULL;
+
+ seq_printf(s, "Index Status\n");
+
+ return seq_list_start(&pcie->ports, *pos);
+}
+
+static void *tegra_pcie_ports_seq_next(struct seq_file *s, void *v, loff_t *pos)
+{
+ struct tegra_pcie *pcie = s->private;
+
+ return seq_list_next(v, &pcie->ports, pos);
+}
+
+static void tegra_pcie_ports_seq_stop(struct seq_file *s, void *v)
+{
+}
+
+static int tegra_pcie_ports_seq_show(struct seq_file *s, void *v)
+{
+ bool up = false, active = false;
+ struct tegra_pcie_port *port;
+ unsigned int value;
+
+ port = list_entry(v, struct tegra_pcie_port, list);
+
+ value = readl(port->base + RP_VEND_XP);
+
+ if (value & RP_VEND_XP_DL_UP)
+ up = true;
+
+ value = readl(port->base + RP_LINK_CONTROL_STATUS);
+
+ if (value & RP_LINK_CONTROL_STATUS_DL_LINK_ACTIVE)
+ active = true;
+
+ seq_printf(s, "%2u ", port->index);
+
+ if (up)
+ seq_printf(s, "up");
+
+ if (active) {
+ if (up)
+ seq_printf(s, ", ");
+
+ seq_printf(s, "active");
+ }
+
+ seq_printf(s, "\n");
+ return 0;
+}
+
+static const struct seq_operations tegra_pcie_ports_seq_ops = {
+ .start = tegra_pcie_ports_seq_start,
+ .next = tegra_pcie_ports_seq_next,
+ .stop = tegra_pcie_ports_seq_stop,
+ .show = tegra_pcie_ports_seq_show,
+};
+
+static int tegra_pcie_ports_open(struct inode *inode, struct file *file)
+{
+ struct tegra_pcie *pcie = inode->i_private;
+ struct seq_file *s;
+ int err;
+
+ err = seq_open(file, &tegra_pcie_ports_seq_ops);
+ if (err)
+ return err;
+
+ s = file->private_data;
+ s->private = pcie;
+
+ return 0;
+}
+
+static const struct file_operations tegra_pcie_ports_ops = {
+ .owner = THIS_MODULE,
+ .open = tegra_pcie_ports_open,
+ .read = seq_read,
+ .llseek = seq_lseek,
+ .release = seq_release,
+};
+
+static int tegra_pcie_debugfs_init(struct tegra_pcie *pcie)
+{
+ struct dentry *file;
+
+ pcie->debugfs = debugfs_create_dir("pcie", NULL);
+ if (!pcie->debugfs)
+ return -ENOMEM;
+
+ file = debugfs_create_file("ports", S_IFREG | S_IRUGO, pcie->debugfs,
+ pcie, &tegra_pcie_ports_ops);
+ if (!file)
+ goto remove;
+
+ return 0;
+
+remove:
+ debugfs_remove_recursive(pcie->debugfs);
+ pcie->debugfs = NULL;
+ return -ENOMEM;
+}
+
+static int tegra_pcie_probe(struct platform_device *pdev)
+{
+ struct device *dev = &pdev->dev;
+ struct pci_host_bridge *host;
+ struct tegra_pcie *pcie;
+ struct pci_bus *child;
+ int err;
+
+ host = devm_pci_alloc_host_bridge(dev, sizeof(*pcie));
+ if (!host)
+ return -ENOMEM;
+
+ pcie = pci_host_bridge_priv(host);
+
+ pcie->soc = of_device_get_match_data(dev);
+ INIT_LIST_HEAD(&pcie->buses);
+ INIT_LIST_HEAD(&pcie->ports);
+ pcie->dev = dev;
+
+ err = tegra_pcie_parse_dt(pcie);
+ if (err < 0)
+ return err;
+
+ err = tegra_pcie_get_resources(pcie);
+ if (err < 0) {
+ dev_err(dev, "failed to request resources: %d\n", err);
+ return err;
+ }
+
+ err = tegra_pcie_enable_controller(pcie);
+ if (err)
+ goto put_resources;
+
+ err = tegra_pcie_request_resources(pcie);
+ if (err)
+ goto put_resources;
+
+ /* setup the AFI address translations */
+ tegra_pcie_setup_translations(pcie);
+
+ if (IS_ENABLED(CONFIG_PCI_MSI)) {
+ err = tegra_pcie_enable_msi(pcie);
+ if (err < 0) {
+ dev_err(dev, "failed to enable MSI support: %d\n", err);
+ goto put_resources;
+ }
+ }
+
+ tegra_pcie_enable_ports(pcie);
+
+ pci_add_flags(PCI_REASSIGN_ALL_RSRC | PCI_REASSIGN_ALL_BUS);
+ host->busnr = pcie->busn.start;
+ host->dev.parent = &pdev->dev;
+ host->ops = &tegra_pcie_ops;
+ host->map_irq = tegra_pcie_map_irq;
+ host->swizzle_irq = pci_common_swizzle;
+
+ err = pci_scan_root_bus_bridge(host);
+ if (err < 0) {
+ dev_err(dev, "failed to register host: %d\n", err);
+ goto disable_msi;
+ }
+
+ pci_bus_size_bridges(host->bus);
+ pci_bus_assign_resources(host->bus);
+
+ list_for_each_entry(child, &host->bus->children, node)
+ pcie_bus_configure_settings(child);
+
+ pci_bus_add_devices(host->bus);
+
+ if (IS_ENABLED(CONFIG_DEBUG_FS)) {
+ err = tegra_pcie_debugfs_init(pcie);
+ if (err < 0)
+ dev_err(dev, "failed to setup debugfs: %d\n", err);
+ }
+
+ return 0;
+
+disable_msi:
+ if (IS_ENABLED(CONFIG_PCI_MSI))
+ tegra_pcie_disable_msi(pcie);
+put_resources:
+ tegra_pcie_put_resources(pcie);
+ return err;
+}
+
+static struct platform_driver tegra_pcie_driver = {
+ .driver = {
+ .name = "tegra-pcie",
+ .of_match_table = tegra_pcie_of_match,
+ .suppress_bind_attrs = true,
+ },
+ .probe = tegra_pcie_probe,
+};
+builtin_platform_driver(tegra_pcie_driver);
diff --git a/src/kernel/linux/v4.14/drivers/pci/host/pci-thunder-ecam.c b/src/kernel/linux/v4.14/drivers/pci/host/pci-thunder-ecam.c
new file mode 100644
index 0000000..fc0ca03
--- /dev/null
+++ b/src/kernel/linux/v4.14/drivers/pci/host/pci-thunder-ecam.c
@@ -0,0 +1,383 @@
+/*
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License. See the file "COPYING" in the main directory of this archive
+ * for more details.
+ *
+ * Copyright (C) 2015, 2016 Cavium, Inc.
+ */
+
+#include <linux/kernel.h>
+#include <linux/init.h>
+#include <linux/ioport.h>
+#include <linux/of_pci.h>
+#include <linux/of.h>
+#include <linux/pci-ecam.h>
+#include <linux/platform_device.h>
+
+#if defined(CONFIG_PCI_HOST_THUNDER_ECAM) || (defined(CONFIG_ACPI) && defined(CONFIG_PCI_QUIRKS))
+
+static void set_val(u32 v, int where, int size, u32 *val)
+{
+ int shift = (where & 3) * 8;
+
+ pr_debug("set_val %04x: %08x\n", (unsigned)(where & ~3), v);
+ v >>= shift;
+ if (size == 1)
+ v &= 0xff;
+ else if (size == 2)
+ v &= 0xffff;
+ *val = v;
+}
+
+static int handle_ea_bar(u32 e0, int bar, struct pci_bus *bus,
+ unsigned int devfn, int where, int size, u32 *val)
+{
+ void __iomem *addr;
+ u32 v;
+
+ /* Entries are 16-byte aligned; bits[2,3] select word in entry */
+ int where_a = where & 0xc;
+
+ if (where_a == 0) {
+ set_val(e0, where, size, val);
+ return PCIBIOS_SUCCESSFUL;
+ }
+ if (where_a == 0x4) {
+ addr = bus->ops->map_bus(bus, devfn, bar); /* BAR 0 */
+ if (!addr) {
+ *val = ~0;
+ return PCIBIOS_DEVICE_NOT_FOUND;
+ }
+ v = readl(addr);
+ v &= ~0xf;
+ v |= 2; /* EA entry-1. Base-L */
+ set_val(v, where, size, val);
+ return PCIBIOS_SUCCESSFUL;
+ }
+ if (where_a == 0x8) {
+ u32 barl_orig;
+ u32 barl_rb;
+
+ addr = bus->ops->map_bus(bus, devfn, bar); /* BAR 0 */
+ if (!addr) {
+ *val = ~0;
+ return PCIBIOS_DEVICE_NOT_FOUND;
+ }
+ barl_orig = readl(addr + 0);
+ writel(0xffffffff, addr + 0);
+ barl_rb = readl(addr + 0);
+ writel(barl_orig, addr + 0);
+ /* zeros in unsettable bits */
+ v = ~barl_rb & ~3;
+ v |= 0xc; /* EA entry-2. Offset-L */
+ set_val(v, where, size, val);
+ return PCIBIOS_SUCCESSFUL;
+ }
+ if (where_a == 0xc) {
+ addr = bus->ops->map_bus(bus, devfn, bar + 4); /* BAR 1 */
+ if (!addr) {
+ *val = ~0;
+ return PCIBIOS_DEVICE_NOT_FOUND;
+ }
+ v = readl(addr); /* EA entry-3. Base-H */
+ set_val(v, where, size, val);
+ return PCIBIOS_SUCCESSFUL;
+ }
+ return PCIBIOS_DEVICE_NOT_FOUND;
+}
+
+static int thunder_ecam_p2_config_read(struct pci_bus *bus, unsigned int devfn,
+ int where, int size, u32 *val)
+{
+ struct pci_config_window *cfg = bus->sysdata;
+ int where_a = where & ~3;
+ void __iomem *addr;
+ u32 node_bits;
+ u32 v;
+
+ /* EA Base[63:32] may be missing some bits ... */
+ switch (where_a) {
+ case 0xa8:
+ case 0xbc:
+ case 0xd0:
+ case 0xe4:
+ break;
+ default:
+ return pci_generic_config_read(bus, devfn, where, size, val);
+ }
+
+ addr = bus->ops->map_bus(bus, devfn, where_a);
+ if (!addr) {
+ *val = ~0;
+ return PCIBIOS_DEVICE_NOT_FOUND;
+ }
+
+ v = readl(addr);
+
+ /*
+ * Bit 44 of the 64-bit Base must match the same bit in
+ * the config space access window. Since we are working with
+ * the high-order 32 bits, shift everything down by 32 bits.
+ */
+ node_bits = (cfg->res.start >> 32) & (1 << 12);
+
+ v |= node_bits;
+ set_val(v, where, size, val);
+
+ return PCIBIOS_SUCCESSFUL;
+}
+
+static int thunder_ecam_config_read(struct pci_bus *bus, unsigned int devfn,
+ int where, int size, u32 *val)
+{
+ u32 v;
+ u32 vendor_device;
+ u32 class_rev;
+ void __iomem *addr;
+ int cfg_type;
+ int where_a = where & ~3;
+
+ addr = bus->ops->map_bus(bus, devfn, 0xc);
+ if (!addr) {
+ *val = ~0;
+ return PCIBIOS_DEVICE_NOT_FOUND;
+ }
+
+ v = readl(addr);
+
+ /* Check for non type-00 header */
+ cfg_type = (v >> 16) & 0x7f;
+
+ addr = bus->ops->map_bus(bus, devfn, 8);
+ if (!addr) {
+ *val = ~0;
+ return PCIBIOS_DEVICE_NOT_FOUND;
+ }
+
+ class_rev = readl(addr);
+ if (class_rev == 0xffffffff)
+ goto no_emulation;
+
+ if ((class_rev & 0xff) >= 8) {
+ /* Pass-2 handling */
+ if (cfg_type)
+ goto no_emulation;
+ return thunder_ecam_p2_config_read(bus, devfn, where,
+ size, val);
+ }
+
+ /*
+ * All BARs have fixed addresses specified by the EA
+ * capability; they must return zero on read.
+ */
+ if (cfg_type == 0 &&
+ ((where >= 0x10 && where < 0x2c) ||
+ (where >= 0x1a4 && where < 0x1bc))) {
+ /* BAR or SR-IOV BAR */
+ *val = 0;
+ return PCIBIOS_SUCCESSFUL;
+ }
+
+ addr = bus->ops->map_bus(bus, devfn, 0);
+ if (!addr) {
+ *val = ~0;
+ return PCIBIOS_DEVICE_NOT_FOUND;
+ }
+
+ vendor_device = readl(addr);
+ if (vendor_device == 0xffffffff)
+ goto no_emulation;
+
+ pr_debug("%04x:%04x - Fix pass#: %08x, where: %03x, devfn: %03x\n",
+ vendor_device & 0xffff, vendor_device >> 16, class_rev,
+ (unsigned) where, devfn);
+
+ /* Check for non type-00 header */
+ if (cfg_type == 0) {
+ bool has_msix;
+ bool is_nic = (vendor_device == 0xa01e177d);
+ bool is_tns = (vendor_device == 0xa01f177d);
+
+ addr = bus->ops->map_bus(bus, devfn, 0x70);
+ if (!addr) {
+ *val = ~0;
+ return PCIBIOS_DEVICE_NOT_FOUND;
+ }
+ /* E_CAP */
+ v = readl(addr);
+ has_msix = (v & 0xff00) != 0;
+
+ if (!has_msix && where_a == 0x70) {
+ v |= 0xbc00; /* next capability is EA at 0xbc */
+ set_val(v, where, size, val);
+ return PCIBIOS_SUCCESSFUL;
+ }
+ if (where_a == 0xb0) {
+ addr = bus->ops->map_bus(bus, devfn, where_a);
+ if (!addr) {
+ *val = ~0;
+ return PCIBIOS_DEVICE_NOT_FOUND;
+ }
+ v = readl(addr);
+ if (v & 0xff00)
+ pr_err("Bad MSIX cap header: %08x\n", v);
+ v |= 0xbc00; /* next capability is EA at 0xbc */
+ set_val(v, where, size, val);
+ return PCIBIOS_SUCCESSFUL;
+ }
+ if (where_a == 0xbc) {
+ if (is_nic)
+ v = 0x40014; /* EA last in chain, 4 entries */
+ else if (is_tns)
+ v = 0x30014; /* EA last in chain, 3 entries */
+ else if (has_msix)
+ v = 0x20014; /* EA last in chain, 2 entries */
+ else
+ v = 0x10014; /* EA last in chain, 1 entry */
+ set_val(v, where, size, val);
+ return PCIBIOS_SUCCESSFUL;
+ }
+ if (where_a >= 0xc0 && where_a < 0xd0)
+ /* EA entry-0. PP=0, BAR0 Size:3 */
+ return handle_ea_bar(0x80ff0003,
+ 0x10, bus, devfn, where,
+ size, val);
+ if (where_a >= 0xd0 && where_a < 0xe0 && has_msix)
+ /* EA entry-1. PP=0, BAR4 Size:3 */
+ return handle_ea_bar(0x80ff0043,
+ 0x20, bus, devfn, where,
+ size, val);
+ if (where_a >= 0xe0 && where_a < 0xf0 && is_tns)
+ /* EA entry-2. PP=0, BAR2, Size:3 */
+ return handle_ea_bar(0x80ff0023,
+ 0x18, bus, devfn, where,
+ size, val);
+ if (where_a >= 0xe0 && where_a < 0xf0 && is_nic)
+ /* EA entry-2. PP=4, VF_BAR0 (9), Size:3 */
+ return handle_ea_bar(0x80ff0493,
+ 0x1a4, bus, devfn, where,
+ size, val);
+ if (where_a >= 0xf0 && where_a < 0x100 && is_nic)
+ /* EA entry-3. PP=4, VF_BAR4 (d), Size:3 */
+ return handle_ea_bar(0x80ff04d3,
+ 0x1b4, bus, devfn, where,
+ size, val);
+ } else if (cfg_type == 1) {
+ bool is_rsl_bridge = devfn == 0x08;
+ bool is_rad_bridge = devfn == 0xa0;
+ bool is_zip_bridge = devfn == 0xa8;
+ bool is_dfa_bridge = devfn == 0xb0;
+ bool is_nic_bridge = devfn == 0x10;
+
+ if (where_a == 0x70) {
+ addr = bus->ops->map_bus(bus, devfn, where_a);
+ if (!addr) {
+ *val = ~0;
+ return PCIBIOS_DEVICE_NOT_FOUND;
+ }
+ v = readl(addr);
+ if (v & 0xff00)
+ pr_err("Bad PCIe cap header: %08x\n", v);
+ v |= 0xbc00; /* next capability is EA at 0xbc */
+ set_val(v, where, size, val);
+ return PCIBIOS_SUCCESSFUL;
+ }
+ if (where_a == 0xbc) {
+ if (is_nic_bridge)
+ v = 0x10014; /* EA last in chain, 1 entry */
+ else
+ v = 0x00014; /* EA last in chain, no entries */
+ set_val(v, where, size, val);
+ return PCIBIOS_SUCCESSFUL;
+ }
+ if (where_a == 0xc0) {
+ if (is_rsl_bridge || is_nic_bridge)
+ v = 0x0101; /* subordinate:secondary = 1:1 */
+ else if (is_rad_bridge)
+ v = 0x0202; /* subordinate:secondary = 2:2 */
+ else if (is_zip_bridge)
+ v = 0x0303; /* subordinate:secondary = 3:3 */
+ else if (is_dfa_bridge)
+ v = 0x0404; /* subordinate:secondary = 4:4 */
+ set_val(v, where, size, val);
+ return PCIBIOS_SUCCESSFUL;
+ }
+ if (where_a == 0xc4 && is_nic_bridge) {
+ /* Enabled, not-Write, SP=ff, PP=05, BEI=6, ES=4 */
+ v = 0x80ff0564;
+ set_val(v, where, size, val);
+ return PCIBIOS_SUCCESSFUL;
+ }
+ if (where_a == 0xc8 && is_nic_bridge) {
+ v = 0x00000002; /* Base-L 64-bit */
+ set_val(v, where, size, val);
+ return PCIBIOS_SUCCESSFUL;
+ }
+ if (where_a == 0xcc && is_nic_bridge) {
+ v = 0xfffffffe; /* MaxOffset-L 64-bit */
+ set_val(v, where, size, val);
+ return PCIBIOS_SUCCESSFUL;
+ }
+ if (where_a == 0xd0 && is_nic_bridge) {
+ v = 0x00008430; /* NIC Base-H */
+ set_val(v, where, size, val);
+ return PCIBIOS_SUCCESSFUL;
+ }
+ if (where_a == 0xd4 && is_nic_bridge) {
+ v = 0x0000000f; /* MaxOffset-H */
+ set_val(v, where, size, val);
+ return PCIBIOS_SUCCESSFUL;
+ }
+ }
+no_emulation:
+ return pci_generic_config_read(bus, devfn, where, size, val);
+}
+
+static int thunder_ecam_config_write(struct pci_bus *bus, unsigned int devfn,
+ int where, int size, u32 val)
+{
+ /*
+ * All BARs have fixed addresses; ignore BAR writes so they
+ * don't get corrupted.
+ */
+ if ((where >= 0x10 && where < 0x2c) ||
+ (where >= 0x1a4 && where < 0x1bc))
+ /* BAR or SR-IOV BAR */
+ return PCIBIOS_SUCCESSFUL;
+
+ return pci_generic_config_write(bus, devfn, where, size, val);
+}
+
+struct pci_ecam_ops pci_thunder_ecam_ops = {
+ .bus_shift = 20,
+ .pci_ops = {
+ .map_bus = pci_ecam_map_bus,
+ .read = thunder_ecam_config_read,
+ .write = thunder_ecam_config_write,
+ }
+};
+
+#ifdef CONFIG_PCI_HOST_THUNDER_ECAM
+
+static const struct of_device_id thunder_ecam_of_match[] = {
+ { .compatible = "cavium,pci-host-thunder-ecam" },
+ { },
+};
+
+static int thunder_ecam_probe(struct platform_device *pdev)
+{
+ return pci_host_common_probe(pdev, &pci_thunder_ecam_ops);
+}
+
+static struct platform_driver thunder_ecam_driver = {
+ .driver = {
+ .name = KBUILD_MODNAME,
+ .of_match_table = thunder_ecam_of_match,
+ .suppress_bind_attrs = true,
+ },
+ .probe = thunder_ecam_probe,
+};
+builtin_platform_driver(thunder_ecam_driver);
+
+#endif
+#endif
diff --git a/src/kernel/linux/v4.14/drivers/pci/host/pci-thunder-pem.c b/src/kernel/linux/v4.14/drivers/pci/host/pci-thunder-pem.c
new file mode 100644
index 0000000..6e066f8
--- /dev/null
+++ b/src/kernel/linux/v4.14/drivers/pci/host/pci-thunder-pem.c
@@ -0,0 +1,484 @@
+/*
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ * Copyright (C) 2015 - 2016 Cavium, Inc.
+ */
+
+#include <linux/bitfield.h>
+#include <linux/kernel.h>
+#include <linux/init.h>
+#include <linux/of_address.h>
+#include <linux/of_pci.h>
+#include <linux/pci-acpi.h>
+#include <linux/pci-ecam.h>
+#include <linux/platform_device.h>
+#include "../pci.h"
+
+#if defined(CONFIG_PCI_HOST_THUNDER_PEM) || (defined(CONFIG_ACPI) && defined(CONFIG_PCI_QUIRKS))
+
+#define PEM_CFG_WR 0x28
+#define PEM_CFG_RD 0x30
+
+struct thunder_pem_pci {
+ u32 ea_entry[3];
+ void __iomem *pem_reg_base;
+};
+
+static int thunder_pem_bridge_read(struct pci_bus *bus, unsigned int devfn,
+ int where, int size, u32 *val)
+{
+ u64 read_val, tmp_val;
+ struct pci_config_window *cfg = bus->sysdata;
+ struct thunder_pem_pci *pem_pci = (struct thunder_pem_pci *)cfg->priv;
+
+ if (devfn != 0 || where >= 2048) {
+ *val = ~0;
+ return PCIBIOS_DEVICE_NOT_FOUND;
+ }
+
+ /*
+ * 32-bit accesses only. Write the address to the low order
+ * bits of PEM_CFG_RD, then trigger the read by reading back.
+ * The config data lands in the upper 32-bits of PEM_CFG_RD.
+ */
+ read_val = where & ~3ull;
+ writeq(read_val, pem_pci->pem_reg_base + PEM_CFG_RD);
+ read_val = readq(pem_pci->pem_reg_base + PEM_CFG_RD);
+ read_val >>= 32;
+
+ /*
+ * The config space contains some garbage, fix it up. Also
+ * synthesize an EA capability for the BAR used by MSI-X.
+ */
+ switch (where & ~3) {
+ case 0x40:
+ read_val &= 0xffff00ff;
+ read_val |= 0x00007000; /* Skip MSI CAP */
+ break;
+ case 0x70: /* Express Cap */
+ /*
+ * Change PME interrupt to vector 2 on T88 where it
+ * reads as 0, else leave it alone.
+ */
+ if (!(read_val & (0x1f << 25)))
+ read_val |= (2u << 25);
+ break;
+ case 0xb0: /* MSI-X Cap */
+ /* TableSize=2 or 4, Next Cap is EA */
+ read_val &= 0xc00000ff;
+ /*
+ * If Express Cap(0x70) raw PME vector reads as 0 we are on
+ * T88 and TableSize is reported as 4, else TableSize
+ * is 2.
+ */
+ writeq(0x70, pem_pci->pem_reg_base + PEM_CFG_RD);
+ tmp_val = readq(pem_pci->pem_reg_base + PEM_CFG_RD);
+ tmp_val >>= 32;
+ if (!(tmp_val & (0x1f << 25)))
+ read_val |= 0x0003bc00;
+ else
+ read_val |= 0x0001bc00;
+ break;
+ case 0xb4:
+ /* Table offset=0, BIR=0 */
+ read_val = 0x00000000;
+ break;
+ case 0xb8:
+ /* BPA offset=0xf0000, BIR=0 */
+ read_val = 0x000f0000;
+ break;
+ case 0xbc:
+ /* EA, 1 entry, no next Cap */
+ read_val = 0x00010014;
+ break;
+ case 0xc0:
+ /* DW2 for type-1 */
+ read_val = 0x00000000;
+ break;
+ case 0xc4:
+ /* Entry BEI=0, PP=0x00, SP=0xff, ES=3 */
+ read_val = 0x80ff0003;
+ break;
+ case 0xc8:
+ read_val = pem_pci->ea_entry[0];
+ break;
+ case 0xcc:
+ read_val = pem_pci->ea_entry[1];
+ break;
+ case 0xd0:
+ read_val = pem_pci->ea_entry[2];
+ break;
+ default:
+ break;
+ }
+ read_val >>= (8 * (where & 3));
+ switch (size) {
+ case 1:
+ read_val &= 0xff;
+ break;
+ case 2:
+ read_val &= 0xffff;
+ break;
+ default:
+ break;
+ }
+ *val = read_val;
+ return PCIBIOS_SUCCESSFUL;
+}
+
+static int thunder_pem_config_read(struct pci_bus *bus, unsigned int devfn,
+ int where, int size, u32 *val)
+{
+ struct pci_config_window *cfg = bus->sysdata;
+
+ if (bus->number < cfg->busr.start ||
+ bus->number > cfg->busr.end)
+ return PCIBIOS_DEVICE_NOT_FOUND;
+
+ /*
+ * The first device on the bus is the PEM PCIe bridge.
+ * Special case its config access.
+ */
+ if (bus->number == cfg->busr.start)
+ return thunder_pem_bridge_read(bus, devfn, where, size, val);
+
+ return pci_generic_config_read(bus, devfn, where, size, val);
+}
+
+/*
+ * Some of the w1c_bits below also include read-only or non-writable
+ * reserved bits, this makes the code simpler and is OK as the bits
+ * are not affected by writing zeros to them.
+ */
+static u32 thunder_pem_bridge_w1c_bits(u64 where_aligned)
+{
+ u32 w1c_bits = 0;
+
+ switch (where_aligned) {
+ case 0x04: /* Command/Status */
+ case 0x1c: /* Base and I/O Limit/Secondary Status */
+ w1c_bits = 0xff000000;
+ break;
+ case 0x44: /* Power Management Control and Status */
+ w1c_bits = 0xfffffe00;
+ break;
+ case 0x78: /* Device Control/Device Status */
+ case 0x80: /* Link Control/Link Status */
+ case 0x88: /* Slot Control/Slot Status */
+ case 0x90: /* Root Status */
+ case 0xa0: /* Link Control 2 Registers/Link Status 2 */
+ w1c_bits = 0xffff0000;
+ break;
+ case 0x104: /* Uncorrectable Error Status */
+ case 0x110: /* Correctable Error Status */
+ case 0x130: /* Error Status */
+ case 0x160: /* Link Control 4 */
+ w1c_bits = 0xffffffff;
+ break;
+ default:
+ break;
+ }
+ return w1c_bits;
+}
+
+/* Some bits must be written to one so they appear to be read-only. */
+static u32 thunder_pem_bridge_w1_bits(u64 where_aligned)
+{
+ u32 w1_bits;
+
+ switch (where_aligned) {
+ case 0x1c: /* I/O Base / I/O Limit, Secondary Status */
+ /* Force 32-bit I/O addressing. */
+ w1_bits = 0x0101;
+ break;
+ case 0x24: /* Prefetchable Memory Base / Prefetchable Memory Limit */
+ /* Force 64-bit addressing */
+ w1_bits = 0x00010001;
+ break;
+ default:
+ w1_bits = 0;
+ break;
+ }
+ return w1_bits;
+}
+
+static int thunder_pem_bridge_write(struct pci_bus *bus, unsigned int devfn,
+ int where, int size, u32 val)
+{
+ struct pci_config_window *cfg = bus->sysdata;
+ struct thunder_pem_pci *pem_pci = (struct thunder_pem_pci *)cfg->priv;
+ u64 write_val, read_val;
+ u64 where_aligned = where & ~3ull;
+ u32 mask = 0;
+
+
+ if (devfn != 0 || where >= 2048)
+ return PCIBIOS_DEVICE_NOT_FOUND;
+
+ /*
+ * 32-bit accesses only. If the write is for a size smaller
+ * than 32-bits, we must first read the 32-bit value and merge
+ * in the desired bits and then write the whole 32-bits back
+ * out.
+ */
+ switch (size) {
+ case 1:
+ writeq(where_aligned, pem_pci->pem_reg_base + PEM_CFG_RD);
+ read_val = readq(pem_pci->pem_reg_base + PEM_CFG_RD);
+ read_val >>= 32;
+ mask = ~(0xff << (8 * (where & 3)));
+ read_val &= mask;
+ val = (val & 0xff) << (8 * (where & 3));
+ val |= (u32)read_val;
+ break;
+ case 2:
+ writeq(where_aligned, pem_pci->pem_reg_base + PEM_CFG_RD);
+ read_val = readq(pem_pci->pem_reg_base + PEM_CFG_RD);
+ read_val >>= 32;
+ mask = ~(0xffff << (8 * (where & 3)));
+ read_val &= mask;
+ val = (val & 0xffff) << (8 * (where & 3));
+ val |= (u32)read_val;
+ break;
+ default:
+ break;
+ }
+
+ /*
+ * By expanding the write width to 32 bits, we may
+ * inadvertently hit some W1C bits that were not intended to
+ * be written. Calculate the mask that must be applied to the
+ * data to be written to avoid these cases.
+ */
+ if (mask) {
+ u32 w1c_bits = thunder_pem_bridge_w1c_bits(where);
+
+ if (w1c_bits) {
+ mask &= w1c_bits;
+ val &= ~mask;
+ }
+ }
+
+ /*
+ * Some bits must be read-only with value of one. Since the
+ * access method allows these to be cleared if a zero is
+ * written, force them to one before writing.
+ */
+ val |= thunder_pem_bridge_w1_bits(where_aligned);
+
+ /*
+ * Low order bits are the config address, the high order 32
+ * bits are the data to be written.
+ */
+ write_val = (((u64)val) << 32) | where_aligned;
+ writeq(write_val, pem_pci->pem_reg_base + PEM_CFG_WR);
+ return PCIBIOS_SUCCESSFUL;
+}
+
+static int thunder_pem_config_write(struct pci_bus *bus, unsigned int devfn,
+ int where, int size, u32 val)
+{
+ struct pci_config_window *cfg = bus->sysdata;
+
+ if (bus->number < cfg->busr.start ||
+ bus->number > cfg->busr.end)
+ return PCIBIOS_DEVICE_NOT_FOUND;
+ /*
+ * The first device on the bus is the PEM PCIe bridge.
+ * Special case its config access.
+ */
+ if (bus->number == cfg->busr.start)
+ return thunder_pem_bridge_write(bus, devfn, where, size, val);
+
+
+ return pci_generic_config_write(bus, devfn, where, size, val);
+}
+
+static int thunder_pem_init(struct device *dev, struct pci_config_window *cfg,
+ struct resource *res_pem)
+{
+ struct thunder_pem_pci *pem_pci;
+ resource_size_t bar4_start;
+
+ pem_pci = devm_kzalloc(dev, sizeof(*pem_pci), GFP_KERNEL);
+ if (!pem_pci)
+ return -ENOMEM;
+
+ pem_pci->pem_reg_base = devm_ioremap(dev, res_pem->start, 0x10000);
+ if (!pem_pci->pem_reg_base)
+ return -ENOMEM;
+
+ /*
+ * The MSI-X BAR for the PEM and AER interrupts is located at
+ * a fixed offset from the PEM register base. Generate a
+ * fragment of the synthesized Enhanced Allocation capability
+ * structure here for the BAR.
+ */
+ bar4_start = res_pem->start + 0xf00000;
+ pem_pci->ea_entry[0] = (u32)bar4_start | 2;
+ pem_pci->ea_entry[1] = (u32)(res_pem->end - bar4_start) & ~3u;
+ pem_pci->ea_entry[2] = (u32)(bar4_start >> 32);
+
+ cfg->priv = pem_pci;
+ return 0;
+}
+
+#if defined(CONFIG_ACPI) && defined(CONFIG_PCI_QUIRKS)
+
+#define PEM_RES_BASE 0x87e0c0000000UL
+#define PEM_NODE_MASK GENMASK(45, 44)
+#define PEM_INDX_MASK GENMASK(26, 24)
+#define PEM_MIN_DOM_IN_NODE 4
+#define PEM_MAX_DOM_IN_NODE 10
+
+static void thunder_pem_reserve_range(struct device *dev, int seg,
+ struct resource *r)
+{
+ resource_size_t start = r->start, end = r->end;
+ struct resource *res;
+ const char *regionid;
+
+ regionid = kasprintf(GFP_KERNEL, "PEM RC:%d", seg);
+ if (!regionid)
+ return;
+
+ res = request_mem_region(start, end - start + 1, regionid);
+ if (res)
+ res->flags &= ~IORESOURCE_BUSY;
+ else
+ kfree(regionid);
+
+ dev_info(dev, "%pR %s reserved\n", r,
+ res ? "has been" : "could not be");
+}
+
+static void thunder_pem_legacy_fw(struct acpi_pci_root *root,
+ struct resource *res_pem)
+{
+ int node = acpi_get_node(root->device->handle);
+ int index;
+
+ if (node == NUMA_NO_NODE)
+ node = 0;
+
+ index = root->segment - PEM_MIN_DOM_IN_NODE;
+ index -= node * PEM_MAX_DOM_IN_NODE;
+ res_pem->start = PEM_RES_BASE | FIELD_PREP(PEM_NODE_MASK, node) |
+ FIELD_PREP(PEM_INDX_MASK, index);
+ res_pem->flags = IORESOURCE_MEM;
+}
+
+static int thunder_pem_acpi_init(struct pci_config_window *cfg)
+{
+ struct device *dev = cfg->parent;
+ struct acpi_device *adev = to_acpi_device(dev);
+ struct acpi_pci_root *root = acpi_driver_data(adev);
+ struct resource *res_pem;
+ int ret;
+
+ res_pem = devm_kzalloc(&adev->dev, sizeof(*res_pem), GFP_KERNEL);
+ if (!res_pem)
+ return -ENOMEM;
+
+ ret = acpi_get_rc_resources(dev, "CAVA02B", root->segment, res_pem);
+
+ /*
+ * If we fail to gather resources it means that we run with old
+ * FW where we need to calculate PEM-specific resources manually.
+ */
+ if (ret) {
+ thunder_pem_legacy_fw(root, res_pem);
+ /*
+ * Reserve 64K size PEM specific resources. The full 16M range
+ * size is required for thunder_pem_init() call.
+ */
+ res_pem->end = res_pem->start + SZ_64K - 1;
+ thunder_pem_reserve_range(dev, root->segment, res_pem);
+ res_pem->end = res_pem->start + SZ_16M - 1;
+
+ /* Reserve PCI configuration space as well. */
+ thunder_pem_reserve_range(dev, root->segment, &cfg->res);
+ }
+
+ return thunder_pem_init(dev, cfg, res_pem);
+}
+
+struct pci_ecam_ops thunder_pem_ecam_ops = {
+ .bus_shift = 24,
+ .init = thunder_pem_acpi_init,
+ .pci_ops = {
+ .map_bus = pci_ecam_map_bus,
+ .read = thunder_pem_config_read,
+ .write = thunder_pem_config_write,
+ }
+};
+
+#endif
+
+#ifdef CONFIG_PCI_HOST_THUNDER_PEM
+
+static int thunder_pem_platform_init(struct pci_config_window *cfg)
+{
+ struct device *dev = cfg->parent;
+ struct platform_device *pdev = to_platform_device(dev);
+ struct resource *res_pem;
+
+ if (!dev->of_node)
+ return -EINVAL;
+
+ /*
+ * The second register range is the PEM bridge to the PCIe
+ * bus. It has a different config access method than those
+ * devices behind the bridge.
+ */
+ res_pem = platform_get_resource(pdev, IORESOURCE_MEM, 1);
+ if (!res_pem) {
+ dev_err(dev, "missing \"reg[1]\"property\n");
+ return -EINVAL;
+ }
+
+ return thunder_pem_init(dev, cfg, res_pem);
+}
+
+static struct pci_ecam_ops pci_thunder_pem_ops = {
+ .bus_shift = 24,
+ .init = thunder_pem_platform_init,
+ .pci_ops = {
+ .map_bus = pci_ecam_map_bus,
+ .read = thunder_pem_config_read,
+ .write = thunder_pem_config_write,
+ }
+};
+
+static const struct of_device_id thunder_pem_of_match[] = {
+ { .compatible = "cavium,pci-host-thunder-pem" },
+ { },
+};
+
+static int thunder_pem_probe(struct platform_device *pdev)
+{
+ return pci_host_common_probe(pdev, &pci_thunder_pem_ops);
+}
+
+static struct platform_driver thunder_pem_driver = {
+ .driver = {
+ .name = KBUILD_MODNAME,
+ .of_match_table = thunder_pem_of_match,
+ .suppress_bind_attrs = true,
+ },
+ .probe = thunder_pem_probe,
+};
+builtin_platform_driver(thunder_pem_driver);
+
+#endif
+#endif
diff --git a/src/kernel/linux/v4.14/drivers/pci/host/pci-versatile.c b/src/kernel/linux/v4.14/drivers/pci/host/pci-versatile.c
new file mode 100644
index 0000000..aff4cfb
--- /dev/null
+++ b/src/kernel/linux/v4.14/drivers/pci/host/pci-versatile.c
@@ -0,0 +1,246 @@
+/*
+ * Copyright 2004 Koninklijke Philips Electronics NV
+ *
+ * Conversion to platform driver and DT:
+ * Copyright 2014 Linaro Ltd.
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * 14/04/2005 Initial version, colin.king@philips.com
+ */
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of_address.h>
+#include <linux/of_pci.h>
+#include <linux/of_platform.h>
+#include <linux/pci.h>
+#include <linux/platform_device.h>
+
+static void __iomem *versatile_pci_base;
+static void __iomem *versatile_cfg_base[2];
+
+#define PCI_IMAP(m) (versatile_pci_base + ((m) * 4))
+#define PCI_SMAP(m) (versatile_pci_base + 0x14 + ((m) * 4))
+#define PCI_SELFID (versatile_pci_base + 0xc)
+
+#define VP_PCI_DEVICE_ID 0x030010ee
+#define VP_PCI_CLASS_ID 0x0b400000
+
+static u32 pci_slot_ignore;
+
+static int __init versatile_pci_slot_ignore(char *str)
+{
+ int retval;
+ int slot;
+
+ while ((retval = get_option(&str, &slot))) {
+ if ((slot < 0) || (slot > 31))
+ pr_err("Illegal slot value: %d\n", slot);
+ else
+ pci_slot_ignore |= (1 << slot);
+ }
+ return 1;
+}
+__setup("pci_slot_ignore=", versatile_pci_slot_ignore);
+
+
+static void __iomem *versatile_map_bus(struct pci_bus *bus,
+ unsigned int devfn, int offset)
+{
+ unsigned int busnr = bus->number;
+
+ if (pci_slot_ignore & (1 << PCI_SLOT(devfn)))
+ return NULL;
+
+ return versatile_cfg_base[1] + ((busnr << 16) | (devfn << 8) | offset);
+}
+
+static struct pci_ops pci_versatile_ops = {
+ .map_bus = versatile_map_bus,
+ .read = pci_generic_config_read32,
+ .write = pci_generic_config_write,
+};
+
+static int versatile_pci_parse_request_of_pci_ranges(struct device *dev,
+ struct list_head *res)
+{
+ int err, mem = 1, res_valid = 0;
+ struct device_node *np = dev->of_node;
+ resource_size_t iobase;
+ struct resource_entry *win, *tmp;
+
+ err = of_pci_get_host_bridge_resources(np, 0, 0xff, res, &iobase);
+ if (err)
+ return err;
+
+ err = devm_request_pci_bus_resources(dev, res);
+ if (err)
+ goto out_release_res;
+
+ resource_list_for_each_entry_safe(win, tmp, res) {
+ struct resource *res = win->res;
+
+ switch (resource_type(res)) {
+ case IORESOURCE_IO:
+ err = devm_pci_remap_iospace(dev, res, iobase);
+ if (err) {
+ dev_warn(dev, "error %d: failed to map resource %pR\n",
+ err, res);
+ resource_list_destroy_entry(win);
+ }
+ break;
+ case IORESOURCE_MEM:
+ res_valid |= !(res->flags & IORESOURCE_PREFETCH);
+
+ writel(res->start >> 28, PCI_IMAP(mem));
+ writel(PHYS_OFFSET >> 28, PCI_SMAP(mem));
+ mem++;
+
+ break;
+ }
+ }
+
+ if (res_valid)
+ return 0;
+
+ dev_err(dev, "non-prefetchable memory resource required\n");
+ err = -EINVAL;
+
+out_release_res:
+ pci_free_resource_list(res);
+ return err;
+}
+
+static int versatile_pci_probe(struct platform_device *pdev)
+{
+ struct device *dev = &pdev->dev;
+ struct resource *res;
+ int ret, i, myslot = -1;
+ u32 val;
+ void __iomem *local_pci_cfg_base;
+ struct pci_bus *bus, *child;
+ struct pci_host_bridge *bridge;
+ LIST_HEAD(pci_res);
+
+ bridge = devm_pci_alloc_host_bridge(dev, 0);
+ if (!bridge)
+ return -ENOMEM;
+
+ res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ versatile_pci_base = devm_ioremap_resource(dev, res);
+ if (IS_ERR(versatile_pci_base))
+ return PTR_ERR(versatile_pci_base);
+
+ res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
+ versatile_cfg_base[0] = devm_ioremap_resource(dev, res);
+ if (IS_ERR(versatile_cfg_base[0]))
+ return PTR_ERR(versatile_cfg_base[0]);
+
+ res = platform_get_resource(pdev, IORESOURCE_MEM, 2);
+ versatile_cfg_base[1] = devm_pci_remap_cfg_resource(dev, res);
+ if (IS_ERR(versatile_cfg_base[1]))
+ return PTR_ERR(versatile_cfg_base[1]);
+
+ ret = versatile_pci_parse_request_of_pci_ranges(dev, &pci_res);
+ if (ret)
+ return ret;
+
+ /*
+ * We need to discover the PCI core first to configure itself
+ * before the main PCI probing is performed
+ */
+ for (i = 0; i < 32; i++) {
+ if ((readl(versatile_cfg_base[0] + (i << 11) + PCI_VENDOR_ID) == VP_PCI_DEVICE_ID) &&
+ (readl(versatile_cfg_base[0] + (i << 11) + PCI_CLASS_REVISION) == VP_PCI_CLASS_ID)) {
+ myslot = i;
+ break;
+ }
+ }
+ if (myslot == -1) {
+ dev_err(dev, "Cannot find PCI core!\n");
+ return -EIO;
+ }
+ /*
+ * Do not to map Versatile FPGA PCI device into memory space
+ */
+ pci_slot_ignore |= (1 << myslot);
+
+ dev_info(dev, "PCI core found (slot %d)\n", myslot);
+
+ writel(myslot, PCI_SELFID);
+ local_pci_cfg_base = versatile_cfg_base[1] + (myslot << 11);
+
+ val = readl(local_pci_cfg_base + PCI_COMMAND);
+ val |= PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER | PCI_COMMAND_INVALIDATE;
+ writel(val, local_pci_cfg_base + PCI_COMMAND);
+
+ /*
+ * Configure the PCI inbound memory windows to be 1:1 mapped to SDRAM
+ */
+ writel(PHYS_OFFSET, local_pci_cfg_base + PCI_BASE_ADDRESS_0);
+ writel(PHYS_OFFSET, local_pci_cfg_base + PCI_BASE_ADDRESS_1);
+ writel(PHYS_OFFSET, local_pci_cfg_base + PCI_BASE_ADDRESS_2);
+
+ /*
+ * For many years the kernel and QEMU were symbiotically buggy
+ * in that they both assumed the same broken IRQ mapping.
+ * QEMU therefore attempts to auto-detect old broken kernels
+ * so that they still work on newer QEMU as they did on old
+ * QEMU. Since we now use the correct (ie matching-hardware)
+ * IRQ mapping we write a definitely different value to a
+ * PCI_INTERRUPT_LINE register to tell QEMU that we expect
+ * real hardware behaviour and it need not be backwards
+ * compatible for us. This write is harmless on real hardware.
+ */
+ writel(0, versatile_cfg_base[0] + PCI_INTERRUPT_LINE);
+
+ pci_add_flags(PCI_ENABLE_PROC_DOMAINS);
+ pci_add_flags(PCI_REASSIGN_ALL_BUS | PCI_REASSIGN_ALL_RSRC);
+
+ list_splice_init(&pci_res, &bridge->windows);
+ bridge->dev.parent = dev;
+ bridge->sysdata = NULL;
+ bridge->busnr = 0;
+ bridge->ops = &pci_versatile_ops;
+ bridge->map_irq = of_irq_parse_and_map_pci;
+ bridge->swizzle_irq = pci_common_swizzle;
+
+ ret = pci_scan_root_bus_bridge(bridge);
+ if (ret < 0)
+ return ret;
+
+ bus = bridge->bus;
+
+ pci_assign_unassigned_bus_resources(bus);
+ list_for_each_entry(child, &bus->children, node)
+ pcie_bus_configure_settings(child);
+ pci_bus_add_devices(bus);
+
+ return 0;
+}
+
+static const struct of_device_id versatile_pci_of_match[] = {
+ { .compatible = "arm,versatile-pci", },
+ { },
+};
+MODULE_DEVICE_TABLE(of, versatile_pci_of_match);
+
+static struct platform_driver versatile_pci_driver = {
+ .driver = {
+ .name = "versatile-pci",
+ .of_match_table = versatile_pci_of_match,
+ .suppress_bind_attrs = true,
+ },
+ .probe = versatile_pci_probe,
+};
+module_platform_driver(versatile_pci_driver);
+
+MODULE_DESCRIPTION("Versatile PCI driver");
+MODULE_LICENSE("GPL v2");
diff --git a/src/kernel/linux/v4.14/drivers/pci/host/pci-xgene-msi.c b/src/kernel/linux/v4.14/drivers/pci/host/pci-xgene-msi.c
new file mode 100644
index 0000000..1f42a20
--- /dev/null
+++ b/src/kernel/linux/v4.14/drivers/pci/host/pci-xgene-msi.c
@@ -0,0 +1,552 @@
+/*
+ * APM X-Gene MSI Driver
+ *
+ * Copyright (c) 2014, Applied Micro Circuits Corporation
+ * Author: Tanmay Inamdar <tinamdar@apm.com>
+ * Duc Dang <dhdang@apm.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2 of the License, or (at your
+ * option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+#include <linux/cpu.h>
+#include <linux/interrupt.h>
+#include <linux/module.h>
+#include <linux/msi.h>
+#include <linux/of_irq.h>
+#include <linux/irqchip/chained_irq.h>
+#include <linux/pci.h>
+#include <linux/platform_device.h>
+#include <linux/of_pci.h>
+
+#define MSI_IR0 0x000000
+#define MSI_INT0 0x800000
+#define IDX_PER_GROUP 8
+#define IRQS_PER_IDX 16
+#define NR_HW_IRQS 16
+#define NR_MSI_VEC (IDX_PER_GROUP * IRQS_PER_IDX * NR_HW_IRQS)
+
+struct xgene_msi_group {
+ struct xgene_msi *msi;
+ int gic_irq;
+ u32 msi_grp;
+};
+
+struct xgene_msi {
+ struct device_node *node;
+ struct irq_domain *inner_domain;
+ struct irq_domain *msi_domain;
+ u64 msi_addr;
+ void __iomem *msi_regs;
+ unsigned long *bitmap;
+ struct mutex bitmap_lock;
+ struct xgene_msi_group *msi_groups;
+ int num_cpus;
+};
+
+/* Global data */
+static struct xgene_msi xgene_msi_ctrl;
+
+static struct irq_chip xgene_msi_top_irq_chip = {
+ .name = "X-Gene1 MSI",
+ .irq_enable = pci_msi_unmask_irq,
+ .irq_disable = pci_msi_mask_irq,
+ .irq_mask = pci_msi_mask_irq,
+ .irq_unmask = pci_msi_unmask_irq,
+};
+
+static struct msi_domain_info xgene_msi_domain_info = {
+ .flags = (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS |
+ MSI_FLAG_PCI_MSIX),
+ .chip = &xgene_msi_top_irq_chip,
+};
+
+/*
+ * X-Gene v1 has 16 groups of MSI termination registers MSInIRx, where
+ * n is group number (0..F), x is index of registers in each group (0..7)
+ * The register layout is as follows:
+ * MSI0IR0 base_addr
+ * MSI0IR1 base_addr + 0x10000
+ * ... ...
+ * MSI0IR6 base_addr + 0x60000
+ * MSI0IR7 base_addr + 0x70000
+ * MSI1IR0 base_addr + 0x80000
+ * MSI1IR1 base_addr + 0x90000
+ * ... ...
+ * MSI1IR7 base_addr + 0xF0000
+ * MSI2IR0 base_addr + 0x100000
+ * ... ...
+ * MSIFIR0 base_addr + 0x780000
+ * MSIFIR1 base_addr + 0x790000
+ * ... ...
+ * MSIFIR7 base_addr + 0x7F0000
+ * MSIINT0 base_addr + 0x800000
+ * MSIINT1 base_addr + 0x810000
+ * ... ...
+ * MSIINTF base_addr + 0x8F0000
+ *
+ * Each index register supports 16 MSI vectors (0..15) to generate interrupt.
+ * There are total 16 GIC IRQs assigned for these 16 groups of MSI termination
+ * registers.
+ *
+ * Each MSI termination group has 1 MSIINTn register (n is 0..15) to indicate
+ * the MSI pending status caused by 1 of its 8 index registers.
+ */
+
+/* MSInIRx read helper */
+static u32 xgene_msi_ir_read(struct xgene_msi *msi,
+ u32 msi_grp, u32 msir_idx)
+{
+ return readl_relaxed(msi->msi_regs + MSI_IR0 +
+ (msi_grp << 19) + (msir_idx << 16));
+}
+
+/* MSIINTn read helper */
+static u32 xgene_msi_int_read(struct xgene_msi *msi, u32 msi_grp)
+{
+ return readl_relaxed(msi->msi_regs + MSI_INT0 + (msi_grp << 16));
+}
+
+/*
+ * With 2048 MSI vectors supported, the MSI message can be constructed using
+ * following scheme:
+ * - Divide into 8 256-vector groups
+ * Group 0: 0-255
+ * Group 1: 256-511
+ * Group 2: 512-767
+ * ...
+ * Group 7: 1792-2047
+ * - Each 256-vector group is divided into 16 16-vector groups
+ * As an example: 16 16-vector groups for 256-vector group 0-255 is
+ * Group 0: 0-15
+ * Group 1: 16-32
+ * ...
+ * Group 15: 240-255
+ * - The termination address of MSI vector in 256-vector group n and 16-vector
+ * group x is the address of MSIxIRn
+ * - The data for MSI vector in 16-vector group x is x
+ */
+static u32 hwirq_to_reg_set(unsigned long hwirq)
+{
+ return (hwirq / (NR_HW_IRQS * IRQS_PER_IDX));
+}
+
+static u32 hwirq_to_group(unsigned long hwirq)
+{
+ return (hwirq % NR_HW_IRQS);
+}
+
+static u32 hwirq_to_msi_data(unsigned long hwirq)
+{
+ return ((hwirq / NR_HW_IRQS) % IRQS_PER_IDX);
+}
+
+static void xgene_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
+{
+ struct xgene_msi *msi = irq_data_get_irq_chip_data(data);
+ u32 reg_set = hwirq_to_reg_set(data->hwirq);
+ u32 group = hwirq_to_group(data->hwirq);
+ u64 target_addr = msi->msi_addr + (((8 * group) + reg_set) << 16);
+
+ msg->address_hi = upper_32_bits(target_addr);
+ msg->address_lo = lower_32_bits(target_addr);
+ msg->data = hwirq_to_msi_data(data->hwirq);
+}
+
+/*
+ * X-Gene v1 only has 16 MSI GIC IRQs for 2048 MSI vectors. To maintain
+ * the expected behaviour of .set_affinity for each MSI interrupt, the 16
+ * MSI GIC IRQs are statically allocated to 8 X-Gene v1 cores (2 GIC IRQs
+ * for each core). The MSI vector is moved fom 1 MSI GIC IRQ to another
+ * MSI GIC IRQ to steer its MSI interrupt to correct X-Gene v1 core. As a
+ * consequence, the total MSI vectors that X-Gene v1 supports will be
+ * reduced to 256 (2048/8) vectors.
+ */
+static int hwirq_to_cpu(unsigned long hwirq)
+{
+ return (hwirq % xgene_msi_ctrl.num_cpus);
+}
+
+static unsigned long hwirq_to_canonical_hwirq(unsigned long hwirq)
+{
+ return (hwirq - hwirq_to_cpu(hwirq));
+}
+
+static int xgene_msi_set_affinity(struct irq_data *irqdata,
+ const struct cpumask *mask, bool force)
+{
+ int target_cpu = cpumask_first(mask);
+ int curr_cpu;
+
+ curr_cpu = hwirq_to_cpu(irqdata->hwirq);
+ if (curr_cpu == target_cpu)
+ return IRQ_SET_MASK_OK_DONE;
+
+ /* Update MSI number to target the new CPU */
+ irqdata->hwirq = hwirq_to_canonical_hwirq(irqdata->hwirq) + target_cpu;
+
+ return IRQ_SET_MASK_OK;
+}
+
+static struct irq_chip xgene_msi_bottom_irq_chip = {
+ .name = "MSI",
+ .irq_set_affinity = xgene_msi_set_affinity,
+ .irq_compose_msi_msg = xgene_compose_msi_msg,
+};
+
+static int xgene_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
+ unsigned int nr_irqs, void *args)
+{
+ struct xgene_msi *msi = domain->host_data;
+ int msi_irq;
+
+ mutex_lock(&msi->bitmap_lock);
+
+ msi_irq = bitmap_find_next_zero_area(msi->bitmap, NR_MSI_VEC, 0,
+ msi->num_cpus, 0);
+ if (msi_irq < NR_MSI_VEC)
+ bitmap_set(msi->bitmap, msi_irq, msi->num_cpus);
+ else
+ msi_irq = -ENOSPC;
+
+ mutex_unlock(&msi->bitmap_lock);
+
+ if (msi_irq < 0)
+ return msi_irq;
+
+ irq_domain_set_info(domain, virq, msi_irq,
+ &xgene_msi_bottom_irq_chip, domain->host_data,
+ handle_simple_irq, NULL, NULL);
+
+ return 0;
+}
+
+static void xgene_irq_domain_free(struct irq_domain *domain,
+ unsigned int virq, unsigned int nr_irqs)
+{
+ struct irq_data *d = irq_domain_get_irq_data(domain, virq);
+ struct xgene_msi *msi = irq_data_get_irq_chip_data(d);
+ u32 hwirq;
+
+ mutex_lock(&msi->bitmap_lock);
+
+ hwirq = hwirq_to_canonical_hwirq(d->hwirq);
+ bitmap_clear(msi->bitmap, hwirq, msi->num_cpus);
+
+ mutex_unlock(&msi->bitmap_lock);
+
+ irq_domain_free_irqs_parent(domain, virq, nr_irqs);
+}
+
+static const struct irq_domain_ops msi_domain_ops = {
+ .alloc = xgene_irq_domain_alloc,
+ .free = xgene_irq_domain_free,
+};
+
+static int xgene_allocate_domains(struct xgene_msi *msi)
+{
+ msi->inner_domain = irq_domain_add_linear(NULL, NR_MSI_VEC,
+ &msi_domain_ops, msi);
+ if (!msi->inner_domain)
+ return -ENOMEM;
+
+ msi->msi_domain = pci_msi_create_irq_domain(of_node_to_fwnode(msi->node),
+ &xgene_msi_domain_info,
+ msi->inner_domain);
+
+ if (!msi->msi_domain) {
+ irq_domain_remove(msi->inner_domain);
+ return -ENOMEM;
+ }
+
+ return 0;
+}
+
+static void xgene_free_domains(struct xgene_msi *msi)
+{
+ if (msi->msi_domain)
+ irq_domain_remove(msi->msi_domain);
+ if (msi->inner_domain)
+ irq_domain_remove(msi->inner_domain);
+}
+
+static int xgene_msi_init_allocator(struct xgene_msi *xgene_msi)
+{
+ int size = BITS_TO_LONGS(NR_MSI_VEC) * sizeof(long);
+
+ xgene_msi->bitmap = kzalloc(size, GFP_KERNEL);
+ if (!xgene_msi->bitmap)
+ return -ENOMEM;
+
+ mutex_init(&xgene_msi->bitmap_lock);
+
+ xgene_msi->msi_groups = kcalloc(NR_HW_IRQS,
+ sizeof(struct xgene_msi_group),
+ GFP_KERNEL);
+ if (!xgene_msi->msi_groups)
+ return -ENOMEM;
+
+ return 0;
+}
+
+static void xgene_msi_isr(struct irq_desc *desc)
+{
+ struct irq_chip *chip = irq_desc_get_chip(desc);
+ struct xgene_msi_group *msi_groups;
+ struct xgene_msi *xgene_msi;
+ unsigned int virq;
+ int msir_index, msir_val, hw_irq;
+ u32 intr_index, grp_select, msi_grp;
+
+ chained_irq_enter(chip, desc);
+
+ msi_groups = irq_desc_get_handler_data(desc);
+ xgene_msi = msi_groups->msi;
+ msi_grp = msi_groups->msi_grp;
+
+ /*
+ * MSIINTn (n is 0..F) indicates if there is a pending MSI interrupt
+ * If bit x of this register is set (x is 0..7), one or more interupts
+ * corresponding to MSInIRx is set.
+ */
+ grp_select = xgene_msi_int_read(xgene_msi, msi_grp);
+ while (grp_select) {
+ msir_index = ffs(grp_select) - 1;
+ /*
+ * Calculate MSInIRx address to read to check for interrupts
+ * (refer to termination address and data assignment
+ * described in xgene_compose_msi_msg() )
+ */
+ msir_val = xgene_msi_ir_read(xgene_msi, msi_grp, msir_index);
+ while (msir_val) {
+ intr_index = ffs(msir_val) - 1;
+ /*
+ * Calculate MSI vector number (refer to the termination
+ * address and data assignment described in
+ * xgene_compose_msi_msg function)
+ */
+ hw_irq = (((msir_index * IRQS_PER_IDX) + intr_index) *
+ NR_HW_IRQS) + msi_grp;
+ /*
+ * As we have multiple hw_irq that maps to single MSI,
+ * always look up the virq using the hw_irq as seen from
+ * CPU0
+ */
+ hw_irq = hwirq_to_canonical_hwirq(hw_irq);
+ virq = irq_find_mapping(xgene_msi->inner_domain, hw_irq);
+ WARN_ON(!virq);
+ if (virq != 0)
+ generic_handle_irq(virq);
+ msir_val &= ~(1 << intr_index);
+ }
+ grp_select &= ~(1 << msir_index);
+
+ if (!grp_select) {
+ /*
+ * We handled all interrupts happened in this group,
+ * resample this group MSI_INTx register in case
+ * something else has been made pending in the meantime
+ */
+ grp_select = xgene_msi_int_read(xgene_msi, msi_grp);
+ }
+ }
+
+ chained_irq_exit(chip, desc);
+}
+
+static enum cpuhp_state pci_xgene_online;
+
+static int xgene_msi_remove(struct platform_device *pdev)
+{
+ struct xgene_msi *msi = platform_get_drvdata(pdev);
+
+ if (pci_xgene_online)
+ cpuhp_remove_state(pci_xgene_online);
+ cpuhp_remove_state(CPUHP_PCI_XGENE_DEAD);
+
+ kfree(msi->msi_groups);
+
+ kfree(msi->bitmap);
+ msi->bitmap = NULL;
+
+ xgene_free_domains(msi);
+
+ return 0;
+}
+
+static int xgene_msi_hwirq_alloc(unsigned int cpu)
+{
+ struct xgene_msi *msi = &xgene_msi_ctrl;
+ struct xgene_msi_group *msi_group;
+ cpumask_var_t mask;
+ int i;
+ int err;
+
+ for (i = cpu; i < NR_HW_IRQS; i += msi->num_cpus) {
+ msi_group = &msi->msi_groups[i];
+ if (!msi_group->gic_irq)
+ continue;
+
+ irq_set_chained_handler(msi_group->gic_irq,
+ xgene_msi_isr);
+ err = irq_set_handler_data(msi_group->gic_irq, msi_group);
+ if (err) {
+ pr_err("failed to register GIC IRQ handler\n");
+ return -EINVAL;
+ }
+ /*
+ * Statically allocate MSI GIC IRQs to each CPU core.
+ * With 8-core X-Gene v1, 2 MSI GIC IRQs are allocated
+ * to each core.
+ */
+ if (alloc_cpumask_var(&mask, GFP_KERNEL)) {
+ cpumask_clear(mask);
+ cpumask_set_cpu(cpu, mask);
+ err = irq_set_affinity(msi_group->gic_irq, mask);
+ if (err)
+ pr_err("failed to set affinity for GIC IRQ");
+ free_cpumask_var(mask);
+ } else {
+ pr_err("failed to alloc CPU mask for affinity\n");
+ err = -EINVAL;
+ }
+
+ if (err) {
+ irq_set_chained_handler_and_data(msi_group->gic_irq,
+ NULL, NULL);
+ return err;
+ }
+ }
+
+ return 0;
+}
+
+static int xgene_msi_hwirq_free(unsigned int cpu)
+{
+ struct xgene_msi *msi = &xgene_msi_ctrl;
+ struct xgene_msi_group *msi_group;
+ int i;
+
+ for (i = cpu; i < NR_HW_IRQS; i += msi->num_cpus) {
+ msi_group = &msi->msi_groups[i];
+ if (!msi_group->gic_irq)
+ continue;
+
+ irq_set_chained_handler_and_data(msi_group->gic_irq, NULL,
+ NULL);
+ }
+ return 0;
+}
+
+static const struct of_device_id xgene_msi_match_table[] = {
+ {.compatible = "apm,xgene1-msi"},
+ {},
+};
+
+static int xgene_msi_probe(struct platform_device *pdev)
+{
+ struct resource *res;
+ int rc, irq_index;
+ struct xgene_msi *xgene_msi;
+ int virt_msir;
+ u32 msi_val, msi_idx;
+
+ xgene_msi = &xgene_msi_ctrl;
+
+ platform_set_drvdata(pdev, xgene_msi);
+
+ res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ xgene_msi->msi_regs = devm_ioremap_resource(&pdev->dev, res);
+ if (IS_ERR(xgene_msi->msi_regs)) {
+ dev_err(&pdev->dev, "no reg space\n");
+ rc = -EINVAL;
+ goto error;
+ }
+ xgene_msi->msi_addr = res->start;
+ xgene_msi->node = pdev->dev.of_node;
+ xgene_msi->num_cpus = num_possible_cpus();
+
+ rc = xgene_msi_init_allocator(xgene_msi);
+ if (rc) {
+ dev_err(&pdev->dev, "Error allocating MSI bitmap\n");
+ goto error;
+ }
+
+ rc = xgene_allocate_domains(xgene_msi);
+ if (rc) {
+ dev_err(&pdev->dev, "Failed to allocate MSI domain\n");
+ goto error;
+ }
+
+ for (irq_index = 0; irq_index < NR_HW_IRQS; irq_index++) {
+ virt_msir = platform_get_irq(pdev, irq_index);
+ if (virt_msir < 0) {
+ dev_err(&pdev->dev, "Cannot translate IRQ index %d\n",
+ irq_index);
+ rc = virt_msir;
+ goto error;
+ }
+ xgene_msi->msi_groups[irq_index].gic_irq = virt_msir;
+ xgene_msi->msi_groups[irq_index].msi_grp = irq_index;
+ xgene_msi->msi_groups[irq_index].msi = xgene_msi;
+ }
+
+ /*
+ * MSInIRx registers are read-to-clear; before registering
+ * interrupt handlers, read all of them to clear spurious
+ * interrupts that may occur before the driver is probed.
+ */
+ for (irq_index = 0; irq_index < NR_HW_IRQS; irq_index++) {
+ for (msi_idx = 0; msi_idx < IDX_PER_GROUP; msi_idx++)
+ msi_val = xgene_msi_ir_read(xgene_msi, irq_index,
+ msi_idx);
+ /* Read MSIINTn to confirm */
+ msi_val = xgene_msi_int_read(xgene_msi, irq_index);
+ if (msi_val) {
+ dev_err(&pdev->dev, "Failed to clear spurious IRQ\n");
+ rc = -EINVAL;
+ goto error;
+ }
+ }
+
+ rc = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "pci/xgene:online",
+ xgene_msi_hwirq_alloc, NULL);
+ if (rc < 0)
+ goto err_cpuhp;
+ pci_xgene_online = rc;
+ rc = cpuhp_setup_state(CPUHP_PCI_XGENE_DEAD, "pci/xgene:dead", NULL,
+ xgene_msi_hwirq_free);
+ if (rc)
+ goto err_cpuhp;
+
+ dev_info(&pdev->dev, "APM X-Gene PCIe MSI driver loaded\n");
+
+ return 0;
+
+err_cpuhp:
+ dev_err(&pdev->dev, "failed to add CPU MSI notifier\n");
+error:
+ xgene_msi_remove(pdev);
+ return rc;
+}
+
+static struct platform_driver xgene_msi_driver = {
+ .driver = {
+ .name = "xgene-msi",
+ .of_match_table = xgene_msi_match_table,
+ },
+ .probe = xgene_msi_probe,
+ .remove = xgene_msi_remove,
+};
+
+static int __init xgene_pcie_msi_init(void)
+{
+ return platform_driver_register(&xgene_msi_driver);
+}
+subsys_initcall(xgene_pcie_msi_init);
diff --git a/src/kernel/linux/v4.14/drivers/pci/host/pci-xgene.c b/src/kernel/linux/v4.14/drivers/pci/host/pci-xgene.c
new file mode 100644
index 0000000..c78fd9c
--- /dev/null
+++ b/src/kernel/linux/v4.14/drivers/pci/host/pci-xgene.c
@@ -0,0 +1,714 @@
+/**
+ * APM X-Gene PCIe Driver
+ *
+ * Copyright (c) 2014 Applied Micro Circuits Corporation.
+ *
+ * Author: Tanmay Inamdar <tinamdar@apm.com>.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2 of the License, or (at your
+ * option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ */
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/io.h>
+#include <linux/jiffies.h>
+#include <linux/memblock.h>
+#include <linux/init.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/of_irq.h>
+#include <linux/of_pci.h>
+#include <linux/pci.h>
+#include <linux/pci-acpi.h>
+#include <linux/pci-ecam.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+
+#define PCIECORE_CTLANDSTATUS 0x50
+#define PIM1_1L 0x80
+#define IBAR2 0x98
+#define IR2MSK 0x9c
+#define PIM2_1L 0xa0
+#define IBAR3L 0xb4
+#define IR3MSKL 0xbc
+#define PIM3_1L 0xc4
+#define OMR1BARL 0x100
+#define OMR2BARL 0x118
+#define OMR3BARL 0x130
+#define CFGBARL 0x154
+#define CFGBARH 0x158
+#define CFGCTL 0x15c
+#define RTDID 0x160
+#define BRIDGE_CFG_0 0x2000
+#define BRIDGE_CFG_4 0x2010
+#define BRIDGE_STATUS_0 0x2600
+
+#define LINK_UP_MASK 0x00000100
+#define AXI_EP_CFG_ACCESS 0x10000
+#define EN_COHERENCY 0xF0000000
+#define EN_REG 0x00000001
+#define OB_LO_IO 0x00000002
+#define XGENE_PCIE_VENDORID 0x10E8
+#define XGENE_PCIE_DEVICEID 0xE004
+#define SZ_1T (SZ_1G*1024ULL)
+#define PIPE_PHY_RATE_RD(src) ((0xc000 & (u32)(src)) >> 0xe)
+
+#define XGENE_V1_PCI_EXP_CAP 0x40
+
+/* PCIe IP version */
+#define XGENE_PCIE_IP_VER_UNKN 0
+#define XGENE_PCIE_IP_VER_1 1
+#define XGENE_PCIE_IP_VER_2 2
+
+#if defined(CONFIG_PCI_XGENE) || (defined(CONFIG_ACPI) && defined(CONFIG_PCI_QUIRKS))
+struct xgene_pcie_port {
+ struct device_node *node;
+ struct device *dev;
+ struct clk *clk;
+ void __iomem *csr_base;
+ void __iomem *cfg_base;
+ unsigned long cfg_addr;
+ bool link_up;
+ u32 version;
+};
+
+static u32 xgene_pcie_readl(struct xgene_pcie_port *port, u32 reg)
+{
+ return readl(port->csr_base + reg);
+}
+
+static void xgene_pcie_writel(struct xgene_pcie_port *port, u32 reg, u32 val)
+{
+ writel(val, port->csr_base + reg);
+}
+
+static inline u32 pcie_bar_low_val(u32 addr, u32 flags)
+{
+ return (addr & PCI_BASE_ADDRESS_MEM_MASK) | flags;
+}
+
+static inline struct xgene_pcie_port *pcie_bus_to_port(struct pci_bus *bus)
+{
+ struct pci_config_window *cfg;
+
+ if (acpi_disabled)
+ return (struct xgene_pcie_port *)(bus->sysdata);
+
+ cfg = bus->sysdata;
+ return (struct xgene_pcie_port *)(cfg->priv);
+}
+
+/*
+ * When the address bit [17:16] is 2'b01, the Configuration access will be
+ * treated as Type 1 and it will be forwarded to external PCIe device.
+ */
+static void __iomem *xgene_pcie_get_cfg_base(struct pci_bus *bus)
+{
+ struct xgene_pcie_port *port = pcie_bus_to_port(bus);
+
+ if (bus->number >= (bus->primary + 1))
+ return port->cfg_base + AXI_EP_CFG_ACCESS;
+
+ return port->cfg_base;
+}
+
+/*
+ * For Configuration request, RTDID register is used as Bus Number,
+ * Device Number and Function number of the header fields.
+ */
+static void xgene_pcie_set_rtdid_reg(struct pci_bus *bus, uint devfn)
+{
+ struct xgene_pcie_port *port = pcie_bus_to_port(bus);
+ unsigned int b, d, f;
+ u32 rtdid_val = 0;
+
+ b = bus->number;
+ d = PCI_SLOT(devfn);
+ f = PCI_FUNC(devfn);
+
+ if (!pci_is_root_bus(bus))
+ rtdid_val = (b << 8) | (d << 3) | f;
+
+ xgene_pcie_writel(port, RTDID, rtdid_val);
+ /* read the register back to ensure flush */
+ xgene_pcie_readl(port, RTDID);
+}
+
+/*
+ * X-Gene PCIe port uses BAR0-BAR1 of RC's configuration space as
+ * the translation from PCI bus to native BUS. Entire DDR region
+ * is mapped into PCIe space using these registers, so it can be
+ * reached by DMA from EP devices. The BAR0/1 of bridge should be
+ * hidden during enumeration to avoid the sizing and resource allocation
+ * by PCIe core.
+ */
+static bool xgene_pcie_hide_rc_bars(struct pci_bus *bus, int offset)
+{
+ if (pci_is_root_bus(bus) && ((offset == PCI_BASE_ADDRESS_0) ||
+ (offset == PCI_BASE_ADDRESS_1)))
+ return true;
+
+ return false;
+}
+
+static void __iomem *xgene_pcie_map_bus(struct pci_bus *bus, unsigned int devfn,
+ int offset)
+{
+ if ((pci_is_root_bus(bus) && devfn != 0) ||
+ xgene_pcie_hide_rc_bars(bus, offset))
+ return NULL;
+
+ xgene_pcie_set_rtdid_reg(bus, devfn);
+ return xgene_pcie_get_cfg_base(bus) + offset;
+}
+
+static int xgene_pcie_config_read32(struct pci_bus *bus, unsigned int devfn,
+ int where, int size, u32 *val)
+{
+ struct xgene_pcie_port *port = pcie_bus_to_port(bus);
+
+ if (pci_generic_config_read32(bus, devfn, where & ~0x3, 4, val) !=
+ PCIBIOS_SUCCESSFUL)
+ return PCIBIOS_DEVICE_NOT_FOUND;
+
+ /*
+ * The v1 controller has a bug in its Configuration Request
+ * Retry Status (CRS) logic: when CRS is enabled and we read the
+ * Vendor and Device ID of a non-existent device, the controller
+ * fabricates return data of 0xFFFF0001 ("device exists but is not
+ * ready") instead of 0xFFFFFFFF ("device does not exist"). This
+ * causes the PCI core to retry the read until it times out.
+ * Avoid this by not claiming to support CRS.
+ */
+ if (pci_is_root_bus(bus) && (port->version == XGENE_PCIE_IP_VER_1) &&
+ ((where & ~0x3) == XGENE_V1_PCI_EXP_CAP + PCI_EXP_RTCTL))
+ *val &= ~(PCI_EXP_RTCAP_CRSVIS << 16);
+
+ if (size <= 2)
+ *val = (*val >> (8 * (where & 3))) & ((1 << (size * 8)) - 1);
+
+ return PCIBIOS_SUCCESSFUL;
+}
+#endif
+
+#if defined(CONFIG_ACPI) && defined(CONFIG_PCI_QUIRKS)
+static int xgene_get_csr_resource(struct acpi_device *adev,
+ struct resource *res)
+{
+ struct device *dev = &adev->dev;
+ struct resource_entry *entry;
+ struct list_head list;
+ unsigned long flags;
+ int ret;
+
+ INIT_LIST_HEAD(&list);
+ flags = IORESOURCE_MEM;
+ ret = acpi_dev_get_resources(adev, &list,
+ acpi_dev_filter_resource_type_cb,
+ (void *) flags);
+ if (ret < 0) {
+ dev_err(dev, "failed to parse _CRS method, error code %d\n",
+ ret);
+ return ret;
+ }
+
+ if (ret == 0) {
+ dev_err(dev, "no IO and memory resources present in _CRS\n");
+ return -EINVAL;
+ }
+
+ entry = list_first_entry(&list, struct resource_entry, node);
+ *res = *entry->res;
+ acpi_dev_free_resource_list(&list);
+ return 0;
+}
+
+static int xgene_pcie_ecam_init(struct pci_config_window *cfg, u32 ipversion)
+{
+ struct device *dev = cfg->parent;
+ struct acpi_device *adev = to_acpi_device(dev);
+ struct xgene_pcie_port *port;
+ struct resource csr;
+ int ret;
+
+ port = devm_kzalloc(dev, sizeof(*port), GFP_KERNEL);
+ if (!port)
+ return -ENOMEM;
+
+ ret = xgene_get_csr_resource(adev, &csr);
+ if (ret) {
+ dev_err(dev, "can't get CSR resource\n");
+ return ret;
+ }
+ port->csr_base = devm_pci_remap_cfg_resource(dev, &csr);
+ if (IS_ERR(port->csr_base))
+ return PTR_ERR(port->csr_base);
+
+ port->cfg_base = cfg->win;
+ port->version = ipversion;
+
+ cfg->priv = port;
+ return 0;
+}
+
+static int xgene_v1_pcie_ecam_init(struct pci_config_window *cfg)
+{
+ return xgene_pcie_ecam_init(cfg, XGENE_PCIE_IP_VER_1);
+}
+
+struct pci_ecam_ops xgene_v1_pcie_ecam_ops = {
+ .bus_shift = 16,
+ .init = xgene_v1_pcie_ecam_init,
+ .pci_ops = {
+ .map_bus = xgene_pcie_map_bus,
+ .read = xgene_pcie_config_read32,
+ .write = pci_generic_config_write,
+ }
+};
+
+static int xgene_v2_pcie_ecam_init(struct pci_config_window *cfg)
+{
+ return xgene_pcie_ecam_init(cfg, XGENE_PCIE_IP_VER_2);
+}
+
+struct pci_ecam_ops xgene_v2_pcie_ecam_ops = {
+ .bus_shift = 16,
+ .init = xgene_v2_pcie_ecam_init,
+ .pci_ops = {
+ .map_bus = xgene_pcie_map_bus,
+ .read = xgene_pcie_config_read32,
+ .write = pci_generic_config_write,
+ }
+};
+#endif
+
+#if defined(CONFIG_PCI_XGENE)
+static u64 xgene_pcie_set_ib_mask(struct xgene_pcie_port *port, u32 addr,
+ u32 flags, u64 size)
+{
+ u64 mask = (~(size - 1) & PCI_BASE_ADDRESS_MEM_MASK) | flags;
+ u32 val32 = 0;
+ u32 val;
+
+ val32 = xgene_pcie_readl(port, addr);
+ val = (val32 & 0x0000ffff) | (lower_32_bits(mask) << 16);
+ xgene_pcie_writel(port, addr, val);
+
+ val32 = xgene_pcie_readl(port, addr + 0x04);
+ val = (val32 & 0xffff0000) | (lower_32_bits(mask) >> 16);
+ xgene_pcie_writel(port, addr + 0x04, val);
+
+ val32 = xgene_pcie_readl(port, addr + 0x04);
+ val = (val32 & 0x0000ffff) | (upper_32_bits(mask) << 16);
+ xgene_pcie_writel(port, addr + 0x04, val);
+
+ val32 = xgene_pcie_readl(port, addr + 0x08);
+ val = (val32 & 0xffff0000) | (upper_32_bits(mask) >> 16);
+ xgene_pcie_writel(port, addr + 0x08, val);
+
+ return mask;
+}
+
+static void xgene_pcie_linkup(struct xgene_pcie_port *port,
+ u32 *lanes, u32 *speed)
+{
+ u32 val32;
+
+ port->link_up = false;
+ val32 = xgene_pcie_readl(port, PCIECORE_CTLANDSTATUS);
+ if (val32 & LINK_UP_MASK) {
+ port->link_up = true;
+ *speed = PIPE_PHY_RATE_RD(val32);
+ val32 = xgene_pcie_readl(port, BRIDGE_STATUS_0);
+ *lanes = val32 >> 26;
+ }
+}
+
+static int xgene_pcie_init_port(struct xgene_pcie_port *port)
+{
+ struct device *dev = port->dev;
+ int rc;
+
+ port->clk = clk_get(dev, NULL);
+ if (IS_ERR(port->clk)) {
+ dev_err(dev, "clock not available\n");
+ return -ENODEV;
+ }
+
+ rc = clk_prepare_enable(port->clk);
+ if (rc) {
+ dev_err(dev, "clock enable failed\n");
+ return rc;
+ }
+
+ return 0;
+}
+
+static int xgene_pcie_map_reg(struct xgene_pcie_port *port,
+ struct platform_device *pdev)
+{
+ struct device *dev = port->dev;
+ struct resource *res;
+
+ res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "csr");
+ port->csr_base = devm_pci_remap_cfg_resource(dev, res);
+ if (IS_ERR(port->csr_base))
+ return PTR_ERR(port->csr_base);
+
+ res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "cfg");
+ port->cfg_base = devm_ioremap_resource(dev, res);
+ if (IS_ERR(port->cfg_base))
+ return PTR_ERR(port->cfg_base);
+ port->cfg_addr = res->start;
+
+ return 0;
+}
+
+static void xgene_pcie_setup_ob_reg(struct xgene_pcie_port *port,
+ struct resource *res, u32 offset,
+ u64 cpu_addr, u64 pci_addr)
+{
+ struct device *dev = port->dev;
+ resource_size_t size = resource_size(res);
+ u64 restype = resource_type(res);
+ u64 mask = 0;
+ u32 min_size;
+ u32 flag = EN_REG;
+
+ if (restype == IORESOURCE_MEM) {
+ min_size = SZ_128M;
+ } else {
+ min_size = 128;
+ flag |= OB_LO_IO;
+ }
+
+ if (size >= min_size)
+ mask = ~(size - 1) | flag;
+ else
+ dev_warn(dev, "res size 0x%llx less than minimum 0x%x\n",
+ (u64)size, min_size);
+
+ xgene_pcie_writel(port, offset, lower_32_bits(cpu_addr));
+ xgene_pcie_writel(port, offset + 0x04, upper_32_bits(cpu_addr));
+ xgene_pcie_writel(port, offset + 0x08, lower_32_bits(mask));
+ xgene_pcie_writel(port, offset + 0x0c, upper_32_bits(mask));
+ xgene_pcie_writel(port, offset + 0x10, lower_32_bits(pci_addr));
+ xgene_pcie_writel(port, offset + 0x14, upper_32_bits(pci_addr));
+}
+
+static void xgene_pcie_setup_cfg_reg(struct xgene_pcie_port *port)
+{
+ u64 addr = port->cfg_addr;
+
+ xgene_pcie_writel(port, CFGBARL, lower_32_bits(addr));
+ xgene_pcie_writel(port, CFGBARH, upper_32_bits(addr));
+ xgene_pcie_writel(port, CFGCTL, EN_REG);
+}
+
+static int xgene_pcie_map_ranges(struct xgene_pcie_port *port,
+ struct list_head *res,
+ resource_size_t io_base)
+{
+ struct resource_entry *window;
+ struct device *dev = port->dev;
+ int ret;
+
+ resource_list_for_each_entry(window, res) {
+ struct resource *res = window->res;
+ u64 restype = resource_type(res);
+
+ dev_dbg(dev, "%pR\n", res);
+
+ switch (restype) {
+ case IORESOURCE_IO:
+ xgene_pcie_setup_ob_reg(port, res, OMR3BARL, io_base,
+ res->start - window->offset);
+ ret = pci_remap_iospace(res, io_base);
+ if (ret < 0)
+ return ret;
+ break;
+ case IORESOURCE_MEM:
+ if (res->flags & IORESOURCE_PREFETCH)
+ xgene_pcie_setup_ob_reg(port, res, OMR2BARL,
+ res->start,
+ res->start -
+ window->offset);
+ else
+ xgene_pcie_setup_ob_reg(port, res, OMR1BARL,
+ res->start,
+ res->start -
+ window->offset);
+ break;
+ case IORESOURCE_BUS:
+ break;
+ default:
+ dev_err(dev, "invalid resource %pR\n", res);
+ return -EINVAL;
+ }
+ }
+ xgene_pcie_setup_cfg_reg(port);
+ return 0;
+}
+
+static void xgene_pcie_setup_pims(struct xgene_pcie_port *port, u32 pim_reg,
+ u64 pim, u64 size)
+{
+ xgene_pcie_writel(port, pim_reg, lower_32_bits(pim));
+ xgene_pcie_writel(port, pim_reg + 0x04,
+ upper_32_bits(pim) | EN_COHERENCY);
+ xgene_pcie_writel(port, pim_reg + 0x10, lower_32_bits(size));
+ xgene_pcie_writel(port, pim_reg + 0x14, upper_32_bits(size));
+}
+
+/*
+ * X-Gene PCIe support maximum 3 inbound memory regions
+ * This function helps to select a region based on size of region
+ */
+static int xgene_pcie_select_ib_reg(u8 *ib_reg_mask, u64 size)
+{
+ if ((size > 4) && (size < SZ_16M) && !(*ib_reg_mask & (1 << 1))) {
+ *ib_reg_mask |= (1 << 1);
+ return 1;
+ }
+
+ if ((size > SZ_1K) && (size < SZ_1T) && !(*ib_reg_mask & (1 << 0))) {
+ *ib_reg_mask |= (1 << 0);
+ return 0;
+ }
+
+ if ((size > SZ_1M) && (size < SZ_1T) && !(*ib_reg_mask & (1 << 2))) {
+ *ib_reg_mask |= (1 << 2);
+ return 2;
+ }
+
+ return -EINVAL;
+}
+
+static void xgene_pcie_setup_ib_reg(struct xgene_pcie_port *port,
+ struct of_pci_range *range, u8 *ib_reg_mask)
+{
+ void __iomem *cfg_base = port->cfg_base;
+ struct device *dev = port->dev;
+ void *bar_addr;
+ u32 pim_reg;
+ u64 cpu_addr = range->cpu_addr;
+ u64 pci_addr = range->pci_addr;
+ u64 size = range->size;
+ u64 mask = ~(size - 1) | EN_REG;
+ u32 flags = PCI_BASE_ADDRESS_MEM_TYPE_64;
+ u32 bar_low;
+ int region;
+
+ region = xgene_pcie_select_ib_reg(ib_reg_mask, range->size);
+ if (region < 0) {
+ dev_warn(dev, "invalid pcie dma-range config\n");
+ return;
+ }
+
+ if (range->flags & IORESOURCE_PREFETCH)
+ flags |= PCI_BASE_ADDRESS_MEM_PREFETCH;
+
+ bar_low = pcie_bar_low_val((u32)cpu_addr, flags);
+ switch (region) {
+ case 0:
+ xgene_pcie_set_ib_mask(port, BRIDGE_CFG_4, flags, size);
+ bar_addr = cfg_base + PCI_BASE_ADDRESS_0;
+ writel(bar_low, bar_addr);
+ writel(upper_32_bits(cpu_addr), bar_addr + 0x4);
+ pim_reg = PIM1_1L;
+ break;
+ case 1:
+ xgene_pcie_writel(port, IBAR2, bar_low);
+ xgene_pcie_writel(port, IR2MSK, lower_32_bits(mask));
+ pim_reg = PIM2_1L;
+ break;
+ case 2:
+ xgene_pcie_writel(port, IBAR3L, bar_low);
+ xgene_pcie_writel(port, IBAR3L + 0x4, upper_32_bits(cpu_addr));
+ xgene_pcie_writel(port, IR3MSKL, lower_32_bits(mask));
+ xgene_pcie_writel(port, IR3MSKL + 0x4, upper_32_bits(mask));
+ pim_reg = PIM3_1L;
+ break;
+ }
+
+ xgene_pcie_setup_pims(port, pim_reg, pci_addr, ~(size - 1));
+}
+
+static int pci_dma_range_parser_init(struct of_pci_range_parser *parser,
+ struct device_node *node)
+{
+ const int na = 3, ns = 2;
+ int rlen;
+
+ parser->node = node;
+ parser->pna = of_n_addr_cells(node);
+ parser->np = parser->pna + na + ns;
+
+ parser->range = of_get_property(node, "dma-ranges", &rlen);
+ if (!parser->range)
+ return -ENOENT;
+ parser->end = parser->range + rlen / sizeof(__be32);
+
+ return 0;
+}
+
+static int xgene_pcie_parse_map_dma_ranges(struct xgene_pcie_port *port)
+{
+ struct device_node *np = port->node;
+ struct of_pci_range range;
+ struct of_pci_range_parser parser;
+ struct device *dev = port->dev;
+ u8 ib_reg_mask = 0;
+
+ if (pci_dma_range_parser_init(&parser, np)) {
+ dev_err(dev, "missing dma-ranges property\n");
+ return -EINVAL;
+ }
+
+ /* Get the dma-ranges from DT */
+ for_each_of_pci_range(&parser, &range) {
+ u64 end = range.cpu_addr + range.size - 1;
+
+ dev_dbg(dev, "0x%08x 0x%016llx..0x%016llx -> 0x%016llx\n",
+ range.flags, range.cpu_addr, end, range.pci_addr);
+ xgene_pcie_setup_ib_reg(port, &range, &ib_reg_mask);
+ }
+ return 0;
+}
+
+/* clear BAR configuration which was done by firmware */
+static void xgene_pcie_clear_config(struct xgene_pcie_port *port)
+{
+ int i;
+
+ for (i = PIM1_1L; i <= CFGCTL; i += 4)
+ xgene_pcie_writel(port, i, 0);
+}
+
+static int xgene_pcie_setup(struct xgene_pcie_port *port, struct list_head *res,
+ resource_size_t io_base)
+{
+ struct device *dev = port->dev;
+ u32 val, lanes = 0, speed = 0;
+ int ret;
+
+ xgene_pcie_clear_config(port);
+
+ /* setup the vendor and device IDs correctly */
+ val = (XGENE_PCIE_DEVICEID << 16) | XGENE_PCIE_VENDORID;
+ xgene_pcie_writel(port, BRIDGE_CFG_0, val);
+
+ ret = xgene_pcie_map_ranges(port, res, io_base);
+ if (ret)
+ return ret;
+
+ ret = xgene_pcie_parse_map_dma_ranges(port);
+ if (ret)
+ return ret;
+
+ xgene_pcie_linkup(port, &lanes, &speed);
+ if (!port->link_up)
+ dev_info(dev, "(rc) link down\n");
+ else
+ dev_info(dev, "(rc) x%d gen-%d link up\n", lanes, speed + 1);
+ return 0;
+}
+
+static struct pci_ops xgene_pcie_ops = {
+ .map_bus = xgene_pcie_map_bus,
+ .read = xgene_pcie_config_read32,
+ .write = pci_generic_config_write32,
+};
+
+static int xgene_pcie_probe_bridge(struct platform_device *pdev)
+{
+ struct device *dev = &pdev->dev;
+ struct device_node *dn = dev->of_node;
+ struct xgene_pcie_port *port;
+ resource_size_t iobase = 0;
+ struct pci_bus *bus, *child;
+ struct pci_host_bridge *bridge;
+ int ret;
+ LIST_HEAD(res);
+
+ bridge = devm_pci_alloc_host_bridge(dev, sizeof(*port));
+ if (!bridge)
+ return -ENOMEM;
+
+ port = pci_host_bridge_priv(bridge);
+
+ port->node = of_node_get(dn);
+ port->dev = dev;
+
+ port->version = XGENE_PCIE_IP_VER_UNKN;
+ if (of_device_is_compatible(port->node, "apm,xgene-pcie"))
+ port->version = XGENE_PCIE_IP_VER_1;
+
+ ret = xgene_pcie_map_reg(port, pdev);
+ if (ret)
+ return ret;
+
+ ret = xgene_pcie_init_port(port);
+ if (ret)
+ return ret;
+
+ ret = of_pci_get_host_bridge_resources(dn, 0, 0xff, &res, &iobase);
+ if (ret)
+ return ret;
+
+ ret = devm_request_pci_bus_resources(dev, &res);
+ if (ret)
+ goto error;
+
+ ret = xgene_pcie_setup(port, &res, iobase);
+ if (ret)
+ goto error;
+
+ list_splice_init(&res, &bridge->windows);
+ bridge->dev.parent = dev;
+ bridge->sysdata = port;
+ bridge->busnr = 0;
+ bridge->ops = &xgene_pcie_ops;
+ bridge->map_irq = of_irq_parse_and_map_pci;
+ bridge->swizzle_irq = pci_common_swizzle;
+
+ ret = pci_scan_root_bus_bridge(bridge);
+ if (ret < 0)
+ goto error;
+
+ bus = bridge->bus;
+
+ pci_assign_unassigned_bus_resources(bus);
+ list_for_each_entry(child, &bus->children, node)
+ pcie_bus_configure_settings(child);
+ pci_bus_add_devices(bus);
+ return 0;
+
+error:
+ pci_free_resource_list(&res);
+ return ret;
+}
+
+static const struct of_device_id xgene_pcie_match_table[] = {
+ {.compatible = "apm,xgene-pcie",},
+ {},
+};
+
+static struct platform_driver xgene_pcie_driver = {
+ .driver = {
+ .name = "xgene-pcie",
+ .of_match_table = of_match_ptr(xgene_pcie_match_table),
+ .suppress_bind_attrs = true,
+ },
+ .probe = xgene_pcie_probe_bridge,
+};
+builtin_platform_driver(xgene_pcie_driver);
+#endif
diff --git a/src/kernel/linux/v4.14/drivers/pci/host/pcie-altera-msi.c b/src/kernel/linux/v4.14/drivers/pci/host/pcie-altera-msi.c
new file mode 100644
index 0000000..d8141f4
--- /dev/null
+++ b/src/kernel/linux/v4.14/drivers/pci/host/pcie-altera-msi.c
@@ -0,0 +1,302 @@
+/*
+ * Altera PCIe MSI support
+ *
+ * Author: Ley Foon Tan <lftan@altera.com>
+ *
+ * Copyright Altera Corporation (C) 2013-2015. All rights reserved
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/interrupt.h>
+#include <linux/irqchip/chained_irq.h>
+#include <linux/init.h>
+#include <linux/msi.h>
+#include <linux/of_address.h>
+#include <linux/of_irq.h>
+#include <linux/of_pci.h>
+#include <linux/pci.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+
+#define MSI_STATUS 0x0
+#define MSI_ERROR 0x4
+#define MSI_INTMASK 0x8
+
+#define MAX_MSI_VECTORS 32
+
+struct altera_msi {
+ DECLARE_BITMAP(used, MAX_MSI_VECTORS);
+ struct mutex lock; /* protect "used" bitmap */
+ struct platform_device *pdev;
+ struct irq_domain *msi_domain;
+ struct irq_domain *inner_domain;
+ void __iomem *csr_base;
+ void __iomem *vector_base;
+ phys_addr_t vector_phy;
+ u32 num_of_vectors;
+ int irq;
+};
+
+static inline void msi_writel(struct altera_msi *msi, const u32 value,
+ const u32 reg)
+{
+ writel_relaxed(value, msi->csr_base + reg);
+}
+
+static inline u32 msi_readl(struct altera_msi *msi, const u32 reg)
+{
+ return readl_relaxed(msi->csr_base + reg);
+}
+
+static void altera_msi_isr(struct irq_desc *desc)
+{
+ struct irq_chip *chip = irq_desc_get_chip(desc);
+ struct altera_msi *msi;
+ unsigned long status;
+ u32 bit;
+ u32 virq;
+
+ chained_irq_enter(chip, desc);
+ msi = irq_desc_get_handler_data(desc);
+
+ while ((status = msi_readl(msi, MSI_STATUS)) != 0) {
+ for_each_set_bit(bit, &status, msi->num_of_vectors) {
+ /* Dummy read from vector to clear the interrupt */
+ readl_relaxed(msi->vector_base + (bit * sizeof(u32)));
+
+ virq = irq_find_mapping(msi->inner_domain, bit);
+ if (virq)
+ generic_handle_irq(virq);
+ else
+ dev_err(&msi->pdev->dev, "unexpected MSI\n");
+ }
+ }
+
+ chained_irq_exit(chip, desc);
+}
+
+static struct irq_chip altera_msi_irq_chip = {
+ .name = "Altera PCIe MSI",
+ .irq_mask = pci_msi_mask_irq,
+ .irq_unmask = pci_msi_unmask_irq,
+};
+
+static struct msi_domain_info altera_msi_domain_info = {
+ .flags = (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS |
+ MSI_FLAG_PCI_MSIX),
+ .chip = &altera_msi_irq_chip,
+};
+
+static void altera_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
+{
+ struct altera_msi *msi = irq_data_get_irq_chip_data(data);
+ phys_addr_t addr = msi->vector_phy + (data->hwirq * sizeof(u32));
+
+ msg->address_lo = lower_32_bits(addr);
+ msg->address_hi = upper_32_bits(addr);
+ msg->data = data->hwirq;
+
+ dev_dbg(&msi->pdev->dev, "msi#%d address_hi %#x address_lo %#x\n",
+ (int)data->hwirq, msg->address_hi, msg->address_lo);
+}
+
+static int altera_msi_set_affinity(struct irq_data *irq_data,
+ const struct cpumask *mask, bool force)
+{
+ return -EINVAL;
+}
+
+static struct irq_chip altera_msi_bottom_irq_chip = {
+ .name = "Altera MSI",
+ .irq_compose_msi_msg = altera_compose_msi_msg,
+ .irq_set_affinity = altera_msi_set_affinity,
+};
+
+static int altera_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
+ unsigned int nr_irqs, void *args)
+{
+ struct altera_msi *msi = domain->host_data;
+ unsigned long bit;
+ u32 mask;
+
+ WARN_ON(nr_irqs != 1);
+ mutex_lock(&msi->lock);
+
+ bit = find_first_zero_bit(msi->used, msi->num_of_vectors);
+ if (bit >= msi->num_of_vectors) {
+ mutex_unlock(&msi->lock);
+ return -ENOSPC;
+ }
+
+ set_bit(bit, msi->used);
+
+ mutex_unlock(&msi->lock);
+
+ irq_domain_set_info(domain, virq, bit, &altera_msi_bottom_irq_chip,
+ domain->host_data, handle_simple_irq,
+ NULL, NULL);
+
+ mask = msi_readl(msi, MSI_INTMASK);
+ mask |= 1 << bit;
+ msi_writel(msi, mask, MSI_INTMASK);
+
+ return 0;
+}
+
+static void altera_irq_domain_free(struct irq_domain *domain,
+ unsigned int virq, unsigned int nr_irqs)
+{
+ struct irq_data *d = irq_domain_get_irq_data(domain, virq);
+ struct altera_msi *msi = irq_data_get_irq_chip_data(d);
+ u32 mask;
+
+ mutex_lock(&msi->lock);
+
+ if (!test_bit(d->hwirq, msi->used)) {
+ dev_err(&msi->pdev->dev, "trying to free unused MSI#%lu\n",
+ d->hwirq);
+ } else {
+ __clear_bit(d->hwirq, msi->used);
+ mask = msi_readl(msi, MSI_INTMASK);
+ mask &= ~(1 << d->hwirq);
+ msi_writel(msi, mask, MSI_INTMASK);
+ }
+
+ mutex_unlock(&msi->lock);
+}
+
+static const struct irq_domain_ops msi_domain_ops = {
+ .alloc = altera_irq_domain_alloc,
+ .free = altera_irq_domain_free,
+};
+
+static int altera_allocate_domains(struct altera_msi *msi)
+{
+ struct fwnode_handle *fwnode = of_node_to_fwnode(msi->pdev->dev.of_node);
+
+ msi->inner_domain = irq_domain_add_linear(NULL, msi->num_of_vectors,
+ &msi_domain_ops, msi);
+ if (!msi->inner_domain) {
+ dev_err(&msi->pdev->dev, "failed to create IRQ domain\n");
+ return -ENOMEM;
+ }
+
+ msi->msi_domain = pci_msi_create_irq_domain(fwnode,
+ &altera_msi_domain_info, msi->inner_domain);
+ if (!msi->msi_domain) {
+ dev_err(&msi->pdev->dev, "failed to create MSI domain\n");
+ irq_domain_remove(msi->inner_domain);
+ return -ENOMEM;
+ }
+
+ return 0;
+}
+
+static void altera_free_domains(struct altera_msi *msi)
+{
+ irq_domain_remove(msi->msi_domain);
+ irq_domain_remove(msi->inner_domain);
+}
+
+static int altera_msi_remove(struct platform_device *pdev)
+{
+ struct altera_msi *msi = platform_get_drvdata(pdev);
+
+ msi_writel(msi, 0, MSI_INTMASK);
+ irq_set_chained_handler(msi->irq, NULL);
+ irq_set_handler_data(msi->irq, NULL);
+
+ altera_free_domains(msi);
+
+ platform_set_drvdata(pdev, NULL);
+ return 0;
+}
+
+static int altera_msi_probe(struct platform_device *pdev)
+{
+ struct altera_msi *msi;
+ struct device_node *np = pdev->dev.of_node;
+ struct resource *res;
+ int ret;
+
+ msi = devm_kzalloc(&pdev->dev, sizeof(struct altera_msi),
+ GFP_KERNEL);
+ if (!msi)
+ return -ENOMEM;
+
+ mutex_init(&msi->lock);
+ msi->pdev = pdev;
+
+ res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "csr");
+ msi->csr_base = devm_ioremap_resource(&pdev->dev, res);
+ if (IS_ERR(msi->csr_base)) {
+ dev_err(&pdev->dev, "failed to map csr memory\n");
+ return PTR_ERR(msi->csr_base);
+ }
+
+ res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
+ "vector_slave");
+ msi->vector_base = devm_ioremap_resource(&pdev->dev, res);
+ if (IS_ERR(msi->vector_base)) {
+ dev_err(&pdev->dev, "failed to map vector_slave memory\n");
+ return PTR_ERR(msi->vector_base);
+ }
+
+ msi->vector_phy = res->start;
+
+ if (of_property_read_u32(np, "num-vectors", &msi->num_of_vectors)) {
+ dev_err(&pdev->dev, "failed to parse the number of vectors\n");
+ return -EINVAL;
+ }
+
+ ret = altera_allocate_domains(msi);
+ if (ret)
+ return ret;
+
+ msi->irq = platform_get_irq(pdev, 0);
+ if (msi->irq < 0) {
+ dev_err(&pdev->dev, "failed to map IRQ: %d\n", msi->irq);
+ ret = msi->irq;
+ goto err;
+ }
+
+ irq_set_chained_handler_and_data(msi->irq, altera_msi_isr, msi);
+ platform_set_drvdata(pdev, msi);
+
+ return 0;
+
+err:
+ altera_msi_remove(pdev);
+ return ret;
+}
+
+static const struct of_device_id altera_msi_of_match[] = {
+ { .compatible = "altr,msi-1.0", NULL },
+ { },
+};
+
+static struct platform_driver altera_msi_driver = {
+ .driver = {
+ .name = "altera-msi",
+ .of_match_table = altera_msi_of_match,
+ },
+ .probe = altera_msi_probe,
+ .remove = altera_msi_remove,
+};
+
+static int __init altera_msi_init(void)
+{
+ return platform_driver_register(&altera_msi_driver);
+}
+subsys_initcall(altera_msi_init);
diff --git a/src/kernel/linux/v4.14/drivers/pci/host/pcie-altera.c b/src/kernel/linux/v4.14/drivers/pci/host/pcie-altera.c
new file mode 100644
index 0000000..b468b8c
--- /dev/null
+++ b/src/kernel/linux/v4.14/drivers/pci/host/pcie-altera.c
@@ -0,0 +1,655 @@
+/*
+ * Copyright Altera Corporation (C) 2013-2015. All rights reserved
+ *
+ * Author: Ley Foon Tan <lftan@altera.com>
+ * Description: Altera PCIe host controller driver
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/delay.h>
+#include <linux/interrupt.h>
+#include <linux/irqchip/chained_irq.h>
+#include <linux/init.h>
+#include <linux/of_address.h>
+#include <linux/of_irq.h>
+#include <linux/of_pci.h>
+#include <linux/pci.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+
+#define RP_TX_REG0 0x2000
+#define RP_TX_REG1 0x2004
+#define RP_TX_CNTRL 0x2008
+#define RP_TX_EOP 0x2
+#define RP_TX_SOP 0x1
+#define RP_RXCPL_STATUS 0x2010
+#define RP_RXCPL_EOP 0x2
+#define RP_RXCPL_SOP 0x1
+#define RP_RXCPL_REG0 0x2014
+#define RP_RXCPL_REG1 0x2018
+#define P2A_INT_STATUS 0x3060
+#define P2A_INT_STS_ALL 0xf
+#define P2A_INT_ENABLE 0x3070
+#define P2A_INT_ENA_ALL 0xf
+#define RP_LTSSM 0x3c64
+#define RP_LTSSM_MASK 0x1f
+#define LTSSM_L0 0xf
+
+#define PCIE_CAP_OFFSET 0x80
+/* TLP configuration type 0 and 1 */
+#define TLP_FMTTYPE_CFGRD0 0x04 /* Configuration Read Type 0 */
+#define TLP_FMTTYPE_CFGWR0 0x44 /* Configuration Write Type 0 */
+#define TLP_FMTTYPE_CFGRD1 0x05 /* Configuration Read Type 1 */
+#define TLP_FMTTYPE_CFGWR1 0x45 /* Configuration Write Type 1 */
+#define TLP_PAYLOAD_SIZE 0x01
+#define TLP_READ_TAG 0x1d
+#define TLP_WRITE_TAG 0x10
+#define RP_DEVFN 0
+#define TLP_REQ_ID(bus, devfn) (((bus) << 8) | (devfn))
+#define TLP_CFGRD_DW0(pcie, bus) \
+ ((((bus == pcie->root_bus_nr) ? TLP_FMTTYPE_CFGRD0 \
+ : TLP_FMTTYPE_CFGRD1) << 24) | \
+ TLP_PAYLOAD_SIZE)
+#define TLP_CFGWR_DW0(pcie, bus) \
+ ((((bus == pcie->root_bus_nr) ? TLP_FMTTYPE_CFGWR0 \
+ : TLP_FMTTYPE_CFGWR1) << 24) | \
+ TLP_PAYLOAD_SIZE)
+#define TLP_CFG_DW1(pcie, tag, be) \
+ (((TLP_REQ_ID(pcie->root_bus_nr, RP_DEVFN)) << 16) | (tag << 8) | (be))
+#define TLP_CFG_DW2(bus, devfn, offset) \
+ (((bus) << 24) | ((devfn) << 16) | (offset))
+#define TLP_COMP_STATUS(s) (((s) >> 13) & 7)
+#define TLP_HDR_SIZE 3
+#define TLP_LOOP 500
+
+#define LINK_UP_TIMEOUT HZ
+#define LINK_RETRAIN_TIMEOUT HZ
+
+#define DWORD_MASK 3
+
+struct altera_pcie {
+ struct platform_device *pdev;
+ void __iomem *cra_base; /* DT Cra */
+ int irq;
+ u8 root_bus_nr;
+ struct irq_domain *irq_domain;
+ struct resource bus_range;
+ struct list_head resources;
+};
+
+struct tlp_rp_regpair_t {
+ u32 ctrl;
+ u32 reg0;
+ u32 reg1;
+};
+
+static inline void cra_writel(struct altera_pcie *pcie, const u32 value,
+ const u32 reg)
+{
+ writel_relaxed(value, pcie->cra_base + reg);
+}
+
+static inline u32 cra_readl(struct altera_pcie *pcie, const u32 reg)
+{
+ return readl_relaxed(pcie->cra_base + reg);
+}
+
+static bool altera_pcie_link_is_up(struct altera_pcie *pcie)
+{
+ return !!((cra_readl(pcie, RP_LTSSM) & RP_LTSSM_MASK) == LTSSM_L0);
+}
+
+/*
+ * Altera PCIe port uses BAR0 of RC's configuration space as the translation
+ * from PCI bus to native BUS. Entire DDR region is mapped into PCIe space
+ * using these registers, so it can be reached by DMA from EP devices.
+ * This BAR0 will also access to MSI vector when receiving MSI/MSIX interrupt
+ * from EP devices, eventually trigger interrupt to GIC. The BAR0 of bridge
+ * should be hidden during enumeration to avoid the sizing and resource
+ * allocation by PCIe core.
+ */
+static bool altera_pcie_hide_rc_bar(struct pci_bus *bus, unsigned int devfn,
+ int offset)
+{
+ if (pci_is_root_bus(bus) && (devfn == 0) &&
+ (offset == PCI_BASE_ADDRESS_0))
+ return true;
+
+ return false;
+}
+
+static void tlp_write_tx(struct altera_pcie *pcie,
+ struct tlp_rp_regpair_t *tlp_rp_regdata)
+{
+ cra_writel(pcie, tlp_rp_regdata->reg0, RP_TX_REG0);
+ cra_writel(pcie, tlp_rp_regdata->reg1, RP_TX_REG1);
+ cra_writel(pcie, tlp_rp_regdata->ctrl, RP_TX_CNTRL);
+}
+
+static bool altera_pcie_valid_device(struct altera_pcie *pcie,
+ struct pci_bus *bus, int dev)
+{
+ /* If there is no link, then there is no device */
+ if (bus->number != pcie->root_bus_nr) {
+ if (!altera_pcie_link_is_up(pcie))
+ return false;
+ }
+
+ /* access only one slot on each root port */
+ if (bus->number == pcie->root_bus_nr && dev > 0)
+ return false;
+
+ return true;
+}
+
+static int tlp_read_packet(struct altera_pcie *pcie, u32 *value)
+{
+ int i;
+ bool sop = 0;
+ u32 ctrl;
+ u32 reg0, reg1;
+ u32 comp_status = 1;
+
+ /*
+ * Minimum 2 loops to read TLP headers and 1 loop to read data
+ * payload.
+ */
+ for (i = 0; i < TLP_LOOP; i++) {
+ ctrl = cra_readl(pcie, RP_RXCPL_STATUS);
+ if ((ctrl & RP_RXCPL_SOP) || (ctrl & RP_RXCPL_EOP) || sop) {
+ reg0 = cra_readl(pcie, RP_RXCPL_REG0);
+ reg1 = cra_readl(pcie, RP_RXCPL_REG1);
+
+ if (ctrl & RP_RXCPL_SOP) {
+ sop = true;
+ comp_status = TLP_COMP_STATUS(reg1);
+ }
+
+ if (ctrl & RP_RXCPL_EOP) {
+ if (comp_status)
+ return PCIBIOS_DEVICE_NOT_FOUND;
+
+ if (value)
+ *value = reg0;
+
+ return PCIBIOS_SUCCESSFUL;
+ }
+ }
+ udelay(5);
+ }
+
+ return PCIBIOS_DEVICE_NOT_FOUND;
+}
+
+static void tlp_write_packet(struct altera_pcie *pcie, u32 *headers,
+ u32 data, bool align)
+{
+ struct tlp_rp_regpair_t tlp_rp_regdata;
+
+ tlp_rp_regdata.reg0 = headers[0];
+ tlp_rp_regdata.reg1 = headers[1];
+ tlp_rp_regdata.ctrl = RP_TX_SOP;
+ tlp_write_tx(pcie, &tlp_rp_regdata);
+
+ if (align) {
+ tlp_rp_regdata.reg0 = headers[2];
+ tlp_rp_regdata.reg1 = 0;
+ tlp_rp_regdata.ctrl = 0;
+ tlp_write_tx(pcie, &tlp_rp_regdata);
+
+ tlp_rp_regdata.reg0 = data;
+ tlp_rp_regdata.reg1 = 0;
+ } else {
+ tlp_rp_regdata.reg0 = headers[2];
+ tlp_rp_regdata.reg1 = data;
+ }
+
+ tlp_rp_regdata.ctrl = RP_TX_EOP;
+ tlp_write_tx(pcie, &tlp_rp_regdata);
+}
+
+static int tlp_cfg_dword_read(struct altera_pcie *pcie, u8 bus, u32 devfn,
+ int where, u8 byte_en, u32 *value)
+{
+ u32 headers[TLP_HDR_SIZE];
+
+ headers[0] = TLP_CFGRD_DW0(pcie, bus);
+ headers[1] = TLP_CFG_DW1(pcie, TLP_READ_TAG, byte_en);
+ headers[2] = TLP_CFG_DW2(bus, devfn, where);
+
+ tlp_write_packet(pcie, headers, 0, false);
+
+ return tlp_read_packet(pcie, value);
+}
+
+static int tlp_cfg_dword_write(struct altera_pcie *pcie, u8 bus, u32 devfn,
+ int where, u8 byte_en, u32 value)
+{
+ u32 headers[TLP_HDR_SIZE];
+ int ret;
+
+ headers[0] = TLP_CFGWR_DW0(pcie, bus);
+ headers[1] = TLP_CFG_DW1(pcie, TLP_WRITE_TAG, byte_en);
+ headers[2] = TLP_CFG_DW2(bus, devfn, where);
+
+ /* check alignment to Qword */
+ if ((where & 0x7) == 0)
+ tlp_write_packet(pcie, headers, value, true);
+ else
+ tlp_write_packet(pcie, headers, value, false);
+
+ ret = tlp_read_packet(pcie, NULL);
+ if (ret != PCIBIOS_SUCCESSFUL)
+ return ret;
+
+ /*
+ * Monitor changes to PCI_PRIMARY_BUS register on root port
+ * and update local copy of root bus number accordingly.
+ */
+ if ((bus == pcie->root_bus_nr) && (where == PCI_PRIMARY_BUS))
+ pcie->root_bus_nr = (u8)(value);
+
+ return PCIBIOS_SUCCESSFUL;
+}
+
+static int _altera_pcie_cfg_read(struct altera_pcie *pcie, u8 busno,
+ unsigned int devfn, int where, int size,
+ u32 *value)
+{
+ int ret;
+ u32 data;
+ u8 byte_en;
+
+ switch (size) {
+ case 1:
+ byte_en = 1 << (where & 3);
+ break;
+ case 2:
+ byte_en = 3 << (where & 3);
+ break;
+ default:
+ byte_en = 0xf;
+ break;
+ }
+
+ ret = tlp_cfg_dword_read(pcie, busno, devfn,
+ (where & ~DWORD_MASK), byte_en, &data);
+ if (ret != PCIBIOS_SUCCESSFUL)
+ return ret;
+
+ switch (size) {
+ case 1:
+ *value = (data >> (8 * (where & 0x3))) & 0xff;
+ break;
+ case 2:
+ *value = (data >> (8 * (where & 0x2))) & 0xffff;
+ break;
+ default:
+ *value = data;
+ break;
+ }
+
+ return PCIBIOS_SUCCESSFUL;
+}
+
+static int _altera_pcie_cfg_write(struct altera_pcie *pcie, u8 busno,
+ unsigned int devfn, int where, int size,
+ u32 value)
+{
+ u32 data32;
+ u32 shift = 8 * (where & 3);
+ u8 byte_en;
+
+ switch (size) {
+ case 1:
+ data32 = (value & 0xff) << shift;
+ byte_en = 1 << (where & 3);
+ break;
+ case 2:
+ data32 = (value & 0xffff) << shift;
+ byte_en = 3 << (where & 3);
+ break;
+ default:
+ data32 = value;
+ byte_en = 0xf;
+ break;
+ }
+
+ return tlp_cfg_dword_write(pcie, busno, devfn, (where & ~DWORD_MASK),
+ byte_en, data32);
+}
+
+static int altera_pcie_cfg_read(struct pci_bus *bus, unsigned int devfn,
+ int where, int size, u32 *value)
+{
+ struct altera_pcie *pcie = bus->sysdata;
+
+ if (altera_pcie_hide_rc_bar(bus, devfn, where))
+ return PCIBIOS_BAD_REGISTER_NUMBER;
+
+ if (!altera_pcie_valid_device(pcie, bus, PCI_SLOT(devfn))) {
+ *value = 0xffffffff;
+ return PCIBIOS_DEVICE_NOT_FOUND;
+ }
+
+ return _altera_pcie_cfg_read(pcie, bus->number, devfn, where, size,
+ value);
+}
+
+static int altera_pcie_cfg_write(struct pci_bus *bus, unsigned int devfn,
+ int where, int size, u32 value)
+{
+ struct altera_pcie *pcie = bus->sysdata;
+
+ if (altera_pcie_hide_rc_bar(bus, devfn, where))
+ return PCIBIOS_BAD_REGISTER_NUMBER;
+
+ if (!altera_pcie_valid_device(pcie, bus, PCI_SLOT(devfn)))
+ return PCIBIOS_DEVICE_NOT_FOUND;
+
+ return _altera_pcie_cfg_write(pcie, bus->number, devfn, where, size,
+ value);
+}
+
+static struct pci_ops altera_pcie_ops = {
+ .read = altera_pcie_cfg_read,
+ .write = altera_pcie_cfg_write,
+};
+
+static int altera_read_cap_word(struct altera_pcie *pcie, u8 busno,
+ unsigned int devfn, int offset, u16 *value)
+{
+ u32 data;
+ int ret;
+
+ ret = _altera_pcie_cfg_read(pcie, busno, devfn,
+ PCIE_CAP_OFFSET + offset, sizeof(*value),
+ &data);
+ *value = data;
+ return ret;
+}
+
+static int altera_write_cap_word(struct altera_pcie *pcie, u8 busno,
+ unsigned int devfn, int offset, u16 value)
+{
+ return _altera_pcie_cfg_write(pcie, busno, devfn,
+ PCIE_CAP_OFFSET + offset, sizeof(value),
+ value);
+}
+
+static void altera_wait_link_retrain(struct altera_pcie *pcie)
+{
+ struct device *dev = &pcie->pdev->dev;
+ u16 reg16;
+ unsigned long start_jiffies;
+
+ /* Wait for link training end. */
+ start_jiffies = jiffies;
+ for (;;) {
+ altera_read_cap_word(pcie, pcie->root_bus_nr, RP_DEVFN,
+ PCI_EXP_LNKSTA, ®16);
+ if (!(reg16 & PCI_EXP_LNKSTA_LT))
+ break;
+
+ if (time_after(jiffies, start_jiffies + LINK_RETRAIN_TIMEOUT)) {
+ dev_err(dev, "link retrain timeout\n");
+ break;
+ }
+ udelay(100);
+ }
+
+ /* Wait for link is up */
+ start_jiffies = jiffies;
+ for (;;) {
+ if (altera_pcie_link_is_up(pcie))
+ break;
+
+ if (time_after(jiffies, start_jiffies + LINK_UP_TIMEOUT)) {
+ dev_err(dev, "link up timeout\n");
+ break;
+ }
+ udelay(100);
+ }
+}
+
+static void altera_pcie_retrain(struct altera_pcie *pcie)
+{
+ u16 linkcap, linkstat, linkctl;
+
+ if (!altera_pcie_link_is_up(pcie))
+ return;
+
+ /*
+ * Set the retrain bit if the PCIe rootport support > 2.5GB/s, but
+ * current speed is 2.5 GB/s.
+ */
+ altera_read_cap_word(pcie, pcie->root_bus_nr, RP_DEVFN, PCI_EXP_LNKCAP,
+ &linkcap);
+ if ((linkcap & PCI_EXP_LNKCAP_SLS) <= PCI_EXP_LNKCAP_SLS_2_5GB)
+ return;
+
+ altera_read_cap_word(pcie, pcie->root_bus_nr, RP_DEVFN, PCI_EXP_LNKSTA,
+ &linkstat);
+ if ((linkstat & PCI_EXP_LNKSTA_CLS) == PCI_EXP_LNKSTA_CLS_2_5GB) {
+ altera_read_cap_word(pcie, pcie->root_bus_nr, RP_DEVFN,
+ PCI_EXP_LNKCTL, &linkctl);
+ linkctl |= PCI_EXP_LNKCTL_RL;
+ altera_write_cap_word(pcie, pcie->root_bus_nr, RP_DEVFN,
+ PCI_EXP_LNKCTL, linkctl);
+
+ altera_wait_link_retrain(pcie);
+ }
+}
+
+static int altera_pcie_intx_map(struct irq_domain *domain, unsigned int irq,
+ irq_hw_number_t hwirq)
+{
+ irq_set_chip_and_handler(irq, &dummy_irq_chip, handle_simple_irq);
+ irq_set_chip_data(irq, domain->host_data);
+ return 0;
+}
+
+static const struct irq_domain_ops intx_domain_ops = {
+ .map = altera_pcie_intx_map,
+ .xlate = pci_irqd_intx_xlate,
+};
+
+static void altera_pcie_isr(struct irq_desc *desc)
+{
+ struct irq_chip *chip = irq_desc_get_chip(desc);
+ struct altera_pcie *pcie;
+ struct device *dev;
+ unsigned long status;
+ u32 bit;
+ u32 virq;
+
+ chained_irq_enter(chip, desc);
+ pcie = irq_desc_get_handler_data(desc);
+ dev = &pcie->pdev->dev;
+
+ while ((status = cra_readl(pcie, P2A_INT_STATUS)
+ & P2A_INT_STS_ALL) != 0) {
+ for_each_set_bit(bit, &status, PCI_NUM_INTX) {
+ /* clear interrupts */
+ cra_writel(pcie, 1 << bit, P2A_INT_STATUS);
+
+ virq = irq_find_mapping(pcie->irq_domain, bit);
+ if (virq)
+ generic_handle_irq(virq);
+ else
+ dev_err(dev, "unexpected IRQ, INT%d\n", bit);
+ }
+ }
+
+ chained_irq_exit(chip, desc);
+}
+
+static int altera_pcie_parse_request_of_pci_ranges(struct altera_pcie *pcie)
+{
+ int err, res_valid = 0;
+ struct device *dev = &pcie->pdev->dev;
+ struct device_node *np = dev->of_node;
+ struct resource_entry *win;
+
+ err = of_pci_get_host_bridge_resources(np, 0, 0xff, &pcie->resources,
+ NULL);
+ if (err)
+ return err;
+
+ err = devm_request_pci_bus_resources(dev, &pcie->resources);
+ if (err)
+ goto out_release_res;
+
+ resource_list_for_each_entry(win, &pcie->resources) {
+ struct resource *res = win->res;
+
+ if (resource_type(res) == IORESOURCE_MEM)
+ res_valid |= !(res->flags & IORESOURCE_PREFETCH);
+ }
+
+ if (res_valid)
+ return 0;
+
+ dev_err(dev, "non-prefetchable memory resource required\n");
+ err = -EINVAL;
+
+out_release_res:
+ pci_free_resource_list(&pcie->resources);
+ return err;
+}
+
+static int altera_pcie_init_irq_domain(struct altera_pcie *pcie)
+{
+ struct device *dev = &pcie->pdev->dev;
+ struct device_node *node = dev->of_node;
+
+ /* Setup INTx */
+ pcie->irq_domain = irq_domain_add_linear(node, PCI_NUM_INTX,
+ &intx_domain_ops, pcie);
+ if (!pcie->irq_domain) {
+ dev_err(dev, "Failed to get a INTx IRQ domain\n");
+ return -ENOMEM;
+ }
+
+ return 0;
+}
+
+static int altera_pcie_parse_dt(struct altera_pcie *pcie)
+{
+ struct device *dev = &pcie->pdev->dev;
+ struct platform_device *pdev = pcie->pdev;
+ struct resource *cra;
+
+ cra = platform_get_resource_byname(pdev, IORESOURCE_MEM, "Cra");
+ pcie->cra_base = devm_ioremap_resource(dev, cra);
+ if (IS_ERR(pcie->cra_base))
+ return PTR_ERR(pcie->cra_base);
+
+ /* setup IRQ */
+ pcie->irq = platform_get_irq(pdev, 0);
+ if (pcie->irq < 0) {
+ dev_err(dev, "failed to get IRQ: %d\n", pcie->irq);
+ return pcie->irq;
+ }
+
+ irq_set_chained_handler_and_data(pcie->irq, altera_pcie_isr, pcie);
+ return 0;
+}
+
+static void altera_pcie_host_init(struct altera_pcie *pcie)
+{
+ altera_pcie_retrain(pcie);
+}
+
+static int altera_pcie_probe(struct platform_device *pdev)
+{
+ struct device *dev = &pdev->dev;
+ struct altera_pcie *pcie;
+ struct pci_bus *bus;
+ struct pci_bus *child;
+ struct pci_host_bridge *bridge;
+ int ret;
+
+ bridge = devm_pci_alloc_host_bridge(dev, sizeof(*pcie));
+ if (!bridge)
+ return -ENOMEM;
+
+ pcie = pci_host_bridge_priv(bridge);
+ pcie->pdev = pdev;
+
+ ret = altera_pcie_parse_dt(pcie);
+ if (ret) {
+ dev_err(dev, "Parsing DT failed\n");
+ return ret;
+ }
+
+ INIT_LIST_HEAD(&pcie->resources);
+
+ ret = altera_pcie_parse_request_of_pci_ranges(pcie);
+ if (ret) {
+ dev_err(dev, "Failed add resources\n");
+ return ret;
+ }
+
+ ret = altera_pcie_init_irq_domain(pcie);
+ if (ret) {
+ dev_err(dev, "Failed creating IRQ Domain\n");
+ return ret;
+ }
+
+ /* clear all interrupts */
+ cra_writel(pcie, P2A_INT_STS_ALL, P2A_INT_STATUS);
+ /* enable all interrupts */
+ cra_writel(pcie, P2A_INT_ENA_ALL, P2A_INT_ENABLE);
+ altera_pcie_host_init(pcie);
+
+ list_splice_init(&pcie->resources, &bridge->windows);
+ bridge->dev.parent = dev;
+ bridge->sysdata = pcie;
+ bridge->busnr = pcie->root_bus_nr;
+ bridge->ops = &altera_pcie_ops;
+ bridge->map_irq = of_irq_parse_and_map_pci;
+ bridge->swizzle_irq = pci_common_swizzle;
+
+ ret = pci_scan_root_bus_bridge(bridge);
+ if (ret < 0)
+ return ret;
+
+ bus = bridge->bus;
+
+ pci_assign_unassigned_bus_resources(bus);
+
+ /* Configure PCI Express setting. */
+ list_for_each_entry(child, &bus->children, node)
+ pcie_bus_configure_settings(child);
+
+ pci_bus_add_devices(bus);
+ return ret;
+}
+
+static const struct of_device_id altera_pcie_of_match[] = {
+ { .compatible = "altr,pcie-root-port-1.0", },
+ {},
+};
+
+static struct platform_driver altera_pcie_driver = {
+ .probe = altera_pcie_probe,
+ .driver = {
+ .name = "altera-pcie",
+ .of_match_table = altera_pcie_of_match,
+ .suppress_bind_attrs = true,
+ },
+};
+
+builtin_platform_driver(altera_pcie_driver);
diff --git a/src/kernel/linux/v4.14/drivers/pci/host/pcie-iproc-bcma.c b/src/kernel/linux/v4.14/drivers/pci/host/pcie-iproc-bcma.c
new file mode 100644
index 0000000..f03d5e3
--- /dev/null
+++ b/src/kernel/linux/v4.14/drivers/pci/host/pcie-iproc-bcma.c
@@ -0,0 +1,121 @@
+/*
+ * Copyright (C) 2015 Broadcom Corporation
+ * Copyright (C) 2015 Hauke Mehrtens <hauke@hauke-m.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation version 2.
+ *
+ * This program is distributed "as is" WITHOUT ANY WARRANTY of any
+ * kind, whether express or implied; without even the implied warranty
+ * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/kernel.h>
+#include <linux/pci.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/phy/phy.h>
+#include <linux/bcma/bcma.h>
+#include <linux/ioport.h>
+
+#include "pcie-iproc.h"
+
+
+/* NS: CLASS field is R/O, and set to wrong 0x200 value */
+static void bcma_pcie2_fixup_class(struct pci_dev *dev)
+{
+ dev->class = PCI_CLASS_BRIDGE_PCI << 8;
+}
+DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0x8011, bcma_pcie2_fixup_class);
+DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0x8012, bcma_pcie2_fixup_class);
+
+static int iproc_pcie_bcma_map_irq(const struct pci_dev *dev, u8 slot, u8 pin)
+{
+ struct pci_sys_data *sys = dev->sysdata;
+ struct iproc_pcie *pcie = sys->private_data;
+ struct bcma_device *bdev = container_of(pcie->dev, struct bcma_device, dev);
+
+ return bcma_core_irq(bdev, 5);
+}
+
+static int iproc_pcie_bcma_probe(struct bcma_device *bdev)
+{
+ struct device *dev = &bdev->dev;
+ struct iproc_pcie *pcie;
+ LIST_HEAD(resources);
+ struct pci_host_bridge *bridge;
+ int ret;
+
+ bridge = devm_pci_alloc_host_bridge(dev, sizeof(*pcie));
+ if (!bridge)
+ return -ENOMEM;
+
+ pcie = pci_host_bridge_priv(bridge);
+
+ pcie->dev = dev;
+
+ pcie->type = IPROC_PCIE_PAXB_BCMA;
+ pcie->base = bdev->io_addr;
+ if (!pcie->base) {
+ dev_err(dev, "no controller registers\n");
+ return -ENOMEM;
+ }
+
+ pcie->base_addr = bdev->addr;
+
+ pcie->mem.start = bdev->addr_s[0];
+ pcie->mem.end = bdev->addr_s[0] + SZ_128M - 1;
+ pcie->mem.name = "PCIe MEM space";
+ pcie->mem.flags = IORESOURCE_MEM;
+ pci_add_resource(&resources, &pcie->mem);
+
+ pcie->map_irq = iproc_pcie_bcma_map_irq;
+
+ ret = iproc_pcie_setup(pcie, &resources);
+ if (ret) {
+ dev_err(dev, "PCIe controller setup failed\n");
+ pci_free_resource_list(&resources);
+ return ret;
+ }
+
+ bcma_set_drvdata(bdev, pcie);
+ return 0;
+}
+
+static void iproc_pcie_bcma_remove(struct bcma_device *bdev)
+{
+ struct iproc_pcie *pcie = bcma_get_drvdata(bdev);
+
+ iproc_pcie_remove(pcie);
+}
+
+static const struct bcma_device_id iproc_pcie_bcma_table[] = {
+ BCMA_CORE(BCMA_MANUF_BCM, BCMA_CORE_NS_PCIEG2, BCMA_ANY_REV, BCMA_ANY_CLASS),
+ {},
+};
+MODULE_DEVICE_TABLE(bcma, iproc_pcie_bcma_table);
+
+static struct bcma_driver iproc_pcie_bcma_driver = {
+ .name = KBUILD_MODNAME,
+ .id_table = iproc_pcie_bcma_table,
+ .probe = iproc_pcie_bcma_probe,
+ .remove = iproc_pcie_bcma_remove,
+};
+
+static int __init iproc_pcie_bcma_init(void)
+{
+ return bcma_driver_register(&iproc_pcie_bcma_driver);
+}
+module_init(iproc_pcie_bcma_init);
+
+static void __exit iproc_pcie_bcma_exit(void)
+{
+ bcma_driver_unregister(&iproc_pcie_bcma_driver);
+}
+module_exit(iproc_pcie_bcma_exit);
+
+MODULE_AUTHOR("Hauke Mehrtens");
+MODULE_DESCRIPTION("Broadcom iProc PCIe BCMA driver");
+MODULE_LICENSE("GPL v2");
diff --git a/src/kernel/linux/v4.14/drivers/pci/host/pcie-iproc-msi.c b/src/kernel/linux/v4.14/drivers/pci/host/pcie-iproc-msi.c
new file mode 100644
index 0000000..2d0f535
--- /dev/null
+++ b/src/kernel/linux/v4.14/drivers/pci/host/pcie-iproc-msi.c
@@ -0,0 +1,674 @@
+/*
+ * Copyright (C) 2015 Broadcom Corporation
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation version 2.
+ *
+ * This program is distributed "as is" WITHOUT ANY WARRANTY of any
+ * kind, whether express or implied; without even the implied warranty
+ * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/interrupt.h>
+#include <linux/irqchip/chained_irq.h>
+#include <linux/irqdomain.h>
+#include <linux/msi.h>
+#include <linux/of_irq.h>
+#include <linux/of_pci.h>
+#include <linux/pci.h>
+
+#include "pcie-iproc.h"
+
+#define IPROC_MSI_INTR_EN_SHIFT 11
+#define IPROC_MSI_INTR_EN BIT(IPROC_MSI_INTR_EN_SHIFT)
+#define IPROC_MSI_INT_N_EVENT_SHIFT 1
+#define IPROC_MSI_INT_N_EVENT BIT(IPROC_MSI_INT_N_EVENT_SHIFT)
+#define IPROC_MSI_EQ_EN_SHIFT 0
+#define IPROC_MSI_EQ_EN BIT(IPROC_MSI_EQ_EN_SHIFT)
+
+#define IPROC_MSI_EQ_MASK 0x3f
+
+/* Max number of GIC interrupts */
+#define NR_HW_IRQS 6
+
+/* Number of entries in each event queue */
+#define EQ_LEN 64
+
+/* Size of each event queue memory region */
+#define EQ_MEM_REGION_SIZE SZ_4K
+
+/* Size of each MSI address region */
+#define MSI_MEM_REGION_SIZE SZ_4K
+
+enum iproc_msi_reg {
+ IPROC_MSI_EQ_PAGE = 0,
+ IPROC_MSI_EQ_PAGE_UPPER,
+ IPROC_MSI_PAGE,
+ IPROC_MSI_PAGE_UPPER,
+ IPROC_MSI_CTRL,
+ IPROC_MSI_EQ_HEAD,
+ IPROC_MSI_EQ_TAIL,
+ IPROC_MSI_INTS_EN,
+ IPROC_MSI_REG_SIZE,
+};
+
+struct iproc_msi;
+
+/**
+ * iProc MSI group
+ *
+ * One MSI group is allocated per GIC interrupt, serviced by one iProc MSI
+ * event queue.
+ *
+ * @msi: pointer to iProc MSI data
+ * @gic_irq: GIC interrupt
+ * @eq: Event queue number
+ */
+struct iproc_msi_grp {
+ struct iproc_msi *msi;
+ int gic_irq;
+ unsigned int eq;
+};
+
+/**
+ * iProc event queue based MSI
+ *
+ * Only meant to be used on platforms without MSI support integrated into the
+ * GIC.
+ *
+ * @pcie: pointer to iProc PCIe data
+ * @reg_offsets: MSI register offsets
+ * @grps: MSI groups
+ * @nr_irqs: number of total interrupts connected to GIC
+ * @nr_cpus: number of toal CPUs
+ * @has_inten_reg: indicates the MSI interrupt enable register needs to be
+ * set explicitly (required for some legacy platforms)
+ * @bitmap: MSI vector bitmap
+ * @bitmap_lock: lock to protect access to the MSI bitmap
+ * @nr_msi_vecs: total number of MSI vectors
+ * @inner_domain: inner IRQ domain
+ * @msi_domain: MSI IRQ domain
+ * @nr_eq_region: required number of 4K aligned memory region for MSI event
+ * queues
+ * @nr_msi_region: required number of 4K aligned address region for MSI posted
+ * writes
+ * @eq_cpu: pointer to allocated memory region for MSI event queues
+ * @eq_dma: DMA address of MSI event queues
+ * @msi_addr: MSI address
+ */
+struct iproc_msi {
+ struct iproc_pcie *pcie;
+ const u16 (*reg_offsets)[IPROC_MSI_REG_SIZE];
+ struct iproc_msi_grp *grps;
+ int nr_irqs;
+ int nr_cpus;
+ bool has_inten_reg;
+ unsigned long *bitmap;
+ struct mutex bitmap_lock;
+ unsigned int nr_msi_vecs;
+ struct irq_domain *inner_domain;
+ struct irq_domain *msi_domain;
+ unsigned int nr_eq_region;
+ unsigned int nr_msi_region;
+ void *eq_cpu;
+ dma_addr_t eq_dma;
+ phys_addr_t msi_addr;
+};
+
+static const u16 iproc_msi_reg_paxb[NR_HW_IRQS][IPROC_MSI_REG_SIZE] = {
+ { 0x200, 0x2c0, 0x204, 0x2c4, 0x210, 0x250, 0x254, 0x208 },
+ { 0x200, 0x2c0, 0x204, 0x2c4, 0x214, 0x258, 0x25c, 0x208 },
+ { 0x200, 0x2c0, 0x204, 0x2c4, 0x218, 0x260, 0x264, 0x208 },
+ { 0x200, 0x2c0, 0x204, 0x2c4, 0x21c, 0x268, 0x26c, 0x208 },
+ { 0x200, 0x2c0, 0x204, 0x2c4, 0x220, 0x270, 0x274, 0x208 },
+ { 0x200, 0x2c0, 0x204, 0x2c4, 0x224, 0x278, 0x27c, 0x208 },
+};
+
+static const u16 iproc_msi_reg_paxc[NR_HW_IRQS][IPROC_MSI_REG_SIZE] = {
+ { 0xc00, 0xc04, 0xc08, 0xc0c, 0xc40, 0xc50, 0xc60 },
+ { 0xc10, 0xc14, 0xc18, 0xc1c, 0xc44, 0xc54, 0xc64 },
+ { 0xc20, 0xc24, 0xc28, 0xc2c, 0xc48, 0xc58, 0xc68 },
+ { 0xc30, 0xc34, 0xc38, 0xc3c, 0xc4c, 0xc5c, 0xc6c },
+};
+
+static inline u32 iproc_msi_read_reg(struct iproc_msi *msi,
+ enum iproc_msi_reg reg,
+ unsigned int eq)
+{
+ struct iproc_pcie *pcie = msi->pcie;
+
+ return readl_relaxed(pcie->base + msi->reg_offsets[eq][reg]);
+}
+
+static inline void iproc_msi_write_reg(struct iproc_msi *msi,
+ enum iproc_msi_reg reg,
+ int eq, u32 val)
+{
+ struct iproc_pcie *pcie = msi->pcie;
+
+ writel_relaxed(val, pcie->base + msi->reg_offsets[eq][reg]);
+}
+
+static inline u32 hwirq_to_group(struct iproc_msi *msi, unsigned long hwirq)
+{
+ return (hwirq % msi->nr_irqs);
+}
+
+static inline unsigned int iproc_msi_addr_offset(struct iproc_msi *msi,
+ unsigned long hwirq)
+{
+ if (msi->nr_msi_region > 1)
+ return hwirq_to_group(msi, hwirq) * MSI_MEM_REGION_SIZE;
+ else
+ return hwirq_to_group(msi, hwirq) * sizeof(u32);
+}
+
+static inline unsigned int iproc_msi_eq_offset(struct iproc_msi *msi, u32 eq)
+{
+ if (msi->nr_eq_region > 1)
+ return eq * EQ_MEM_REGION_SIZE;
+ else
+ return eq * EQ_LEN * sizeof(u32);
+}
+
+static struct irq_chip iproc_msi_irq_chip = {
+ .name = "iProc-MSI",
+};
+
+static struct msi_domain_info iproc_msi_domain_info = {
+ .flags = MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS |
+ MSI_FLAG_PCI_MSIX,
+ .chip = &iproc_msi_irq_chip,
+};
+
+/*
+ * In iProc PCIe core, each MSI group is serviced by a GIC interrupt and a
+ * dedicated event queue. Each MSI group can support up to 64 MSI vectors.
+ *
+ * The number of MSI groups varies between different iProc SoCs. The total
+ * number of CPU cores also varies. To support MSI IRQ affinity, we
+ * distribute GIC interrupts across all available CPUs. MSI vector is moved
+ * from one GIC interrupt to another to steer to the target CPU.
+ *
+ * Assuming:
+ * - the number of MSI groups is M
+ * - the number of CPU cores is N
+ * - M is always a multiple of N
+ *
+ * Total number of raw MSI vectors = M * 64
+ * Total number of supported MSI vectors = (M * 64) / N
+ */
+static inline int hwirq_to_cpu(struct iproc_msi *msi, unsigned long hwirq)
+{
+ return (hwirq % msi->nr_cpus);
+}
+
+static inline unsigned long hwirq_to_canonical_hwirq(struct iproc_msi *msi,
+ unsigned long hwirq)
+{
+ return (hwirq - hwirq_to_cpu(msi, hwirq));
+}
+
+static int iproc_msi_irq_set_affinity(struct irq_data *data,
+ const struct cpumask *mask, bool force)
+{
+ struct iproc_msi *msi = irq_data_get_irq_chip_data(data);
+ int target_cpu = cpumask_first(mask);
+ int curr_cpu;
+
+ curr_cpu = hwirq_to_cpu(msi, data->hwirq);
+ if (curr_cpu == target_cpu)
+ return IRQ_SET_MASK_OK_DONE;
+
+ /* steer MSI to the target CPU */
+ data->hwirq = hwirq_to_canonical_hwirq(msi, data->hwirq) + target_cpu;
+
+ return IRQ_SET_MASK_OK;
+}
+
+static void iproc_msi_irq_compose_msi_msg(struct irq_data *data,
+ struct msi_msg *msg)
+{
+ struct iproc_msi *msi = irq_data_get_irq_chip_data(data);
+ dma_addr_t addr;
+
+ addr = msi->msi_addr + iproc_msi_addr_offset(msi, data->hwirq);
+ msg->address_lo = lower_32_bits(addr);
+ msg->address_hi = upper_32_bits(addr);
+ msg->data = data->hwirq;
+}
+
+static struct irq_chip iproc_msi_bottom_irq_chip = {
+ .name = "MSI",
+ .irq_set_affinity = iproc_msi_irq_set_affinity,
+ .irq_compose_msi_msg = iproc_msi_irq_compose_msi_msg,
+};
+
+static int iproc_msi_irq_domain_alloc(struct irq_domain *domain,
+ unsigned int virq, unsigned int nr_irqs,
+ void *args)
+{
+ struct iproc_msi *msi = domain->host_data;
+ int hwirq;
+
+ mutex_lock(&msi->bitmap_lock);
+
+ /* Allocate 'nr_cpus' number of MSI vectors each time */
+ hwirq = bitmap_find_next_zero_area(msi->bitmap, msi->nr_msi_vecs, 0,
+ msi->nr_cpus, 0);
+ if (hwirq < msi->nr_msi_vecs) {
+ bitmap_set(msi->bitmap, hwirq, msi->nr_cpus);
+ } else {
+ mutex_unlock(&msi->bitmap_lock);
+ return -ENOSPC;
+ }
+
+ mutex_unlock(&msi->bitmap_lock);
+
+ irq_domain_set_info(domain, virq, hwirq, &iproc_msi_bottom_irq_chip,
+ domain->host_data, handle_simple_irq, NULL, NULL);
+
+ return 0;
+}
+
+static void iproc_msi_irq_domain_free(struct irq_domain *domain,
+ unsigned int virq, unsigned int nr_irqs)
+{
+ struct irq_data *data = irq_domain_get_irq_data(domain, virq);
+ struct iproc_msi *msi = irq_data_get_irq_chip_data(data);
+ unsigned int hwirq;
+
+ mutex_lock(&msi->bitmap_lock);
+
+ hwirq = hwirq_to_canonical_hwirq(msi, data->hwirq);
+ bitmap_clear(msi->bitmap, hwirq, msi->nr_cpus);
+
+ mutex_unlock(&msi->bitmap_lock);
+
+ irq_domain_free_irqs_parent(domain, virq, nr_irqs);
+}
+
+static const struct irq_domain_ops msi_domain_ops = {
+ .alloc = iproc_msi_irq_domain_alloc,
+ .free = iproc_msi_irq_domain_free,
+};
+
+static inline u32 decode_msi_hwirq(struct iproc_msi *msi, u32 eq, u32 head)
+{
+ u32 *msg, hwirq;
+ unsigned int offs;
+
+ offs = iproc_msi_eq_offset(msi, eq) + head * sizeof(u32);
+ msg = (u32 *)(msi->eq_cpu + offs);
+ hwirq = *msg & IPROC_MSI_EQ_MASK;
+
+ /*
+ * Since we have multiple hwirq mapped to a single MSI vector,
+ * now we need to derive the hwirq at CPU0. It can then be used to
+ * mapped back to virq.
+ */
+ return hwirq_to_canonical_hwirq(msi, hwirq);
+}
+
+static void iproc_msi_handler(struct irq_desc *desc)
+{
+ struct irq_chip *chip = irq_desc_get_chip(desc);
+ struct iproc_msi_grp *grp;
+ struct iproc_msi *msi;
+ u32 eq, head, tail, nr_events;
+ unsigned long hwirq;
+ int virq;
+
+ chained_irq_enter(chip, desc);
+
+ grp = irq_desc_get_handler_data(desc);
+ msi = grp->msi;
+ eq = grp->eq;
+
+ /*
+ * iProc MSI event queue is tracked by head and tail pointers. Head
+ * pointer indicates the next entry (MSI data) to be consumed by SW in
+ * the queue and needs to be updated by SW. iProc MSI core uses the
+ * tail pointer as the next data insertion point.
+ *
+ * Entries between head and tail pointers contain valid MSI data. MSI
+ * data is guaranteed to be in the event queue memory before the tail
+ * pointer is updated by the iProc MSI core.
+ */
+ head = iproc_msi_read_reg(msi, IPROC_MSI_EQ_HEAD,
+ eq) & IPROC_MSI_EQ_MASK;
+ do {
+ tail = iproc_msi_read_reg(msi, IPROC_MSI_EQ_TAIL,
+ eq) & IPROC_MSI_EQ_MASK;
+
+ /*
+ * Figure out total number of events (MSI data) to be
+ * processed.
+ */
+ nr_events = (tail < head) ?
+ (EQ_LEN - (head - tail)) : (tail - head);
+ if (!nr_events)
+ break;
+
+ /* process all outstanding events */
+ while (nr_events--) {
+ hwirq = decode_msi_hwirq(msi, eq, head);
+ virq = irq_find_mapping(msi->inner_domain, hwirq);
+ generic_handle_irq(virq);
+
+ head++;
+ head %= EQ_LEN;
+ }
+
+ /*
+ * Now all outstanding events have been processed. Update the
+ * head pointer.
+ */
+ iproc_msi_write_reg(msi, IPROC_MSI_EQ_HEAD, eq, head);
+
+ /*
+ * Now go read the tail pointer again to see if there are new
+ * oustanding events that came in during the above window.
+ */
+ } while (true);
+
+ chained_irq_exit(chip, desc);
+}
+
+static void iproc_msi_enable(struct iproc_msi *msi)
+{
+ int i, eq;
+ u32 val;
+
+ /* Program memory region for each event queue */
+ for (i = 0; i < msi->nr_eq_region; i++) {
+ dma_addr_t addr = msi->eq_dma + (i * EQ_MEM_REGION_SIZE);
+
+ iproc_msi_write_reg(msi, IPROC_MSI_EQ_PAGE, i,
+ lower_32_bits(addr));
+ iproc_msi_write_reg(msi, IPROC_MSI_EQ_PAGE_UPPER, i,
+ upper_32_bits(addr));
+ }
+
+ /* Program address region for MSI posted writes */
+ for (i = 0; i < msi->nr_msi_region; i++) {
+ phys_addr_t addr = msi->msi_addr + (i * MSI_MEM_REGION_SIZE);
+
+ iproc_msi_write_reg(msi, IPROC_MSI_PAGE, i,
+ lower_32_bits(addr));
+ iproc_msi_write_reg(msi, IPROC_MSI_PAGE_UPPER, i,
+ upper_32_bits(addr));
+ }
+
+ for (eq = 0; eq < msi->nr_irqs; eq++) {
+ /* Enable MSI event queue */
+ val = IPROC_MSI_INTR_EN | IPROC_MSI_INT_N_EVENT |
+ IPROC_MSI_EQ_EN;
+ iproc_msi_write_reg(msi, IPROC_MSI_CTRL, eq, val);
+
+ /*
+ * Some legacy platforms require the MSI interrupt enable
+ * register to be set explicitly.
+ */
+ if (msi->has_inten_reg) {
+ val = iproc_msi_read_reg(msi, IPROC_MSI_INTS_EN, eq);
+ val |= BIT(eq);
+ iproc_msi_write_reg(msi, IPROC_MSI_INTS_EN, eq, val);
+ }
+ }
+}
+
+static void iproc_msi_disable(struct iproc_msi *msi)
+{
+ u32 eq, val;
+
+ for (eq = 0; eq < msi->nr_irqs; eq++) {
+ if (msi->has_inten_reg) {
+ val = iproc_msi_read_reg(msi, IPROC_MSI_INTS_EN, eq);
+ val &= ~BIT(eq);
+ iproc_msi_write_reg(msi, IPROC_MSI_INTS_EN, eq, val);
+ }
+
+ val = iproc_msi_read_reg(msi, IPROC_MSI_CTRL, eq);
+ val &= ~(IPROC_MSI_INTR_EN | IPROC_MSI_INT_N_EVENT |
+ IPROC_MSI_EQ_EN);
+ iproc_msi_write_reg(msi, IPROC_MSI_CTRL, eq, val);
+ }
+}
+
+static int iproc_msi_alloc_domains(struct device_node *node,
+ struct iproc_msi *msi)
+{
+ msi->inner_domain = irq_domain_add_linear(NULL, msi->nr_msi_vecs,
+ &msi_domain_ops, msi);
+ if (!msi->inner_domain)
+ return -ENOMEM;
+
+ msi->msi_domain = pci_msi_create_irq_domain(of_node_to_fwnode(node),
+ &iproc_msi_domain_info,
+ msi->inner_domain);
+ if (!msi->msi_domain) {
+ irq_domain_remove(msi->inner_domain);
+ return -ENOMEM;
+ }
+
+ return 0;
+}
+
+static void iproc_msi_free_domains(struct iproc_msi *msi)
+{
+ if (msi->msi_domain)
+ irq_domain_remove(msi->msi_domain);
+
+ if (msi->inner_domain)
+ irq_domain_remove(msi->inner_domain);
+}
+
+static void iproc_msi_irq_free(struct iproc_msi *msi, unsigned int cpu)
+{
+ int i;
+
+ for (i = cpu; i < msi->nr_irqs; i += msi->nr_cpus) {
+ irq_set_chained_handler_and_data(msi->grps[i].gic_irq,
+ NULL, NULL);
+ }
+}
+
+static int iproc_msi_irq_setup(struct iproc_msi *msi, unsigned int cpu)
+{
+ int i, ret;
+ cpumask_var_t mask;
+ struct iproc_pcie *pcie = msi->pcie;
+
+ for (i = cpu; i < msi->nr_irqs; i += msi->nr_cpus) {
+ irq_set_chained_handler_and_data(msi->grps[i].gic_irq,
+ iproc_msi_handler,
+ &msi->grps[i]);
+ /* Dedicate GIC interrupt to each CPU core */
+ if (alloc_cpumask_var(&mask, GFP_KERNEL)) {
+ cpumask_clear(mask);
+ cpumask_set_cpu(cpu, mask);
+ ret = irq_set_affinity(msi->grps[i].gic_irq, mask);
+ if (ret)
+ dev_err(pcie->dev,
+ "failed to set affinity for IRQ%d\n",
+ msi->grps[i].gic_irq);
+ free_cpumask_var(mask);
+ } else {
+ dev_err(pcie->dev, "failed to alloc CPU mask\n");
+ ret = -EINVAL;
+ }
+
+ if (ret) {
+ /* Free all configured/unconfigured IRQs */
+ iproc_msi_irq_free(msi, cpu);
+ return ret;
+ }
+ }
+
+ return 0;
+}
+
+int iproc_msi_init(struct iproc_pcie *pcie, struct device_node *node)
+{
+ struct iproc_msi *msi;
+ int i, ret;
+ unsigned int cpu;
+
+ if (!of_device_is_compatible(node, "brcm,iproc-msi"))
+ return -ENODEV;
+
+ if (!of_find_property(node, "msi-controller", NULL))
+ return -ENODEV;
+
+ if (pcie->msi)
+ return -EBUSY;
+
+ msi = devm_kzalloc(pcie->dev, sizeof(*msi), GFP_KERNEL);
+ if (!msi)
+ return -ENOMEM;
+
+ msi->pcie = pcie;
+ pcie->msi = msi;
+ msi->msi_addr = pcie->base_addr;
+ mutex_init(&msi->bitmap_lock);
+ msi->nr_cpus = num_possible_cpus();
+
+ msi->nr_irqs = of_irq_count(node);
+ if (!msi->nr_irqs) {
+ dev_err(pcie->dev, "found no MSI GIC interrupt\n");
+ return -ENODEV;
+ }
+
+ if (msi->nr_irqs > NR_HW_IRQS) {
+ dev_warn(pcie->dev, "too many MSI GIC interrupts defined %d\n",
+ msi->nr_irqs);
+ msi->nr_irqs = NR_HW_IRQS;
+ }
+
+ if (msi->nr_irqs < msi->nr_cpus) {
+ dev_err(pcie->dev,
+ "not enough GIC interrupts for MSI affinity\n");
+ return -EINVAL;
+ }
+
+ if (msi->nr_irqs % msi->nr_cpus != 0) {
+ msi->nr_irqs -= msi->nr_irqs % msi->nr_cpus;
+ dev_warn(pcie->dev, "Reducing number of interrupts to %d\n",
+ msi->nr_irqs);
+ }
+
+ switch (pcie->type) {
+ case IPROC_PCIE_PAXB_BCMA:
+ case IPROC_PCIE_PAXB:
+ msi->reg_offsets = iproc_msi_reg_paxb;
+ msi->nr_eq_region = 1;
+ msi->nr_msi_region = 1;
+ break;
+ case IPROC_PCIE_PAXC:
+ msi->reg_offsets = iproc_msi_reg_paxc;
+ msi->nr_eq_region = msi->nr_irqs;
+ msi->nr_msi_region = msi->nr_irqs;
+ break;
+ default:
+ dev_err(pcie->dev, "incompatible iProc PCIe interface\n");
+ return -EINVAL;
+ }
+
+ if (of_find_property(node, "brcm,pcie-msi-inten", NULL))
+ msi->has_inten_reg = true;
+
+ msi->nr_msi_vecs = msi->nr_irqs * EQ_LEN;
+ msi->bitmap = devm_kcalloc(pcie->dev, BITS_TO_LONGS(msi->nr_msi_vecs),
+ sizeof(*msi->bitmap), GFP_KERNEL);
+ if (!msi->bitmap)
+ return -ENOMEM;
+
+ msi->grps = devm_kcalloc(pcie->dev, msi->nr_irqs, sizeof(*msi->grps),
+ GFP_KERNEL);
+ if (!msi->grps)
+ return -ENOMEM;
+
+ for (i = 0; i < msi->nr_irqs; i++) {
+ unsigned int irq = irq_of_parse_and_map(node, i);
+
+ if (!irq) {
+ dev_err(pcie->dev, "unable to parse/map interrupt\n");
+ ret = -ENODEV;
+ goto free_irqs;
+ }
+ msi->grps[i].gic_irq = irq;
+ msi->grps[i].msi = msi;
+ msi->grps[i].eq = i;
+ }
+
+ /* Reserve memory for event queue and make sure memories are zeroed */
+ msi->eq_cpu = dma_zalloc_coherent(pcie->dev,
+ msi->nr_eq_region * EQ_MEM_REGION_SIZE,
+ &msi->eq_dma, GFP_KERNEL);
+ if (!msi->eq_cpu) {
+ ret = -ENOMEM;
+ goto free_irqs;
+ }
+
+ ret = iproc_msi_alloc_domains(node, msi);
+ if (ret) {
+ dev_err(pcie->dev, "failed to create MSI domains\n");
+ goto free_eq_dma;
+ }
+
+ for_each_online_cpu(cpu) {
+ ret = iproc_msi_irq_setup(msi, cpu);
+ if (ret)
+ goto free_msi_irq;
+ }
+
+ iproc_msi_enable(msi);
+
+ return 0;
+
+free_msi_irq:
+ for_each_online_cpu(cpu)
+ iproc_msi_irq_free(msi, cpu);
+ iproc_msi_free_domains(msi);
+
+free_eq_dma:
+ dma_free_coherent(pcie->dev, msi->nr_eq_region * EQ_MEM_REGION_SIZE,
+ msi->eq_cpu, msi->eq_dma);
+
+free_irqs:
+ for (i = 0; i < msi->nr_irqs; i++) {
+ if (msi->grps[i].gic_irq)
+ irq_dispose_mapping(msi->grps[i].gic_irq);
+ }
+ pcie->msi = NULL;
+ return ret;
+}
+EXPORT_SYMBOL(iproc_msi_init);
+
+void iproc_msi_exit(struct iproc_pcie *pcie)
+{
+ struct iproc_msi *msi = pcie->msi;
+ unsigned int i, cpu;
+
+ if (!msi)
+ return;
+
+ iproc_msi_disable(msi);
+
+ for_each_online_cpu(cpu)
+ iproc_msi_irq_free(msi, cpu);
+
+ iproc_msi_free_domains(msi);
+
+ dma_free_coherent(pcie->dev, msi->nr_eq_region * EQ_MEM_REGION_SIZE,
+ msi->eq_cpu, msi->eq_dma);
+
+ for (i = 0; i < msi->nr_irqs; i++) {
+ if (msi->grps[i].gic_irq)
+ irq_dispose_mapping(msi->grps[i].gic_irq);
+ }
+}
+EXPORT_SYMBOL(iproc_msi_exit);
diff --git a/src/kernel/linux/v4.14/drivers/pci/host/pcie-iproc-platform.c b/src/kernel/linux/v4.14/drivers/pci/host/pcie-iproc-platform.c
new file mode 100644
index 0000000..32228d4
--- /dev/null
+++ b/src/kernel/linux/v4.14/drivers/pci/host/pcie-iproc-platform.c
@@ -0,0 +1,164 @@
+/*
+ * Copyright (C) 2015 Broadcom Corporation
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation version 2.
+ *
+ * This program is distributed "as is" WITHOUT ANY WARRANTY of any
+ * kind, whether express or implied; without even the implied warranty
+ * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/kernel.h>
+#include <linux/pci.h>
+#include <linux/clk.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/interrupt.h>
+#include <linux/platform_device.h>
+#include <linux/of_address.h>
+#include <linux/of_pci.h>
+#include <linux/of_irq.h>
+#include <linux/of_platform.h>
+#include <linux/phy/phy.h>
+
+#include "pcie-iproc.h"
+
+static const struct of_device_id iproc_pcie_of_match_table[] = {
+ {
+ .compatible = "brcm,iproc-pcie",
+ .data = (int *)IPROC_PCIE_PAXB,
+ }, {
+ .compatible = "brcm,iproc-pcie-paxb-v2",
+ .data = (int *)IPROC_PCIE_PAXB_V2,
+ }, {
+ .compatible = "brcm,iproc-pcie-paxc",
+ .data = (int *)IPROC_PCIE_PAXC,
+ }, {
+ .compatible = "brcm,iproc-pcie-paxc-v2",
+ .data = (int *)IPROC_PCIE_PAXC_V2,
+ },
+ { /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, iproc_pcie_of_match_table);
+
+static int iproc_pcie_pltfm_probe(struct platform_device *pdev)
+{
+ struct device *dev = &pdev->dev;
+ struct iproc_pcie *pcie;
+ struct device_node *np = dev->of_node;
+ struct resource reg;
+ resource_size_t iobase = 0;
+ LIST_HEAD(resources);
+ struct pci_host_bridge *bridge;
+ int ret;
+
+ bridge = devm_pci_alloc_host_bridge(dev, sizeof(*pcie));
+ if (!bridge)
+ return -ENOMEM;
+
+ pcie = pci_host_bridge_priv(bridge);
+
+ pcie->dev = dev;
+ pcie->type = (enum iproc_pcie_type) of_device_get_match_data(dev);
+
+ ret = of_address_to_resource(np, 0, ®);
+ if (ret < 0) {
+ dev_err(dev, "unable to obtain controller resources\n");
+ return ret;
+ }
+
+ pcie->base = devm_pci_remap_cfgspace(dev, reg.start,
+ resource_size(®));
+ if (!pcie->base) {
+ dev_err(dev, "unable to map controller registers\n");
+ return -ENOMEM;
+ }
+ pcie->base_addr = reg.start;
+
+ if (of_property_read_bool(np, "brcm,pcie-ob")) {
+ u32 val;
+
+ ret = of_property_read_u32(np, "brcm,pcie-ob-axi-offset",
+ &val);
+ if (ret) {
+ dev_err(dev,
+ "missing brcm,pcie-ob-axi-offset property\n");
+ return ret;
+ }
+ pcie->ob.axi_offset = val;
+ pcie->need_ob_cfg = true;
+ }
+
+ /*
+ * DT nodes are not used by all platforms that use the iProc PCIe
+ * core driver. For platforms that require explict inbound mapping
+ * configuration, "dma-ranges" would have been present in DT
+ */
+ pcie->need_ib_cfg = of_property_read_bool(np, "dma-ranges");
+
+ /* PHY use is optional */
+ pcie->phy = devm_phy_get(dev, "pcie-phy");
+ if (IS_ERR(pcie->phy)) {
+ if (PTR_ERR(pcie->phy) == -EPROBE_DEFER)
+ return -EPROBE_DEFER;
+ pcie->phy = NULL;
+ }
+
+ ret = of_pci_get_host_bridge_resources(np, 0, 0xff, &resources,
+ &iobase);
+ if (ret) {
+ dev_err(dev, "unable to get PCI host bridge resources\n");
+ return ret;
+ }
+
+ /* PAXC doesn't support legacy IRQs, skip mapping */
+ switch (pcie->type) {
+ case IPROC_PCIE_PAXC:
+ case IPROC_PCIE_PAXC_V2:
+ break;
+ default:
+ pcie->map_irq = of_irq_parse_and_map_pci;
+ }
+
+ ret = iproc_pcie_setup(pcie, &resources);
+ if (ret) {
+ dev_err(dev, "PCIe controller setup failed\n");
+ pci_free_resource_list(&resources);
+ return ret;
+ }
+
+ platform_set_drvdata(pdev, pcie);
+ return 0;
+}
+
+static int iproc_pcie_pltfm_remove(struct platform_device *pdev)
+{
+ struct iproc_pcie *pcie = platform_get_drvdata(pdev);
+
+ return iproc_pcie_remove(pcie);
+}
+
+static void iproc_pcie_pltfm_shutdown(struct platform_device *pdev)
+{
+ struct iproc_pcie *pcie = platform_get_drvdata(pdev);
+
+ iproc_pcie_shutdown(pcie);
+}
+
+static struct platform_driver iproc_pcie_pltfm_driver = {
+ .driver = {
+ .name = "iproc-pcie",
+ .of_match_table = of_match_ptr(iproc_pcie_of_match_table),
+ },
+ .probe = iproc_pcie_pltfm_probe,
+ .remove = iproc_pcie_pltfm_remove,
+ .shutdown = iproc_pcie_pltfm_shutdown,
+};
+module_platform_driver(iproc_pcie_pltfm_driver);
+
+MODULE_AUTHOR("Ray Jui <rjui@broadcom.com>");
+MODULE_DESCRIPTION("Broadcom iPROC PCIe platform driver");
+MODULE_LICENSE("GPL v2");
diff --git a/src/kernel/linux/v4.14/drivers/pci/host/pcie-iproc.c b/src/kernel/linux/v4.14/drivers/pci/host/pcie-iproc.c
new file mode 100644
index 0000000..8f8dac0
--- /dev/null
+++ b/src/kernel/linux/v4.14/drivers/pci/host/pcie-iproc.c
@@ -0,0 +1,1465 @@
+/*
+ * Copyright (C) 2014 Hauke Mehrtens <hauke@hauke-m.de>
+ * Copyright (C) 2015 Broadcom Corporation
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation version 2.
+ *
+ * This program is distributed "as is" WITHOUT ANY WARRANTY of any
+ * kind, whether express or implied; without even the implied warranty
+ * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/kernel.h>
+#include <linux/pci.h>
+#include <linux/msi.h>
+#include <linux/clk.h>
+#include <linux/module.h>
+#include <linux/mbus.h>
+#include <linux/slab.h>
+#include <linux/delay.h>
+#include <linux/interrupt.h>
+#include <linux/irqchip/arm-gic-v3.h>
+#include <linux/platform_device.h>
+#include <linux/of_address.h>
+#include <linux/of_pci.h>
+#include <linux/of_irq.h>
+#include <linux/of_platform.h>
+#include <linux/phy/phy.h>
+
+#include "pcie-iproc.h"
+
+#define EP_PERST_SOURCE_SELECT_SHIFT 2
+#define EP_PERST_SOURCE_SELECT BIT(EP_PERST_SOURCE_SELECT_SHIFT)
+#define EP_MODE_SURVIVE_PERST_SHIFT 1
+#define EP_MODE_SURVIVE_PERST BIT(EP_MODE_SURVIVE_PERST_SHIFT)
+#define RC_PCIE_RST_OUTPUT_SHIFT 0
+#define RC_PCIE_RST_OUTPUT BIT(RC_PCIE_RST_OUTPUT_SHIFT)
+#define PAXC_RESET_MASK 0x7f
+
+#define GIC_V3_CFG_SHIFT 0
+#define GIC_V3_CFG BIT(GIC_V3_CFG_SHIFT)
+
+#define MSI_ENABLE_CFG_SHIFT 0
+#define MSI_ENABLE_CFG BIT(MSI_ENABLE_CFG_SHIFT)
+
+#define CFG_IND_ADDR_MASK 0x00001ffc
+
+#define CFG_ADDR_BUS_NUM_SHIFT 20
+#define CFG_ADDR_BUS_NUM_MASK 0x0ff00000
+#define CFG_ADDR_DEV_NUM_SHIFT 15
+#define CFG_ADDR_DEV_NUM_MASK 0x000f8000
+#define CFG_ADDR_FUNC_NUM_SHIFT 12
+#define CFG_ADDR_FUNC_NUM_MASK 0x00007000
+#define CFG_ADDR_REG_NUM_SHIFT 2
+#define CFG_ADDR_REG_NUM_MASK 0x00000ffc
+#define CFG_ADDR_CFG_TYPE_SHIFT 0
+#define CFG_ADDR_CFG_TYPE_MASK 0x00000003
+
+#define SYS_RC_INTX_MASK 0xf
+
+#define PCIE_PHYLINKUP_SHIFT 3
+#define PCIE_PHYLINKUP BIT(PCIE_PHYLINKUP_SHIFT)
+#define PCIE_DL_ACTIVE_SHIFT 2
+#define PCIE_DL_ACTIVE BIT(PCIE_DL_ACTIVE_SHIFT)
+
+#define APB_ERR_EN_SHIFT 0
+#define APB_ERR_EN BIT(APB_ERR_EN_SHIFT)
+
+#define CFG_RETRY_STATUS 0xffff0001
+#define CFG_RETRY_STATUS_TIMEOUT_US 500000 /* 500 milliseconds */
+
+/* derive the enum index of the outbound/inbound mapping registers */
+#define MAP_REG(base_reg, index) ((base_reg) + (index) * 2)
+
+/*
+ * Maximum number of outbound mapping window sizes that can be supported by any
+ * OARR/OMAP mapping pair
+ */
+#define MAX_NUM_OB_WINDOW_SIZES 4
+
+#define OARR_VALID_SHIFT 0
+#define OARR_VALID BIT(OARR_VALID_SHIFT)
+#define OARR_SIZE_CFG_SHIFT 1
+
+/*
+ * Maximum number of inbound mapping region sizes that can be supported by an
+ * IARR
+ */
+#define MAX_NUM_IB_REGION_SIZES 9
+
+#define IMAP_VALID_SHIFT 0
+#define IMAP_VALID BIT(IMAP_VALID_SHIFT)
+
+#define IPROC_PCI_EXP_CAP 0xac
+
+#define IPROC_PCIE_REG_INVALID 0xffff
+
+/**
+ * iProc PCIe outbound mapping controller specific parameters
+ *
+ * @window_sizes: list of supported outbound mapping window sizes in MB
+ * @nr_sizes: number of supported outbound mapping window sizes
+ */
+struct iproc_pcie_ob_map {
+ resource_size_t window_sizes[MAX_NUM_OB_WINDOW_SIZES];
+ unsigned int nr_sizes;
+};
+
+static const struct iproc_pcie_ob_map paxb_ob_map[] = {
+ {
+ /* OARR0/OMAP0 */
+ .window_sizes = { 128, 256 },
+ .nr_sizes = 2,
+ },
+ {
+ /* OARR1/OMAP1 */
+ .window_sizes = { 128, 256 },
+ .nr_sizes = 2,
+ },
+};
+
+static const struct iproc_pcie_ob_map paxb_v2_ob_map[] = {
+ {
+ /* OARR0/OMAP0 */
+ .window_sizes = { 128, 256 },
+ .nr_sizes = 2,
+ },
+ {
+ /* OARR1/OMAP1 */
+ .window_sizes = { 128, 256 },
+ .nr_sizes = 2,
+ },
+ {
+ /* OARR2/OMAP2 */
+ .window_sizes = { 128, 256, 512, 1024 },
+ .nr_sizes = 4,
+ },
+ {
+ /* OARR3/OMAP3 */
+ .window_sizes = { 128, 256, 512, 1024 },
+ .nr_sizes = 4,
+ },
+};
+
+/**
+ * iProc PCIe inbound mapping type
+ */
+enum iproc_pcie_ib_map_type {
+ /* for DDR memory */
+ IPROC_PCIE_IB_MAP_MEM = 0,
+
+ /* for device I/O memory */
+ IPROC_PCIE_IB_MAP_IO,
+
+ /* invalid or unused */
+ IPROC_PCIE_IB_MAP_INVALID
+};
+
+/**
+ * iProc PCIe inbound mapping controller specific parameters
+ *
+ * @type: inbound mapping region type
+ * @size_unit: inbound mapping region size unit, could be SZ_1K, SZ_1M, or
+ * SZ_1G
+ * @region_sizes: list of supported inbound mapping region sizes in KB, MB, or
+ * GB, depedning on the size unit
+ * @nr_sizes: number of supported inbound mapping region sizes
+ * @nr_windows: number of supported inbound mapping windows for the region
+ * @imap_addr_offset: register offset between the upper and lower 32-bit
+ * IMAP address registers
+ * @imap_window_offset: register offset between each IMAP window
+ */
+struct iproc_pcie_ib_map {
+ enum iproc_pcie_ib_map_type type;
+ unsigned int size_unit;
+ resource_size_t region_sizes[MAX_NUM_IB_REGION_SIZES];
+ unsigned int nr_sizes;
+ unsigned int nr_windows;
+ u16 imap_addr_offset;
+ u16 imap_window_offset;
+};
+
+static const struct iproc_pcie_ib_map paxb_v2_ib_map[] = {
+ {
+ /* IARR0/IMAP0 */
+ .type = IPROC_PCIE_IB_MAP_IO,
+ .size_unit = SZ_1K,
+ .region_sizes = { 32 },
+ .nr_sizes = 1,
+ .nr_windows = 8,
+ .imap_addr_offset = 0x40,
+ .imap_window_offset = 0x4,
+ },
+ {
+ /* IARR1/IMAP1 (currently unused) */
+ .type = IPROC_PCIE_IB_MAP_INVALID,
+ },
+ {
+ /* IARR2/IMAP2 */
+ .type = IPROC_PCIE_IB_MAP_MEM,
+ .size_unit = SZ_1M,
+ .region_sizes = { 64, 128, 256, 512, 1024, 2048, 4096, 8192,
+ 16384 },
+ .nr_sizes = 9,
+ .nr_windows = 1,
+ .imap_addr_offset = 0x4,
+ .imap_window_offset = 0x8,
+ },
+ {
+ /* IARR3/IMAP3 */
+ .type = IPROC_PCIE_IB_MAP_MEM,
+ .size_unit = SZ_1G,
+ .region_sizes = { 1, 2, 4, 8, 16, 32 },
+ .nr_sizes = 6,
+ .nr_windows = 8,
+ .imap_addr_offset = 0x4,
+ .imap_window_offset = 0x8,
+ },
+ {
+ /* IARR4/IMAP4 */
+ .type = IPROC_PCIE_IB_MAP_MEM,
+ .size_unit = SZ_1G,
+ .region_sizes = { 32, 64, 128, 256, 512 },
+ .nr_sizes = 5,
+ .nr_windows = 8,
+ .imap_addr_offset = 0x4,
+ .imap_window_offset = 0x8,
+ },
+};
+
+/*
+ * iProc PCIe host registers
+ */
+enum iproc_pcie_reg {
+ /* clock/reset signal control */
+ IPROC_PCIE_CLK_CTRL = 0,
+
+ /*
+ * To allow MSI to be steered to an external MSI controller (e.g., ARM
+ * GICv3 ITS)
+ */
+ IPROC_PCIE_MSI_GIC_MODE,
+
+ /*
+ * IPROC_PCIE_MSI_BASE_ADDR and IPROC_PCIE_MSI_WINDOW_SIZE define the
+ * window where the MSI posted writes are written, for the writes to be
+ * interpreted as MSI writes.
+ */
+ IPROC_PCIE_MSI_BASE_ADDR,
+ IPROC_PCIE_MSI_WINDOW_SIZE,
+
+ /*
+ * To hold the address of the register where the MSI writes are
+ * programed. When ARM GICv3 ITS is used, this should be programmed
+ * with the address of the GITS_TRANSLATER register.
+ */
+ IPROC_PCIE_MSI_ADDR_LO,
+ IPROC_PCIE_MSI_ADDR_HI,
+
+ /* enable MSI */
+ IPROC_PCIE_MSI_EN_CFG,
+
+ /* allow access to root complex configuration space */
+ IPROC_PCIE_CFG_IND_ADDR,
+ IPROC_PCIE_CFG_IND_DATA,
+
+ /* allow access to device configuration space */
+ IPROC_PCIE_CFG_ADDR,
+ IPROC_PCIE_CFG_DATA,
+
+ /* enable INTx */
+ IPROC_PCIE_INTX_EN,
+
+ /* outbound address mapping */
+ IPROC_PCIE_OARR0,
+ IPROC_PCIE_OMAP0,
+ IPROC_PCIE_OARR1,
+ IPROC_PCIE_OMAP1,
+ IPROC_PCIE_OARR2,
+ IPROC_PCIE_OMAP2,
+ IPROC_PCIE_OARR3,
+ IPROC_PCIE_OMAP3,
+
+ /* inbound address mapping */
+ IPROC_PCIE_IARR0,
+ IPROC_PCIE_IMAP0,
+ IPROC_PCIE_IARR1,
+ IPROC_PCIE_IMAP1,
+ IPROC_PCIE_IARR2,
+ IPROC_PCIE_IMAP2,
+ IPROC_PCIE_IARR3,
+ IPROC_PCIE_IMAP3,
+ IPROC_PCIE_IARR4,
+ IPROC_PCIE_IMAP4,
+
+ /* link status */
+ IPROC_PCIE_LINK_STATUS,
+
+ /* enable APB error for unsupported requests */
+ IPROC_PCIE_APB_ERR_EN,
+
+ /* total number of core registers */
+ IPROC_PCIE_MAX_NUM_REG,
+};
+
+/* iProc PCIe PAXB BCMA registers */
+static const u16 iproc_pcie_reg_paxb_bcma[] = {
+ [IPROC_PCIE_CLK_CTRL] = 0x000,
+ [IPROC_PCIE_CFG_IND_ADDR] = 0x120,
+ [IPROC_PCIE_CFG_IND_DATA] = 0x124,
+ [IPROC_PCIE_CFG_ADDR] = 0x1f8,
+ [IPROC_PCIE_CFG_DATA] = 0x1fc,
+ [IPROC_PCIE_INTX_EN] = 0x330,
+ [IPROC_PCIE_LINK_STATUS] = 0xf0c,
+};
+
+/* iProc PCIe PAXB registers */
+static const u16 iproc_pcie_reg_paxb[] = {
+ [IPROC_PCIE_CLK_CTRL] = 0x000,
+ [IPROC_PCIE_CFG_IND_ADDR] = 0x120,
+ [IPROC_PCIE_CFG_IND_DATA] = 0x124,
+ [IPROC_PCIE_CFG_ADDR] = 0x1f8,
+ [IPROC_PCIE_CFG_DATA] = 0x1fc,
+ [IPROC_PCIE_INTX_EN] = 0x330,
+ [IPROC_PCIE_OARR0] = 0xd20,
+ [IPROC_PCIE_OMAP0] = 0xd40,
+ [IPROC_PCIE_OARR1] = 0xd28,
+ [IPROC_PCIE_OMAP1] = 0xd48,
+ [IPROC_PCIE_LINK_STATUS] = 0xf0c,
+ [IPROC_PCIE_APB_ERR_EN] = 0xf40,
+};
+
+/* iProc PCIe PAXB v2 registers */
+static const u16 iproc_pcie_reg_paxb_v2[] = {
+ [IPROC_PCIE_CLK_CTRL] = 0x000,
+ [IPROC_PCIE_CFG_IND_ADDR] = 0x120,
+ [IPROC_PCIE_CFG_IND_DATA] = 0x124,
+ [IPROC_PCIE_CFG_ADDR] = 0x1f8,
+ [IPROC_PCIE_CFG_DATA] = 0x1fc,
+ [IPROC_PCIE_INTX_EN] = 0x330,
+ [IPROC_PCIE_OARR0] = 0xd20,
+ [IPROC_PCIE_OMAP0] = 0xd40,
+ [IPROC_PCIE_OARR1] = 0xd28,
+ [IPROC_PCIE_OMAP1] = 0xd48,
+ [IPROC_PCIE_OARR2] = 0xd60,
+ [IPROC_PCIE_OMAP2] = 0xd68,
+ [IPROC_PCIE_OARR3] = 0xdf0,
+ [IPROC_PCIE_OMAP3] = 0xdf8,
+ [IPROC_PCIE_IARR0] = 0xd00,
+ [IPROC_PCIE_IMAP0] = 0xc00,
+ [IPROC_PCIE_IARR2] = 0xd10,
+ [IPROC_PCIE_IMAP2] = 0xcc0,
+ [IPROC_PCIE_IARR3] = 0xe00,
+ [IPROC_PCIE_IMAP3] = 0xe08,
+ [IPROC_PCIE_IARR4] = 0xe68,
+ [IPROC_PCIE_IMAP4] = 0xe70,
+ [IPROC_PCIE_LINK_STATUS] = 0xf0c,
+ [IPROC_PCIE_APB_ERR_EN] = 0xf40,
+};
+
+/* iProc PCIe PAXC v1 registers */
+static const u16 iproc_pcie_reg_paxc[] = {
+ [IPROC_PCIE_CLK_CTRL] = 0x000,
+ [IPROC_PCIE_CFG_IND_ADDR] = 0x1f0,
+ [IPROC_PCIE_CFG_IND_DATA] = 0x1f4,
+ [IPROC_PCIE_CFG_ADDR] = 0x1f8,
+ [IPROC_PCIE_CFG_DATA] = 0x1fc,
+};
+
+/* iProc PCIe PAXC v2 registers */
+static const u16 iproc_pcie_reg_paxc_v2[] = {
+ [IPROC_PCIE_MSI_GIC_MODE] = 0x050,
+ [IPROC_PCIE_MSI_BASE_ADDR] = 0x074,
+ [IPROC_PCIE_MSI_WINDOW_SIZE] = 0x078,
+ [IPROC_PCIE_MSI_ADDR_LO] = 0x07c,
+ [IPROC_PCIE_MSI_ADDR_HI] = 0x080,
+ [IPROC_PCIE_MSI_EN_CFG] = 0x09c,
+ [IPROC_PCIE_CFG_IND_ADDR] = 0x1f0,
+ [IPROC_PCIE_CFG_IND_DATA] = 0x1f4,
+ [IPROC_PCIE_CFG_ADDR] = 0x1f8,
+ [IPROC_PCIE_CFG_DATA] = 0x1fc,
+};
+
+static inline struct iproc_pcie *iproc_data(struct pci_bus *bus)
+{
+ struct iproc_pcie *pcie;
+#ifdef CONFIG_ARM
+ struct pci_sys_data *sys = bus->sysdata;
+
+ pcie = sys->private_data;
+#else
+ pcie = bus->sysdata;
+#endif
+ return pcie;
+}
+
+static inline bool iproc_pcie_reg_is_invalid(u16 reg_offset)
+{
+ return !!(reg_offset == IPROC_PCIE_REG_INVALID);
+}
+
+static inline u16 iproc_pcie_reg_offset(struct iproc_pcie *pcie,
+ enum iproc_pcie_reg reg)
+{
+ return pcie->reg_offsets[reg];
+}
+
+static inline u32 iproc_pcie_read_reg(struct iproc_pcie *pcie,
+ enum iproc_pcie_reg reg)
+{
+ u16 offset = iproc_pcie_reg_offset(pcie, reg);
+
+ if (iproc_pcie_reg_is_invalid(offset))
+ return 0;
+
+ return readl(pcie->base + offset);
+}
+
+static inline void iproc_pcie_write_reg(struct iproc_pcie *pcie,
+ enum iproc_pcie_reg reg, u32 val)
+{
+ u16 offset = iproc_pcie_reg_offset(pcie, reg);
+
+ if (iproc_pcie_reg_is_invalid(offset))
+ return;
+
+ writel(val, pcie->base + offset);
+}
+
+/**
+ * APB error forwarding can be disabled during access of configuration
+ * registers of the endpoint device, to prevent unsupported requests
+ * (typically seen during enumeration with multi-function devices) from
+ * triggering a system exception.
+ */
+static inline void iproc_pcie_apb_err_disable(struct pci_bus *bus,
+ bool disable)
+{
+ struct iproc_pcie *pcie = iproc_data(bus);
+ u32 val;
+
+ if (bus->number && pcie->has_apb_err_disable) {
+ val = iproc_pcie_read_reg(pcie, IPROC_PCIE_APB_ERR_EN);
+ if (disable)
+ val &= ~APB_ERR_EN;
+ else
+ val |= APB_ERR_EN;
+ iproc_pcie_write_reg(pcie, IPROC_PCIE_APB_ERR_EN, val);
+ }
+}
+
+static void __iomem *iproc_pcie_map_ep_cfg_reg(struct iproc_pcie *pcie,
+ unsigned int busno,
+ unsigned int slot,
+ unsigned int fn,
+ int where)
+{
+ u16 offset;
+ u32 val;
+
+ /* EP device access */
+ val = (busno << CFG_ADDR_BUS_NUM_SHIFT) |
+ (slot << CFG_ADDR_DEV_NUM_SHIFT) |
+ (fn << CFG_ADDR_FUNC_NUM_SHIFT) |
+ (where & CFG_ADDR_REG_NUM_MASK) |
+ (1 & CFG_ADDR_CFG_TYPE_MASK);
+
+ iproc_pcie_write_reg(pcie, IPROC_PCIE_CFG_ADDR, val);
+ offset = iproc_pcie_reg_offset(pcie, IPROC_PCIE_CFG_DATA);
+
+ if (iproc_pcie_reg_is_invalid(offset))
+ return NULL;
+
+ return (pcie->base + offset);
+}
+
+static unsigned int iproc_pcie_cfg_retry(void __iomem *cfg_data_p)
+{
+ int timeout = CFG_RETRY_STATUS_TIMEOUT_US;
+ unsigned int data;
+
+ /*
+ * As per PCIe spec r3.1, sec 2.3.2, CRS Software Visibility only
+ * affects config reads of the Vendor ID. For config writes or any
+ * other config reads, the Root may automatically reissue the
+ * configuration request again as a new request.
+ *
+ * For config reads, this hardware returns CFG_RETRY_STATUS data
+ * when it receives a CRS completion, regardless of the address of
+ * the read or the CRS Software Visibility Enable bit. As a
+ * partial workaround for this, we retry in software any read that
+ * returns CFG_RETRY_STATUS.
+ *
+ * Note that a non-Vendor ID config register may have a value of
+ * CFG_RETRY_STATUS. If we read that, we can't distinguish it from
+ * a CRS completion, so we will incorrectly retry the read and
+ * eventually return the wrong data (0xffffffff).
+ */
+ data = readl(cfg_data_p);
+ while (data == CFG_RETRY_STATUS && timeout--) {
+ udelay(1);
+ data = readl(cfg_data_p);
+ }
+
+ if (data == CFG_RETRY_STATUS)
+ data = 0xffffffff;
+
+ return data;
+}
+
+static int iproc_pcie_config_read(struct pci_bus *bus, unsigned int devfn,
+ int where, int size, u32 *val)
+{
+ struct iproc_pcie *pcie = iproc_data(bus);
+ unsigned int slot = PCI_SLOT(devfn);
+ unsigned int fn = PCI_FUNC(devfn);
+ unsigned int busno = bus->number;
+ void __iomem *cfg_data_p;
+ unsigned int data;
+ int ret;
+
+ /* root complex access */
+ if (busno == 0) {
+ ret = pci_generic_config_read32(bus, devfn, where, size, val);
+ if (ret != PCIBIOS_SUCCESSFUL)
+ return ret;
+
+ /* Don't advertise CRS SV support */
+ if ((where & ~0x3) == IPROC_PCI_EXP_CAP + PCI_EXP_RTCTL)
+ *val &= ~(PCI_EXP_RTCAP_CRSVIS << 16);
+ return PCIBIOS_SUCCESSFUL;
+ }
+
+ cfg_data_p = iproc_pcie_map_ep_cfg_reg(pcie, busno, slot, fn, where);
+
+ if (!cfg_data_p)
+ return PCIBIOS_DEVICE_NOT_FOUND;
+
+ data = iproc_pcie_cfg_retry(cfg_data_p);
+
+ *val = data;
+ if (size <= 2)
+ *val = (data >> (8 * (where & 3))) & ((1 << (size * 8)) - 1);
+
+ return PCIBIOS_SUCCESSFUL;
+}
+
+/**
+ * Note access to the configuration registers are protected at the higher layer
+ * by 'pci_lock' in drivers/pci/access.c
+ */
+static void __iomem *iproc_pcie_map_cfg_bus(struct iproc_pcie *pcie,
+ int busno, unsigned int devfn,
+ int where)
+{
+ unsigned slot = PCI_SLOT(devfn);
+ unsigned fn = PCI_FUNC(devfn);
+ u16 offset;
+
+ /* root complex access */
+ if (busno == 0) {
+ if (slot > 0 || fn > 0)
+ return NULL;
+
+ iproc_pcie_write_reg(pcie, IPROC_PCIE_CFG_IND_ADDR,
+ where & CFG_IND_ADDR_MASK);
+ offset = iproc_pcie_reg_offset(pcie, IPROC_PCIE_CFG_IND_DATA);
+ if (iproc_pcie_reg_is_invalid(offset))
+ return NULL;
+ else
+ return (pcie->base + offset);
+ }
+
+ return iproc_pcie_map_ep_cfg_reg(pcie, busno, slot, fn, where);
+}
+
+static void __iomem *iproc_pcie_bus_map_cfg_bus(struct pci_bus *bus,
+ unsigned int devfn,
+ int where)
+{
+ return iproc_pcie_map_cfg_bus(iproc_data(bus), bus->number, devfn,
+ where);
+}
+
+static int iproc_pci_raw_config_read32(struct iproc_pcie *pcie,
+ unsigned int devfn, int where,
+ int size, u32 *val)
+{
+ void __iomem *addr;
+
+ addr = iproc_pcie_map_cfg_bus(pcie, 0, devfn, where & ~0x3);
+ if (!addr) {
+ *val = ~0;
+ return PCIBIOS_DEVICE_NOT_FOUND;
+ }
+
+ *val = readl(addr);
+
+ if (size <= 2)
+ *val = (*val >> (8 * (where & 3))) & ((1 << (size * 8)) - 1);
+
+ return PCIBIOS_SUCCESSFUL;
+}
+
+static int iproc_pci_raw_config_write32(struct iproc_pcie *pcie,
+ unsigned int devfn, int where,
+ int size, u32 val)
+{
+ void __iomem *addr;
+ u32 mask, tmp;
+
+ addr = iproc_pcie_map_cfg_bus(pcie, 0, devfn, where & ~0x3);
+ if (!addr)
+ return PCIBIOS_DEVICE_NOT_FOUND;
+
+ if (size == 4) {
+ writel(val, addr);
+ return PCIBIOS_SUCCESSFUL;
+ }
+
+ mask = ~(((1 << (size * 8)) - 1) << ((where & 0x3) * 8));
+ tmp = readl(addr) & mask;
+ tmp |= val << ((where & 0x3) * 8);
+ writel(tmp, addr);
+
+ return PCIBIOS_SUCCESSFUL;
+}
+
+static int iproc_pcie_config_read32(struct pci_bus *bus, unsigned int devfn,
+ int where, int size, u32 *val)
+{
+ int ret;
+ struct iproc_pcie *pcie = iproc_data(bus);
+
+ iproc_pcie_apb_err_disable(bus, true);
+ if (pcie->type == IPROC_PCIE_PAXB_V2)
+ ret = iproc_pcie_config_read(bus, devfn, where, size, val);
+ else
+ ret = pci_generic_config_read32(bus, devfn, where, size, val);
+ iproc_pcie_apb_err_disable(bus, false);
+
+ return ret;
+}
+
+static int iproc_pcie_config_write32(struct pci_bus *bus, unsigned int devfn,
+ int where, int size, u32 val)
+{
+ int ret;
+
+ iproc_pcie_apb_err_disable(bus, true);
+ ret = pci_generic_config_write32(bus, devfn, where, size, val);
+ iproc_pcie_apb_err_disable(bus, false);
+
+ return ret;
+}
+
+static struct pci_ops iproc_pcie_ops = {
+ .map_bus = iproc_pcie_bus_map_cfg_bus,
+ .read = iproc_pcie_config_read32,
+ .write = iproc_pcie_config_write32,
+};
+
+static void iproc_pcie_perst_ctrl(struct iproc_pcie *pcie, bool assert)
+{
+ u32 val;
+
+ /*
+ * PAXC and the internal emulated endpoint device downstream should not
+ * be reset. If firmware has been loaded on the endpoint device at an
+ * earlier boot stage, reset here causes issues.
+ */
+ if (pcie->ep_is_internal)
+ return;
+
+ if (assert) {
+ val = iproc_pcie_read_reg(pcie, IPROC_PCIE_CLK_CTRL);
+ val &= ~EP_PERST_SOURCE_SELECT & ~EP_MODE_SURVIVE_PERST &
+ ~RC_PCIE_RST_OUTPUT;
+ iproc_pcie_write_reg(pcie, IPROC_PCIE_CLK_CTRL, val);
+ udelay(250);
+ } else {
+ val = iproc_pcie_read_reg(pcie, IPROC_PCIE_CLK_CTRL);
+ val |= RC_PCIE_RST_OUTPUT;
+ iproc_pcie_write_reg(pcie, IPROC_PCIE_CLK_CTRL, val);
+ msleep(100);
+ }
+}
+
+int iproc_pcie_shutdown(struct iproc_pcie *pcie)
+{
+ iproc_pcie_perst_ctrl(pcie, true);
+ msleep(500);
+
+ return 0;
+}
+EXPORT_SYMBOL_GPL(iproc_pcie_shutdown);
+
+static int iproc_pcie_check_link(struct iproc_pcie *pcie)
+{
+ struct device *dev = pcie->dev;
+ u32 hdr_type, link_ctrl, link_status, class, val;
+ bool link_is_active = false;
+
+ /*
+ * PAXC connects to emulated endpoint devices directly and does not
+ * have a Serdes. Therefore skip the link detection logic here.
+ */
+ if (pcie->ep_is_internal)
+ return 0;
+
+ val = iproc_pcie_read_reg(pcie, IPROC_PCIE_LINK_STATUS);
+ if (!(val & PCIE_PHYLINKUP) || !(val & PCIE_DL_ACTIVE)) {
+ dev_err(dev, "PHY or data link is INACTIVE!\n");
+ return -ENODEV;
+ }
+
+ /* make sure we are not in EP mode */
+ iproc_pci_raw_config_read32(pcie, 0, PCI_HEADER_TYPE, 1, &hdr_type);
+ if ((hdr_type & 0x7f) != PCI_HEADER_TYPE_BRIDGE) {
+ dev_err(dev, "in EP mode, hdr=%#02x\n", hdr_type);
+ return -EFAULT;
+ }
+
+ /* force class to PCI_CLASS_BRIDGE_PCI (0x0604) */
+#define PCI_BRIDGE_CTRL_REG_OFFSET 0x43c
+#define PCI_CLASS_BRIDGE_MASK 0xffff00
+#define PCI_CLASS_BRIDGE_SHIFT 8
+ iproc_pci_raw_config_read32(pcie, 0, PCI_BRIDGE_CTRL_REG_OFFSET,
+ 4, &class);
+ class &= ~PCI_CLASS_BRIDGE_MASK;
+ class |= (PCI_CLASS_BRIDGE_PCI << PCI_CLASS_BRIDGE_SHIFT);
+ iproc_pci_raw_config_write32(pcie, 0, PCI_BRIDGE_CTRL_REG_OFFSET,
+ 4, class);
+
+ /* check link status to see if link is active */
+ iproc_pci_raw_config_read32(pcie, 0, IPROC_PCI_EXP_CAP + PCI_EXP_LNKSTA,
+ 2, &link_status);
+ if (link_status & PCI_EXP_LNKSTA_NLW)
+ link_is_active = true;
+
+ if (!link_is_active) {
+ /* try GEN 1 link speed */
+#define PCI_TARGET_LINK_SPEED_MASK 0xf
+#define PCI_TARGET_LINK_SPEED_GEN2 0x2
+#define PCI_TARGET_LINK_SPEED_GEN1 0x1
+ iproc_pci_raw_config_read32(pcie, 0,
+ IPROC_PCI_EXP_CAP + PCI_EXP_LNKCTL2,
+ 4, &link_ctrl);
+ if ((link_ctrl & PCI_TARGET_LINK_SPEED_MASK) ==
+ PCI_TARGET_LINK_SPEED_GEN2) {
+ link_ctrl &= ~PCI_TARGET_LINK_SPEED_MASK;
+ link_ctrl |= PCI_TARGET_LINK_SPEED_GEN1;
+ iproc_pci_raw_config_write32(pcie, 0,
+ IPROC_PCI_EXP_CAP + PCI_EXP_LNKCTL2,
+ 4, link_ctrl);
+ msleep(100);
+
+ iproc_pci_raw_config_read32(pcie, 0,
+ IPROC_PCI_EXP_CAP + PCI_EXP_LNKSTA,
+ 2, &link_status);
+ if (link_status & PCI_EXP_LNKSTA_NLW)
+ link_is_active = true;
+ }
+ }
+
+ dev_info(dev, "link: %s\n", link_is_active ? "UP" : "DOWN");
+
+ return link_is_active ? 0 : -ENODEV;
+}
+
+static void iproc_pcie_enable(struct iproc_pcie *pcie)
+{
+ iproc_pcie_write_reg(pcie, IPROC_PCIE_INTX_EN, SYS_RC_INTX_MASK);
+}
+
+static inline bool iproc_pcie_ob_is_valid(struct iproc_pcie *pcie,
+ int window_idx)
+{
+ u32 val;
+
+ val = iproc_pcie_read_reg(pcie, MAP_REG(IPROC_PCIE_OARR0, window_idx));
+
+ return !!(val & OARR_VALID);
+}
+
+static inline int iproc_pcie_ob_write(struct iproc_pcie *pcie, int window_idx,
+ int size_idx, u64 axi_addr, u64 pci_addr)
+{
+ struct device *dev = pcie->dev;
+ u16 oarr_offset, omap_offset;
+
+ /*
+ * Derive the OARR/OMAP offset from the first pair (OARR0/OMAP0) based
+ * on window index.
+ */
+ oarr_offset = iproc_pcie_reg_offset(pcie, MAP_REG(IPROC_PCIE_OARR0,
+ window_idx));
+ omap_offset = iproc_pcie_reg_offset(pcie, MAP_REG(IPROC_PCIE_OMAP0,
+ window_idx));
+ if (iproc_pcie_reg_is_invalid(oarr_offset) ||
+ iproc_pcie_reg_is_invalid(omap_offset))
+ return -EINVAL;
+
+ /*
+ * Program the OARR registers. The upper 32-bit OARR register is
+ * always right after the lower 32-bit OARR register.
+ */
+ writel(lower_32_bits(axi_addr) | (size_idx << OARR_SIZE_CFG_SHIFT) |
+ OARR_VALID, pcie->base + oarr_offset);
+ writel(upper_32_bits(axi_addr), pcie->base + oarr_offset + 4);
+
+ /* now program the OMAP registers */
+ writel(lower_32_bits(pci_addr), pcie->base + omap_offset);
+ writel(upper_32_bits(pci_addr), pcie->base + omap_offset + 4);
+
+ dev_info(dev, "ob window [%d]: offset 0x%x axi %pap pci %pap\n",
+ window_idx, oarr_offset, &axi_addr, &pci_addr);
+ dev_info(dev, "oarr lo 0x%x oarr hi 0x%x\n",
+ readl(pcie->base + oarr_offset),
+ readl(pcie->base + oarr_offset + 4));
+ dev_info(dev, "omap lo 0x%x omap hi 0x%x\n",
+ readl(pcie->base + omap_offset),
+ readl(pcie->base + omap_offset + 4));
+
+ return 0;
+}
+
+/**
+ * Some iProc SoCs require the SW to configure the outbound address mapping
+ *
+ * Outbound address translation:
+ *
+ * iproc_pcie_address = axi_address - axi_offset
+ * OARR = iproc_pcie_address
+ * OMAP = pci_addr
+ *
+ * axi_addr -> iproc_pcie_address -> OARR -> OMAP -> pci_address
+ */
+static int iproc_pcie_setup_ob(struct iproc_pcie *pcie, u64 axi_addr,
+ u64 pci_addr, resource_size_t size)
+{
+ struct iproc_pcie_ob *ob = &pcie->ob;
+ struct device *dev = pcie->dev;
+ int ret = -EINVAL, window_idx, size_idx;
+
+ if (axi_addr < ob->axi_offset) {
+ dev_err(dev, "axi address %pap less than offset %pap\n",
+ &axi_addr, &ob->axi_offset);
+ return -EINVAL;
+ }
+
+ /*
+ * Translate the AXI address to the internal address used by the iProc
+ * PCIe core before programming the OARR
+ */
+ axi_addr -= ob->axi_offset;
+
+ /* iterate through all OARR/OMAP mapping windows */
+ for (window_idx = ob->nr_windows - 1; window_idx >= 0; window_idx--) {
+ const struct iproc_pcie_ob_map *ob_map =
+ &pcie->ob_map[window_idx];
+
+ /*
+ * If current outbound window is already in use, move on to the
+ * next one.
+ */
+ if (iproc_pcie_ob_is_valid(pcie, window_idx))
+ continue;
+
+ /*
+ * Iterate through all supported window sizes within the
+ * OARR/OMAP pair to find a match. Go through the window sizes
+ * in a descending order.
+ */
+ for (size_idx = ob_map->nr_sizes - 1; size_idx >= 0;
+ size_idx--) {
+ resource_size_t window_size =
+ ob_map->window_sizes[size_idx] * SZ_1M;
+
+ if (size < window_size)
+ continue;
+
+ if (!IS_ALIGNED(axi_addr, window_size) ||
+ !IS_ALIGNED(pci_addr, window_size)) {
+ dev_err(dev,
+ "axi %pap or pci %pap not aligned\n",
+ &axi_addr, &pci_addr);
+ return -EINVAL;
+ }
+
+ /*
+ * Match found! Program both OARR and OMAP and mark
+ * them as a valid entry.
+ */
+ ret = iproc_pcie_ob_write(pcie, window_idx, size_idx,
+ axi_addr, pci_addr);
+ if (ret)
+ goto err_ob;
+
+ size -= window_size;
+ if (size == 0)
+ return 0;
+
+ /*
+ * If we are here, we are done with the current window,
+ * but not yet finished all mappings. Need to move on
+ * to the next window.
+ */
+ axi_addr += window_size;
+ pci_addr += window_size;
+ break;
+ }
+ }
+
+err_ob:
+ dev_err(dev, "unable to configure outbound mapping\n");
+ dev_err(dev,
+ "axi %pap, axi offset %pap, pci %pap, res size %pap\n",
+ &axi_addr, &ob->axi_offset, &pci_addr, &size);
+
+ return ret;
+}
+
+static int iproc_pcie_map_ranges(struct iproc_pcie *pcie,
+ struct list_head *resources)
+{
+ struct device *dev = pcie->dev;
+ struct resource_entry *window;
+ int ret;
+
+ resource_list_for_each_entry(window, resources) {
+ struct resource *res = window->res;
+ u64 res_type = resource_type(res);
+
+ switch (res_type) {
+ case IORESOURCE_IO:
+ case IORESOURCE_BUS:
+ break;
+ case IORESOURCE_MEM:
+ ret = iproc_pcie_setup_ob(pcie, res->start,
+ res->start - window->offset,
+ resource_size(res));
+ if (ret)
+ return ret;
+ break;
+ default:
+ dev_err(dev, "invalid resource %pR\n", res);
+ return -EINVAL;
+ }
+ }
+
+ return 0;
+}
+
+static inline bool iproc_pcie_ib_is_in_use(struct iproc_pcie *pcie,
+ int region_idx)
+{
+ const struct iproc_pcie_ib_map *ib_map = &pcie->ib_map[region_idx];
+ u32 val;
+
+ val = iproc_pcie_read_reg(pcie, MAP_REG(IPROC_PCIE_IARR0, region_idx));
+
+ return !!(val & (BIT(ib_map->nr_sizes) - 1));
+}
+
+static inline bool iproc_pcie_ib_check_type(const struct iproc_pcie_ib_map *ib_map,
+ enum iproc_pcie_ib_map_type type)
+{
+ return !!(ib_map->type == type);
+}
+
+static int iproc_pcie_ib_write(struct iproc_pcie *pcie, int region_idx,
+ int size_idx, int nr_windows, u64 axi_addr,
+ u64 pci_addr, resource_size_t size)
+{
+ struct device *dev = pcie->dev;
+ const struct iproc_pcie_ib_map *ib_map = &pcie->ib_map[region_idx];
+ u16 iarr_offset, imap_offset;
+ u32 val;
+ int window_idx;
+
+ iarr_offset = iproc_pcie_reg_offset(pcie,
+ MAP_REG(IPROC_PCIE_IARR0, region_idx));
+ imap_offset = iproc_pcie_reg_offset(pcie,
+ MAP_REG(IPROC_PCIE_IMAP0, region_idx));
+ if (iproc_pcie_reg_is_invalid(iarr_offset) ||
+ iproc_pcie_reg_is_invalid(imap_offset))
+ return -EINVAL;
+
+ dev_info(dev, "ib region [%d]: offset 0x%x axi %pap pci %pap\n",
+ region_idx, iarr_offset, &axi_addr, &pci_addr);
+
+ /*
+ * Program the IARR registers. The upper 32-bit IARR register is
+ * always right after the lower 32-bit IARR register.
+ */
+ writel(lower_32_bits(pci_addr) | BIT(size_idx),
+ pcie->base + iarr_offset);
+ writel(upper_32_bits(pci_addr), pcie->base + iarr_offset + 4);
+
+ dev_info(dev, "iarr lo 0x%x iarr hi 0x%x\n",
+ readl(pcie->base + iarr_offset),
+ readl(pcie->base + iarr_offset + 4));
+
+ /*
+ * Now program the IMAP registers. Each IARR region may have one or
+ * more IMAP windows.
+ */
+ size >>= ilog2(nr_windows);
+ for (window_idx = 0; window_idx < nr_windows; window_idx++) {
+ val = readl(pcie->base + imap_offset);
+ val |= lower_32_bits(axi_addr) | IMAP_VALID;
+ writel(val, pcie->base + imap_offset);
+ writel(upper_32_bits(axi_addr),
+ pcie->base + imap_offset + ib_map->imap_addr_offset);
+
+ dev_info(dev, "imap window [%d] lo 0x%x hi 0x%x\n",
+ window_idx, readl(pcie->base + imap_offset),
+ readl(pcie->base + imap_offset +
+ ib_map->imap_addr_offset));
+
+ imap_offset += ib_map->imap_window_offset;
+ axi_addr += size;
+ }
+
+ return 0;
+}
+
+static int iproc_pcie_setup_ib(struct iproc_pcie *pcie,
+ struct of_pci_range *range,
+ enum iproc_pcie_ib_map_type type)
+{
+ struct device *dev = pcie->dev;
+ struct iproc_pcie_ib *ib = &pcie->ib;
+ int ret;
+ unsigned int region_idx, size_idx;
+ u64 axi_addr = range->cpu_addr, pci_addr = range->pci_addr;
+ resource_size_t size = range->size;
+
+ /* iterate through all IARR mapping regions */
+ for (region_idx = 0; region_idx < ib->nr_regions; region_idx++) {
+ const struct iproc_pcie_ib_map *ib_map =
+ &pcie->ib_map[region_idx];
+
+ /*
+ * If current inbound region is already in use or not a
+ * compatible type, move on to the next.
+ */
+ if (iproc_pcie_ib_is_in_use(pcie, region_idx) ||
+ !iproc_pcie_ib_check_type(ib_map, type))
+ continue;
+
+ /* iterate through all supported region sizes to find a match */
+ for (size_idx = 0; size_idx < ib_map->nr_sizes; size_idx++) {
+ resource_size_t region_size =
+ ib_map->region_sizes[size_idx] * ib_map->size_unit;
+
+ if (size != region_size)
+ continue;
+
+ if (!IS_ALIGNED(axi_addr, region_size) ||
+ !IS_ALIGNED(pci_addr, region_size)) {
+ dev_err(dev,
+ "axi %pap or pci %pap not aligned\n",
+ &axi_addr, &pci_addr);
+ return -EINVAL;
+ }
+
+ /* Match found! Program IARR and all IMAP windows. */
+ ret = iproc_pcie_ib_write(pcie, region_idx, size_idx,
+ ib_map->nr_windows, axi_addr,
+ pci_addr, size);
+ if (ret)
+ goto err_ib;
+ else
+ return 0;
+
+ }
+ }
+ ret = -EINVAL;
+
+err_ib:
+ dev_err(dev, "unable to configure inbound mapping\n");
+ dev_err(dev, "axi %pap, pci %pap, res size %pap\n",
+ &axi_addr, &pci_addr, &size);
+
+ return ret;
+}
+
+static int pci_dma_range_parser_init(struct of_pci_range_parser *parser,
+ struct device_node *node)
+{
+ const int na = 3, ns = 2;
+ int rlen;
+
+ parser->node = node;
+ parser->pna = of_n_addr_cells(node);
+ parser->np = parser->pna + na + ns;
+
+ parser->range = of_get_property(node, "dma-ranges", &rlen);
+ if (!parser->range)
+ return -ENOENT;
+
+ parser->end = parser->range + rlen / sizeof(__be32);
+ return 0;
+}
+
+static int iproc_pcie_map_dma_ranges(struct iproc_pcie *pcie)
+{
+ struct of_pci_range range;
+ struct of_pci_range_parser parser;
+ int ret;
+
+ /* Get the dma-ranges from DT */
+ ret = pci_dma_range_parser_init(&parser, pcie->dev->of_node);
+ if (ret)
+ return ret;
+
+ for_each_of_pci_range(&parser, &range) {
+ /* Each range entry corresponds to an inbound mapping region */
+ ret = iproc_pcie_setup_ib(pcie, &range, IPROC_PCIE_IB_MAP_MEM);
+ if (ret)
+ return ret;
+ }
+
+ return 0;
+}
+
+static int iproce_pcie_get_msi(struct iproc_pcie *pcie,
+ struct device_node *msi_node,
+ u64 *msi_addr)
+{
+ struct device *dev = pcie->dev;
+ int ret;
+ struct resource res;
+
+ /*
+ * Check if 'msi-map' points to ARM GICv3 ITS, which is the only
+ * supported external MSI controller that requires steering.
+ */
+ if (!of_device_is_compatible(msi_node, "arm,gic-v3-its")) {
+ dev_err(dev, "unable to find compatible MSI controller\n");
+ return -ENODEV;
+ }
+
+ /* derive GITS_TRANSLATER address from GICv3 */
+ ret = of_address_to_resource(msi_node, 0, &res);
+ if (ret < 0) {
+ dev_err(dev, "unable to obtain MSI controller resources\n");
+ return ret;
+ }
+
+ *msi_addr = res.start + GITS_TRANSLATER;
+ return 0;
+}
+
+static int iproc_pcie_paxb_v2_msi_steer(struct iproc_pcie *pcie, u64 msi_addr)
+{
+ int ret;
+ struct of_pci_range range;
+
+ memset(&range, 0, sizeof(range));
+ range.size = SZ_32K;
+ range.pci_addr = range.cpu_addr = msi_addr & ~(range.size - 1);
+
+ ret = iproc_pcie_setup_ib(pcie, &range, IPROC_PCIE_IB_MAP_IO);
+ return ret;
+}
+
+static void iproc_pcie_paxc_v2_msi_steer(struct iproc_pcie *pcie, u64 msi_addr)
+{
+ u32 val;
+
+ /*
+ * Program bits [43:13] of address of GITS_TRANSLATER register into
+ * bits [30:0] of the MSI base address register. In fact, in all iProc
+ * based SoCs, all I/O register bases are well below the 32-bit
+ * boundary, so we can safely assume bits [43:32] are always zeros.
+ */
+ iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_BASE_ADDR,
+ (u32)(msi_addr >> 13));
+
+ /* use a default 8K window size */
+ iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_WINDOW_SIZE, 0);
+
+ /* steering MSI to GICv3 ITS */
+ val = iproc_pcie_read_reg(pcie, IPROC_PCIE_MSI_GIC_MODE);
+ val |= GIC_V3_CFG;
+ iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_GIC_MODE, val);
+
+ /*
+ * Program bits [43:2] of address of GITS_TRANSLATER register into the
+ * iProc MSI address registers.
+ */
+ msi_addr >>= 2;
+ iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_ADDR_HI,
+ upper_32_bits(msi_addr));
+ iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_ADDR_LO,
+ lower_32_bits(msi_addr));
+
+ /* enable MSI */
+ val = iproc_pcie_read_reg(pcie, IPROC_PCIE_MSI_EN_CFG);
+ val |= MSI_ENABLE_CFG;
+ iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_EN_CFG, val);
+}
+
+static int iproc_pcie_msi_steer(struct iproc_pcie *pcie,
+ struct device_node *msi_node)
+{
+ struct device *dev = pcie->dev;
+ int ret;
+ u64 msi_addr;
+
+ ret = iproce_pcie_get_msi(pcie, msi_node, &msi_addr);
+ if (ret < 0) {
+ dev_err(dev, "msi steering failed\n");
+ return ret;
+ }
+
+ switch (pcie->type) {
+ case IPROC_PCIE_PAXB_V2:
+ ret = iproc_pcie_paxb_v2_msi_steer(pcie, msi_addr);
+ if (ret)
+ return ret;
+ break;
+ case IPROC_PCIE_PAXC_V2:
+ iproc_pcie_paxc_v2_msi_steer(pcie, msi_addr);
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static int iproc_pcie_msi_enable(struct iproc_pcie *pcie)
+{
+ struct device_node *msi_node;
+ int ret;
+
+ /*
+ * Either the "msi-parent" or the "msi-map" phandle needs to exist
+ * for us to obtain the MSI node.
+ */
+
+ msi_node = of_parse_phandle(pcie->dev->of_node, "msi-parent", 0);
+ if (!msi_node) {
+ const __be32 *msi_map = NULL;
+ int len;
+ u32 phandle;
+
+ msi_map = of_get_property(pcie->dev->of_node, "msi-map", &len);
+ if (!msi_map)
+ return -ENODEV;
+
+ phandle = be32_to_cpup(msi_map + 1);
+ msi_node = of_find_node_by_phandle(phandle);
+ if (!msi_node)
+ return -ENODEV;
+ }
+
+ /*
+ * Certain revisions of the iProc PCIe controller require additional
+ * configurations to steer the MSI writes towards an external MSI
+ * controller.
+ */
+ if (pcie->need_msi_steer) {
+ ret = iproc_pcie_msi_steer(pcie, msi_node);
+ if (ret)
+ return ret;
+ }
+
+ /*
+ * If another MSI controller is being used, the call below should fail
+ * but that is okay
+ */
+ return iproc_msi_init(pcie, msi_node);
+}
+
+static void iproc_pcie_msi_disable(struct iproc_pcie *pcie)
+{
+ iproc_msi_exit(pcie);
+}
+
+static int iproc_pcie_rev_init(struct iproc_pcie *pcie)
+{
+ struct device *dev = pcie->dev;
+ unsigned int reg_idx;
+ const u16 *regs;
+
+ switch (pcie->type) {
+ case IPROC_PCIE_PAXB_BCMA:
+ regs = iproc_pcie_reg_paxb_bcma;
+ break;
+ case IPROC_PCIE_PAXB:
+ regs = iproc_pcie_reg_paxb;
+ pcie->has_apb_err_disable = true;
+ if (pcie->need_ob_cfg) {
+ pcie->ob_map = paxb_ob_map;
+ pcie->ob.nr_windows = ARRAY_SIZE(paxb_ob_map);
+ }
+ break;
+ case IPROC_PCIE_PAXB_V2:
+ regs = iproc_pcie_reg_paxb_v2;
+ pcie->has_apb_err_disable = true;
+ if (pcie->need_ob_cfg) {
+ pcie->ob_map = paxb_v2_ob_map;
+ pcie->ob.nr_windows = ARRAY_SIZE(paxb_v2_ob_map);
+ }
+ pcie->ib.nr_regions = ARRAY_SIZE(paxb_v2_ib_map);
+ pcie->ib_map = paxb_v2_ib_map;
+ pcie->need_msi_steer = true;
+ dev_warn(dev, "reads of config registers that contain %#x return incorrect data\n",
+ CFG_RETRY_STATUS);
+ break;
+ case IPROC_PCIE_PAXC:
+ regs = iproc_pcie_reg_paxc;
+ pcie->ep_is_internal = true;
+ break;
+ case IPROC_PCIE_PAXC_V2:
+ regs = iproc_pcie_reg_paxc_v2;
+ pcie->ep_is_internal = true;
+ pcie->need_msi_steer = true;
+ break;
+ default:
+ dev_err(dev, "incompatible iProc PCIe interface\n");
+ return -EINVAL;
+ }
+
+ pcie->reg_offsets = devm_kcalloc(dev, IPROC_PCIE_MAX_NUM_REG,
+ sizeof(*pcie->reg_offsets),
+ GFP_KERNEL);
+ if (!pcie->reg_offsets)
+ return -ENOMEM;
+
+ /* go through the register table and populate all valid registers */
+ pcie->reg_offsets[0] = (pcie->type == IPROC_PCIE_PAXC_V2) ?
+ IPROC_PCIE_REG_INVALID : regs[0];
+ for (reg_idx = 1; reg_idx < IPROC_PCIE_MAX_NUM_REG; reg_idx++)
+ pcie->reg_offsets[reg_idx] = regs[reg_idx] ?
+ regs[reg_idx] : IPROC_PCIE_REG_INVALID;
+
+ return 0;
+}
+
+int iproc_pcie_setup(struct iproc_pcie *pcie, struct list_head *res)
+{
+ struct device *dev;
+ int ret;
+ void *sysdata;
+ struct pci_bus *child;
+ struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie);
+
+ dev = pcie->dev;
+
+ ret = iproc_pcie_rev_init(pcie);
+ if (ret) {
+ dev_err(dev, "unable to initialize controller parameters\n");
+ return ret;
+ }
+
+ ret = devm_request_pci_bus_resources(dev, res);
+ if (ret)
+ return ret;
+
+ ret = phy_init(pcie->phy);
+ if (ret) {
+ dev_err(dev, "unable to initialize PCIe PHY\n");
+ return ret;
+ }
+
+ ret = phy_power_on(pcie->phy);
+ if (ret) {
+ dev_err(dev, "unable to power on PCIe PHY\n");
+ goto err_exit_phy;
+ }
+
+ iproc_pcie_perst_ctrl(pcie, true);
+ iproc_pcie_perst_ctrl(pcie, false);
+
+ if (pcie->need_ob_cfg) {
+ ret = iproc_pcie_map_ranges(pcie, res);
+ if (ret) {
+ dev_err(dev, "map failed\n");
+ goto err_power_off_phy;
+ }
+ }
+
+ if (pcie->need_ib_cfg) {
+ ret = iproc_pcie_map_dma_ranges(pcie);
+ if (ret && ret != -ENOENT)
+ goto err_power_off_phy;
+ }
+
+#ifdef CONFIG_ARM
+ pcie->sysdata.private_data = pcie;
+ sysdata = &pcie->sysdata;
+#else
+ sysdata = pcie;
+#endif
+
+ ret = iproc_pcie_check_link(pcie);
+ if (ret) {
+ dev_err(dev, "no PCIe EP device detected\n");
+ goto err_power_off_phy;
+ }
+
+ iproc_pcie_enable(pcie);
+
+ if (IS_ENABLED(CONFIG_PCI_MSI))
+ if (iproc_pcie_msi_enable(pcie))
+ dev_info(dev, "not using iProc MSI\n");
+
+ list_splice_init(res, &host->windows);
+ host->busnr = 0;
+ host->dev.parent = dev;
+ host->ops = &iproc_pcie_ops;
+ host->sysdata = sysdata;
+ host->map_irq = pcie->map_irq;
+ host->swizzle_irq = pci_common_swizzle;
+
+ ret = pci_scan_root_bus_bridge(host);
+ if (ret < 0) {
+ dev_err(dev, "failed to scan host: %d\n", ret);
+ goto err_power_off_phy;
+ }
+
+ pci_assign_unassigned_bus_resources(host->bus);
+
+ pcie->root_bus = host->bus;
+
+ list_for_each_entry(child, &host->bus->children, node)
+ pcie_bus_configure_settings(child);
+
+ pci_bus_add_devices(host->bus);
+
+ return 0;
+
+err_power_off_phy:
+ phy_power_off(pcie->phy);
+err_exit_phy:
+ phy_exit(pcie->phy);
+ return ret;
+}
+EXPORT_SYMBOL(iproc_pcie_setup);
+
+int iproc_pcie_remove(struct iproc_pcie *pcie)
+{
+ pci_stop_root_bus(pcie->root_bus);
+ pci_remove_root_bus(pcie->root_bus);
+
+ iproc_pcie_msi_disable(pcie);
+
+ phy_power_off(pcie->phy);
+ phy_exit(pcie->phy);
+
+ return 0;
+}
+EXPORT_SYMBOL(iproc_pcie_remove);
+
+MODULE_AUTHOR("Ray Jui <rjui@broadcom.com>");
+MODULE_DESCRIPTION("Broadcom iPROC PCIe common driver");
+MODULE_LICENSE("GPL v2");
diff --git a/src/kernel/linux/v4.14/drivers/pci/host/pcie-iproc.h b/src/kernel/linux/v4.14/drivers/pci/host/pcie-iproc.h
new file mode 100644
index 0000000..4ac6282
--- /dev/null
+++ b/src/kernel/linux/v4.14/drivers/pci/host/pcie-iproc.h
@@ -0,0 +1,131 @@
+/*
+ * Copyright (C) 2014-2015 Broadcom Corporation
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation version 2.
+ *
+ * This program is distributed "as is" WITHOUT ANY WARRANTY of any
+ * kind, whether express or implied; without even the implied warranty
+ * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef _PCIE_IPROC_H
+#define _PCIE_IPROC_H
+
+/**
+ * iProc PCIe interface type
+ *
+ * PAXB is the wrapper used in root complex that can be connected to an
+ * external endpoint device.
+ *
+ * PAXC is the wrapper used in root complex dedicated for internal emulated
+ * endpoint devices.
+ */
+enum iproc_pcie_type {
+ IPROC_PCIE_PAXB_BCMA = 0,
+ IPROC_PCIE_PAXB,
+ IPROC_PCIE_PAXB_V2,
+ IPROC_PCIE_PAXC,
+ IPROC_PCIE_PAXC_V2,
+};
+
+/**
+ * iProc PCIe outbound mapping
+ * @axi_offset: offset from the AXI address to the internal address used by
+ * the iProc PCIe core
+ * @nr_windows: total number of supported outbound mapping windows
+ */
+struct iproc_pcie_ob {
+ resource_size_t axi_offset;
+ unsigned int nr_windows;
+};
+
+/**
+ * iProc PCIe inbound mapping
+ * @nr_regions: total number of supported inbound mapping regions
+ */
+struct iproc_pcie_ib {
+ unsigned int nr_regions;
+};
+
+struct iproc_pcie_ob_map;
+struct iproc_pcie_ib_map;
+struct iproc_msi;
+
+/**
+ * iProc PCIe device
+ *
+ * @dev: pointer to device data structure
+ * @type: iProc PCIe interface type
+ * @reg_offsets: register offsets
+ * @base: PCIe host controller I/O register base
+ * @base_addr: PCIe host controller register base physical address
+ * @sysdata: Per PCI controller data (ARM-specific)
+ * @root_bus: pointer to root bus
+ * @phy: optional PHY device that controls the Serdes
+ * @map_irq: function callback to map interrupts
+ * @ep_is_internal: indicates an internal emulated endpoint device is connected
+ * @has_apb_err_disable: indicates the controller can be configured to prevent
+ * unsupported request from being forwarded as an APB bus error
+ *
+ * @need_ob_cfg: indicates SW needs to configure the outbound mapping window
+ * @ob: outbound mapping related parameters
+ * @ob_map: outbound mapping related parameters specific to the controller
+ *
+ * @need_ib_cfg: indicates SW needs to configure the inbound mapping window
+ * @ib: inbound mapping related parameters
+ * @ib_map: outbound mapping region related parameters
+ *
+ * @need_msi_steer: indicates additional configuration of the iProc PCIe
+ * controller is required to steer MSI writes to external interrupt controller
+ * @msi: MSI data
+ */
+struct iproc_pcie {
+ struct device *dev;
+ enum iproc_pcie_type type;
+ u16 *reg_offsets;
+ void __iomem *base;
+ phys_addr_t base_addr;
+#ifdef CONFIG_ARM
+ struct pci_sys_data sysdata;
+#endif
+ struct resource mem;
+ struct pci_bus *root_bus;
+ struct phy *phy;
+ int (*map_irq)(const struct pci_dev *, u8, u8);
+ bool ep_is_internal;
+ bool has_apb_err_disable;
+
+ bool need_ob_cfg;
+ struct iproc_pcie_ob ob;
+ const struct iproc_pcie_ob_map *ob_map;
+
+ bool need_ib_cfg;
+ struct iproc_pcie_ib ib;
+ const struct iproc_pcie_ib_map *ib_map;
+
+ bool need_msi_steer;
+ struct iproc_msi *msi;
+};
+
+int iproc_pcie_setup(struct iproc_pcie *pcie, struct list_head *res);
+int iproc_pcie_remove(struct iproc_pcie *pcie);
+int iproc_pcie_shutdown(struct iproc_pcie *pcie);
+
+#ifdef CONFIG_PCIE_IPROC_MSI
+int iproc_msi_init(struct iproc_pcie *pcie, struct device_node *node);
+void iproc_msi_exit(struct iproc_pcie *pcie);
+#else
+static inline int iproc_msi_init(struct iproc_pcie *pcie,
+ struct device_node *node)
+{
+ return -ENODEV;
+}
+static inline void iproc_msi_exit(struct iproc_pcie *pcie)
+{
+}
+#endif
+
+#endif /* _PCIE_IPROC_H */
diff --git a/src/kernel/linux/v4.14/drivers/pci/host/pcie-mediatek.c b/src/kernel/linux/v4.14/drivers/pci/host/pcie-mediatek.c
new file mode 100644
index 0000000..d31d719
--- /dev/null
+++ b/src/kernel/linux/v4.14/drivers/pci/host/pcie-mediatek.c
@@ -0,0 +1,1304 @@
+/*
+ * MediaTek PCIe host controller driver.
+ *
+ * Copyright (c) 2017 MediaTek Inc.
+ * Author: Ryder Lee <ryder.lee@mediatek.com>
+ * Honghui Zhang <honghui.zhang@mediatek.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/iopoll.h>
+#include <linux/irq.h>
+#include <linux/irqchip/chained_irq.h>
+#include <linux/irqdomain.h>
+#include <linux/kernel.h>
+#include <linux/msi.h>
+#include <linux/module.h>
+#include <linux/of_address.h>
+#include <linux/of_pci.h>
+#include <linux/of_platform.h>
+#include <linux/pci.h>
+#include <linux/phy/phy.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/reset.h>
+
+/* PCIe shared registers */
+#define PCIE_SYS_CFG 0x00
+#define PCIE_INT_ENABLE 0x0c
+#define PCIE_CFG_ADDR 0x20
+#define PCIE_CFG_DATA 0x24
+
+/* PCIe per port registers */
+#define PCIE_BAR0_SETUP 0x10
+#define PCIE_CLASS 0x34
+#define PCIE_LINK_STATUS 0x50
+
+#define PCIE_PORT_INT_EN(x) BIT(20 + (x))
+#define PCIE_PORT_PERST(x) BIT(1 + (x))
+#define PCIE_PORT_LINKUP BIT(0)
+#define PCIE_BAR_MAP_MAX GENMASK(31, 16)
+
+#define PCIE_BAR_ENABLE BIT(0)
+#define PCIE_REVISION_ID BIT(0)
+#define PCIE_CLASS_CODE (0x60400 << 8)
+#define PCIE_CONF_REG(regn) (((regn) & GENMASK(7, 2)) | \
+ ((((regn) >> 8) & GENMASK(3, 0)) << 24))
+#define PCIE_CONF_FUN(fun) (((fun) << 8) & GENMASK(10, 8))
+#define PCIE_CONF_DEV(dev) (((dev) << 11) & GENMASK(15, 11))
+#define PCIE_CONF_BUS(bus) (((bus) << 16) & GENMASK(23, 16))
+#define PCIE_CONF_ADDR(regn, fun, dev, bus) \
+ (PCIE_CONF_REG(regn) | PCIE_CONF_FUN(fun) | \
+ PCIE_CONF_DEV(dev) | PCIE_CONF_BUS(bus))
+
+/* MediaTek specific configuration registers */
+#define PCIE_FTS_NUM 0x70c
+#define PCIE_FTS_NUM_MASK GENMASK(15, 8)
+#define PCIE_FTS_NUM_L0(x) ((x) & 0xff << 8)
+
+#define PCIE_FC_CREDIT 0x73c
+#define PCIE_FC_CREDIT_MASK (GENMASK(31, 31) | GENMASK(28, 16))
+#define PCIE_FC_CREDIT_VAL(x) ((x) << 16)
+
+/* PCIe V2 share registers */
+#define PCIE_SYS_CFG_V2 0x0
+#define PCIE_CSR_LTSSM_EN(x) BIT(0 + (x) * 8)
+#define PCIE_CSR_ASPM_L1_EN(x) BIT(1 + (x) * 8)
+
+/* PCIe V2 per-port registers */
+#define PCIE_MSI_VECTOR 0x0c0
+
+#define PCIE_CONF_VEND_ID 0x100
+#define PCIE_CONF_CLASS_ID 0x106
+
+#define PCIE_INT_MASK 0x420
+#define INTX_MASK GENMASK(19, 16)
+#define INTX_SHIFT 16
+#define PCIE_INT_STATUS 0x424
+#define MSI_STATUS BIT(23)
+#define PCIE_IMSI_STATUS 0x42c
+#define PCIE_IMSI_ADDR 0x430
+#define MSI_MASK BIT(23)
+#define MTK_MSI_IRQS_NUM 32
+
+#define PCIE_AHB_TRANS_BASE0_L 0x438
+#define PCIE_AHB_TRANS_BASE0_H 0x43c
+#define AHB2PCIE_SIZE(x) ((x) & GENMASK(4, 0))
+#define PCIE_AXI_WINDOW0 0x448
+#define WIN_ENABLE BIT(7)
+
+/* PCIe V2 configuration transaction header */
+#define PCIE_CFG_HEADER0 0x460
+#define PCIE_CFG_HEADER1 0x464
+#define PCIE_CFG_HEADER2 0x468
+#define PCIE_CFG_WDATA 0x470
+#define PCIE_APP_TLP_REQ 0x488
+#define PCIE_CFG_RDATA 0x48c
+#define APP_CFG_REQ BIT(0)
+#define APP_CPL_STATUS GENMASK(7, 5)
+
+#define CFG_WRRD_TYPE_0 4
+#define CFG_WR_FMT 2
+#define CFG_RD_FMT 0
+
+#define CFG_DW0_LENGTH(length) ((length) & GENMASK(9, 0))
+#define CFG_DW0_TYPE(type) (((type) << 24) & GENMASK(28, 24))
+#define CFG_DW0_FMT(fmt) (((fmt) << 29) & GENMASK(31, 29))
+#define CFG_DW2_REGN(regn) ((regn) & GENMASK(11, 2))
+#define CFG_DW2_FUN(fun) (((fun) << 16) & GENMASK(18, 16))
+#define CFG_DW2_DEV(dev) (((dev) << 19) & GENMASK(23, 19))
+#define CFG_DW2_BUS(bus) (((bus) << 24) & GENMASK(31, 24))
+#define CFG_HEADER_DW0(type, fmt) \
+ (CFG_DW0_LENGTH(1) | CFG_DW0_TYPE(type) | CFG_DW0_FMT(fmt))
+#define CFG_HEADER_DW1(where, size) \
+ (GENMASK(((size) - 1), 0) << ((where) & 0x3))
+#define CFG_HEADER_DW2(regn, fun, dev, bus) \
+ (CFG_DW2_REGN(regn) | CFG_DW2_FUN(fun) | \
+ CFG_DW2_DEV(dev) | CFG_DW2_BUS(bus))
+
+#define PCIE_RST_CTRL 0x510
+#define PCIE_PHY_RSTB BIT(0)
+#define PCIE_PIPE_SRSTB BIT(1)
+#define PCIE_MAC_SRSTB BIT(2)
+#define PCIE_CRSTB BIT(3)
+#define PCIE_PERSTB BIT(8)
+#define PCIE_LINKDOWN_RST_EN GENMASK(15, 13)
+#define PCIE_LINK_STATUS_V2 0x804
+#define PCIE_PORT_LINKUP_V2 BIT(10)
+
+struct mtk_pcie_port;
+
+/**
+ * struct mtk_pcie_soc - differentiate between host generations
+ * @need_fix_class_id: whether this host's class ID needed to be fixed or not
+ * @ops: pointer to configuration access functions
+ * @startup: pointer to controller setting functions
+ * @setup_irq: pointer to initialize IRQ functions
+ */
+struct mtk_pcie_soc {
+ bool need_fix_class_id;
+ struct pci_ops *ops;
+ int (*startup)(struct mtk_pcie_port *port);
+ int (*setup_irq)(struct mtk_pcie_port *port, struct device_node *node);
+};
+
+/**
+ * struct mtk_pcie_port - PCIe port information
+ * @base: IO mapped register base
+ * @list: port list
+ * @pcie: pointer to PCIe host info
+ * @reset: pointer to port reset control
+ * @sys_ck: pointer to transaction/data link layer clock
+ * @ahb_ck: pointer to AHB slave interface operating clock for CSR access
+ * and RC initiated MMIO access
+ * @axi_ck: pointer to application layer MMIO channel operating clock
+ * @aux_ck: pointer to pe2_mac_bridge and pe2_mac_core operating clock
+ * when pcie_mac_ck/pcie_pipe_ck is turned off
+ * @obff_ck: pointer to OBFF functional block operating clock
+ * @pipe_ck: pointer to LTSSM and PHY/MAC layer operating clock
+ * @phy: pointer to PHY control block
+ * @slot: port slot
+ * @irq: GIC irq
+ * @irq_domain: legacy INTx IRQ domain
+ * @inner_domain: inner IRQ domain
+ * @msi_domain: MSI IRQ domain
+ * @lock: protect the msi_irq_in_use bitmap
+ * @msi_irq_in_use: bit map for assigned MSI IRQ
+ */
+struct mtk_pcie_port {
+ void __iomem *base;
+ struct list_head list;
+ struct mtk_pcie *pcie;
+ struct reset_control *reset;
+ struct clk *sys_ck;
+ struct clk *ahb_ck;
+ struct clk *axi_ck;
+ struct clk *aux_ck;
+ struct clk *obff_ck;
+ struct clk *pipe_ck;
+ struct phy *phy;
+ u32 slot;
+ int irq;
+ struct irq_domain *irq_domain;
+ struct irq_domain *inner_domain;
+ struct irq_domain *msi_domain;
+ struct mutex lock;
+ DECLARE_BITMAP(msi_irq_in_use, MTK_MSI_IRQS_NUM);
+};
+
+/**
+ * struct mtk_pcie - PCIe host information
+ * @dev: pointer to PCIe device
+ * @base: IO mapped register base
+ * @free_ck: free-run reference clock
+ * @mem: non-prefetchable memory resource
+ * @ports: pointer to PCIe port information
+ * @soc: pointer to SoC-dependent operations
+ * @busnr: root bus number
+ */
+struct mtk_pcie {
+ struct device *dev;
+ void __iomem *base;
+ struct clk *free_ck;
+
+ struct resource mem;
+ struct list_head ports;
+ const struct mtk_pcie_soc *soc;
+ unsigned int busnr;
+};
+
+static void mtk_pcie_subsys_powerdown(struct mtk_pcie *pcie)
+{
+ struct device *dev = pcie->dev;
+
+ clk_disable_unprepare(pcie->free_ck);
+
+ pm_runtime_put_sync(dev);
+ pm_runtime_disable(dev);
+}
+
+static void mtk_pcie_port_free(struct mtk_pcie_port *port)
+{
+ struct mtk_pcie *pcie = port->pcie;
+ struct device *dev = pcie->dev;
+
+ devm_iounmap(dev, port->base);
+ list_del(&port->list);
+ devm_kfree(dev, port);
+}
+
+static void mtk_pcie_put_resources(struct mtk_pcie *pcie)
+{
+ struct mtk_pcie_port *port, *tmp;
+
+ list_for_each_entry_safe(port, tmp, &pcie->ports, list) {
+ phy_power_off(port->phy);
+ phy_exit(port->phy);
+ clk_disable_unprepare(port->pipe_ck);
+ clk_disable_unprepare(port->obff_ck);
+ clk_disable_unprepare(port->axi_ck);
+ clk_disable_unprepare(port->aux_ck);
+ clk_disable_unprepare(port->ahb_ck);
+ clk_disable_unprepare(port->sys_ck);
+ mtk_pcie_port_free(port);
+ }
+
+ mtk_pcie_subsys_powerdown(pcie);
+}
+
+static int mtk_pcie_check_cfg_cpld(struct mtk_pcie_port *port)
+{
+ u32 val;
+ int err;
+
+ err = readl_poll_timeout_atomic(port->base + PCIE_APP_TLP_REQ, val,
+ !(val & APP_CFG_REQ), 10,
+ 100 * USEC_PER_MSEC);
+ if (err)
+ return PCIBIOS_SET_FAILED;
+
+ if (readl(port->base + PCIE_APP_TLP_REQ) & APP_CPL_STATUS)
+ return PCIBIOS_SET_FAILED;
+
+ return PCIBIOS_SUCCESSFUL;
+}
+
+static int mtk_pcie_hw_rd_cfg(struct mtk_pcie_port *port, u32 bus, u32 devfn,
+ int where, int size, u32 *val)
+{
+ u32 tmp;
+
+ /* Write PCIe configuration transaction header for Cfgrd */
+ writel(CFG_HEADER_DW0(CFG_WRRD_TYPE_0, CFG_RD_FMT),
+ port->base + PCIE_CFG_HEADER0);
+ writel(CFG_HEADER_DW1(where, size), port->base + PCIE_CFG_HEADER1);
+ writel(CFG_HEADER_DW2(where, PCI_FUNC(devfn), PCI_SLOT(devfn), bus),
+ port->base + PCIE_CFG_HEADER2);
+
+ /* Trigger h/w to transmit Cfgrd TLP */
+ tmp = readl(port->base + PCIE_APP_TLP_REQ);
+ tmp |= APP_CFG_REQ;
+ writel(tmp, port->base + PCIE_APP_TLP_REQ);
+
+ /* Check completion status */
+ if (mtk_pcie_check_cfg_cpld(port))
+ return PCIBIOS_SET_FAILED;
+
+ /* Read cpld payload of Cfgrd */
+ *val = readl(port->base + PCIE_CFG_RDATA);
+
+ if (size == 1)
+ *val = (*val >> (8 * (where & 3))) & 0xff;
+ else if (size == 2)
+ *val = (*val >> (8 * (where & 3))) & 0xffff;
+
+ return PCIBIOS_SUCCESSFUL;
+}
+
+static int mtk_pcie_hw_wr_cfg(struct mtk_pcie_port *port, u32 bus, u32 devfn,
+ int where, int size, u32 val)
+{
+ /* Write PCIe configuration transaction header for Cfgwr */
+ writel(CFG_HEADER_DW0(CFG_WRRD_TYPE_0, CFG_WR_FMT),
+ port->base + PCIE_CFG_HEADER0);
+ writel(CFG_HEADER_DW1(where, size), port->base + PCIE_CFG_HEADER1);
+ writel(CFG_HEADER_DW2(where, PCI_FUNC(devfn), PCI_SLOT(devfn), bus),
+ port->base + PCIE_CFG_HEADER2);
+
+ /* Write Cfgwr data */
+ val = val << 8 * (where & 3);
+ writel(val, port->base + PCIE_CFG_WDATA);
+
+ /* Trigger h/w to transmit Cfgwr TLP */
+ val = readl(port->base + PCIE_APP_TLP_REQ);
+ val |= APP_CFG_REQ;
+ writel(val, port->base + PCIE_APP_TLP_REQ);
+
+ /* Check completion status */
+ return mtk_pcie_check_cfg_cpld(port);
+}
+
+static struct mtk_pcie_port *mtk_pcie_find_port(struct pci_bus *bus,
+ unsigned int devfn)
+{
+ struct mtk_pcie *pcie = bus->sysdata;
+ struct mtk_pcie_port *port;
+ struct pci_dev *dev = NULL;
+
+ /*
+ * Walk the bus hierarchy to get the devfn value
+ * of the port in the root bus.
+ */
+ while (bus && bus->number) {
+ dev = bus->self;
+ bus = dev->bus;
+ devfn = dev->devfn;
+ }
+
+ list_for_each_entry(port, &pcie->ports, list)
+ if (port->slot == PCI_SLOT(devfn))
+ return port;
+
+ return NULL;
+}
+
+static int mtk_pcie_config_read(struct pci_bus *bus, unsigned int devfn,
+ int where, int size, u32 *val)
+{
+ struct mtk_pcie_port *port;
+ u32 bn = bus->number;
+ int ret;
+
+ port = mtk_pcie_find_port(bus, devfn);
+ if (!port) {
+ *val = ~0;
+ return PCIBIOS_DEVICE_NOT_FOUND;
+ }
+
+ ret = mtk_pcie_hw_rd_cfg(port, bn, devfn, where, size, val);
+ if (ret)
+ *val = ~0;
+
+ return ret;
+}
+
+static int mtk_pcie_config_write(struct pci_bus *bus, unsigned int devfn,
+ int where, int size, u32 val)
+{
+ struct mtk_pcie_port *port;
+ u32 bn = bus->number;
+
+ port = mtk_pcie_find_port(bus, devfn);
+ if (!port)
+ return PCIBIOS_DEVICE_NOT_FOUND;
+
+ return mtk_pcie_hw_wr_cfg(port, bn, devfn, where, size, val);
+}
+
+static struct pci_ops mtk_pcie_ops_v2 = {
+ .read = mtk_pcie_config_read,
+ .write = mtk_pcie_config_write,
+};
+
+static void mtk_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
+{
+ struct mtk_pcie_port *port = irq_data_get_irq_chip_data(data);
+ phys_addr_t addr;
+
+ /* MT2712/MT7622 only support 32-bit MSI addresses */
+ addr = virt_to_phys(port->base + PCIE_MSI_VECTOR);
+ msg->address_hi = 0;
+ msg->address_lo = lower_32_bits(addr);
+
+ msg->data = data->hwirq;
+
+ dev_dbg(port->pcie->dev, "msi#%d address_hi %#x address_lo %#x\n",
+ (int)data->hwirq, msg->address_hi, msg->address_lo);
+}
+
+static int mtk_msi_set_affinity(struct irq_data *irq_data,
+ const struct cpumask *mask, bool force)
+{
+ return -EINVAL;
+}
+
+static void mtk_msi_ack_irq(struct irq_data *data)
+{
+ struct mtk_pcie_port *port = irq_data_get_irq_chip_data(data);
+ u32 hwirq = data->hwirq;
+
+ writel(1 << hwirq, port->base + PCIE_IMSI_STATUS);
+}
+
+static struct irq_chip mtk_msi_bottom_irq_chip = {
+ .name = "MTK MSI",
+ .irq_compose_msi_msg = mtk_compose_msi_msg,
+ .irq_set_affinity = mtk_msi_set_affinity,
+ .irq_ack = mtk_msi_ack_irq,
+};
+
+static int mtk_pcie_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
+ unsigned int nr_irqs, void *args)
+{
+ struct mtk_pcie_port *port = domain->host_data;
+ unsigned long bit;
+
+ WARN_ON(nr_irqs != 1);
+ mutex_lock(&port->lock);
+
+ bit = find_first_zero_bit(port->msi_irq_in_use, MTK_MSI_IRQS_NUM);
+ if (bit >= MTK_MSI_IRQS_NUM) {
+ mutex_unlock(&port->lock);
+ return -ENOSPC;
+ }
+
+ __set_bit(bit, port->msi_irq_in_use);
+
+ mutex_unlock(&port->lock);
+
+ irq_domain_set_info(domain, virq, bit, &mtk_msi_bottom_irq_chip,
+ domain->host_data, handle_edge_irq,
+ NULL, NULL);
+
+ return 0;
+}
+
+static void mtk_pcie_irq_domain_free(struct irq_domain *domain,
+ unsigned int virq, unsigned int nr_irqs)
+{
+ struct irq_data *d = irq_domain_get_irq_data(domain, virq);
+ struct mtk_pcie_port *port = irq_data_get_irq_chip_data(d);
+
+ mutex_lock(&port->lock);
+
+ if (!test_bit(d->hwirq, port->msi_irq_in_use))
+ dev_err(port->pcie->dev, "trying to free unused MSI#%lu\n",
+ d->hwirq);
+ else
+ __clear_bit(d->hwirq, port->msi_irq_in_use);
+
+ mutex_unlock(&port->lock);
+
+ irq_domain_free_irqs_parent(domain, virq, nr_irqs);
+}
+
+static const struct irq_domain_ops msi_domain_ops = {
+ .alloc = mtk_pcie_irq_domain_alloc,
+ .free = mtk_pcie_irq_domain_free,
+};
+
+static struct irq_chip mtk_msi_irq_chip = {
+ .name = "MTK PCIe MSI",
+ .irq_ack = irq_chip_ack_parent,
+ .irq_mask = pci_msi_mask_irq,
+ .irq_unmask = pci_msi_unmask_irq,
+};
+
+static struct msi_domain_info mtk_msi_domain_info = {
+ .flags = (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS |
+ MSI_FLAG_PCI_MSIX),
+ .chip = &mtk_msi_irq_chip,
+};
+
+static int mtk_pcie_allocate_msi_domains(struct mtk_pcie_port *port)
+{
+ struct fwnode_handle *fwnode = of_node_to_fwnode(port->pcie->dev->of_node);
+
+ mutex_init(&port->lock);
+
+ port->inner_domain = irq_domain_create_linear(fwnode, MTK_MSI_IRQS_NUM,
+ &msi_domain_ops, port);
+ if (!port->inner_domain) {
+ dev_err(port->pcie->dev, "failed to create IRQ domain\n");
+ return -ENOMEM;
+ }
+
+ port->msi_domain = pci_msi_create_irq_domain(fwnode, &mtk_msi_domain_info,
+ port->inner_domain);
+ if (!port->msi_domain) {
+ dev_err(port->pcie->dev, "failed to create MSI domain\n");
+ irq_domain_remove(port->inner_domain);
+ return -ENOMEM;
+ }
+
+ return 0;
+}
+
+static void mtk_pcie_enable_msi(struct mtk_pcie_port *port)
+{
+ u32 val;
+ phys_addr_t msg_addr;
+
+ msg_addr = virt_to_phys(port->base + PCIE_MSI_VECTOR);
+ val = lower_32_bits(msg_addr);
+ writel(val, port->base + PCIE_IMSI_ADDR);
+
+ val = readl(port->base + PCIE_INT_MASK);
+ val &= ~MSI_MASK;
+ writel(val, port->base + PCIE_INT_MASK);
+}
+
+static void mtk_pcie_irq_teardown(struct mtk_pcie *pcie)
+{
+ struct mtk_pcie_port *port, *tmp;
+
+ list_for_each_entry_safe(port, tmp, &pcie->ports, list) {
+ irq_set_chained_handler_and_data(port->irq, NULL, NULL);
+
+ if (port->irq_domain)
+ irq_domain_remove(port->irq_domain);
+
+ if (IS_ENABLED(CONFIG_PCI_MSI)) {
+ if (port->msi_domain)
+ irq_domain_remove(port->msi_domain);
+ if (port->inner_domain)
+ irq_domain_remove(port->inner_domain);
+ }
+
+ irq_dispose_mapping(port->irq);
+ }
+}
+
+static int mtk_pcie_intx_map(struct irq_domain *domain, unsigned int irq,
+ irq_hw_number_t hwirq)
+{
+ irq_set_chip_and_handler(irq, &dummy_irq_chip, handle_simple_irq);
+ irq_set_chip_data(irq, domain->host_data);
+
+ return 0;
+}
+
+static const struct irq_domain_ops intx_domain_ops = {
+ .map = mtk_pcie_intx_map,
+};
+
+static int mtk_pcie_init_irq_domain(struct mtk_pcie_port *port,
+ struct device_node *node)
+{
+ struct device *dev = port->pcie->dev;
+ struct device_node *pcie_intc_node;
+ int ret;
+
+ /* Setup INTx */
+ pcie_intc_node = of_get_next_child(node, NULL);
+ if (!pcie_intc_node) {
+ dev_err(dev, "no PCIe Intc node found\n");
+ return -ENODEV;
+ }
+
+ port->irq_domain = irq_domain_add_linear(pcie_intc_node, PCI_NUM_INTX,
+ &intx_domain_ops, port);
+ if (!port->irq_domain) {
+ dev_err(dev, "failed to get INTx IRQ domain\n");
+ return -ENODEV;
+ }
+
+ if (IS_ENABLED(CONFIG_PCI_MSI)) {
+ ret = mtk_pcie_allocate_msi_domains(port);
+ if (ret)
+ return ret;
+ }
+
+ return 0;
+}
+
+static void mtk_pcie_intr_handler(struct irq_desc *desc)
+{
+ struct mtk_pcie_port *port = irq_desc_get_handler_data(desc);
+ struct irq_chip *irqchip = irq_desc_get_chip(desc);
+ unsigned long status;
+ u32 virq;
+ u32 bit = INTX_SHIFT;
+
+ chained_irq_enter(irqchip, desc);
+
+ status = readl(port->base + PCIE_INT_STATUS);
+ if (status & INTX_MASK) {
+ for_each_set_bit_from(bit, &status, PCI_NUM_INTX + INTX_SHIFT) {
+ /* Clear the INTx */
+ writel(1 << bit, port->base + PCIE_INT_STATUS);
+ virq = irq_find_mapping(port->irq_domain,
+ bit - INTX_SHIFT);
+ generic_handle_irq(virq);
+ }
+ }
+
+ if (IS_ENABLED(CONFIG_PCI_MSI)) {
+ if (status & MSI_STATUS){
+ unsigned long imsi_status;
+
+ while ((imsi_status = readl(port->base + PCIE_IMSI_STATUS))) {
+ for_each_set_bit(bit, &imsi_status, MTK_MSI_IRQS_NUM) {
+ virq = irq_find_mapping(port->inner_domain, bit);
+ generic_handle_irq(virq);
+ }
+ }
+ /* Clear MSI interrupt status */
+ writel(MSI_STATUS, port->base + PCIE_INT_STATUS);
+ }
+ }
+
+ chained_irq_exit(irqchip, desc);
+
+ return;
+}
+
+static int mtk_pcie_setup_irq(struct mtk_pcie_port *port,
+ struct device_node *node)
+{
+ struct mtk_pcie *pcie = port->pcie;
+ struct device *dev = pcie->dev;
+ struct platform_device *pdev = to_platform_device(dev);
+ int err;
+
+ err = mtk_pcie_init_irq_domain(port, node);
+ if (err) {
+ dev_err(dev, "failed to init PCIe IRQ domain\n");
+ return err;
+ }
+
+ port->irq = platform_get_irq(pdev, port->slot);
+ irq_set_chained_handler_and_data(port->irq,
+ mtk_pcie_intr_handler, port);
+
+ return 0;
+}
+
+static int mtk_pcie_startup_port_v2(struct mtk_pcie_port *port)
+{
+ struct mtk_pcie *pcie = port->pcie;
+ struct resource *mem = &pcie->mem;
+ const struct mtk_pcie_soc *soc = port->pcie->soc;
+ u32 val;
+ size_t size;
+ int err;
+
+ /* MT7622 platforms need to enable LTSSM and ASPM from PCIe subsys */
+ if (pcie->base) {
+ val = readl(pcie->base + PCIE_SYS_CFG_V2);
+ val |= PCIE_CSR_LTSSM_EN(port->slot) |
+ PCIE_CSR_ASPM_L1_EN(port->slot);
+ writel(val, pcie->base + PCIE_SYS_CFG_V2);
+ }
+
+ /* Assert all reset signals */
+ writel(0, port->base + PCIE_RST_CTRL);
+
+ /*
+ * Enable PCIe link down reset, if link status changed from link up to
+ * link down, this will reset MAC control registers and configuration
+ * space.
+ */
+ writel(PCIE_LINKDOWN_RST_EN, port->base + PCIE_RST_CTRL);
+
+ /* De-assert PHY, PE, PIPE, MAC and configuration reset */
+ val = readl(port->base + PCIE_RST_CTRL);
+ val |= PCIE_PHY_RSTB | PCIE_PIPE_SRSTB |
+ PCIE_MAC_SRSTB | PCIE_CRSTB;
+ writel(val, port->base + PCIE_RST_CTRL);
+
+ usleep_range(100 * 1000, 120 * 1000);
+ val |= PCIE_PERSTB;
+ writel(val, port->base + PCIE_RST_CTRL);
+
+ /* Set up vendor ID and class code */
+ if (soc->need_fix_class_id) {
+ val = PCI_VENDOR_ID_MEDIATEK;
+ writew(val, port->base + PCIE_CONF_VEND_ID);
+
+ val = PCI_CLASS_BRIDGE_PCI;
+ writew(val, port->base + PCIE_CONF_CLASS_ID);
+ }
+
+ /* 100ms timeout value should be enough for Gen1/2 training */
+ err = readl_poll_timeout(port->base + PCIE_LINK_STATUS_V2, val,
+ !!(val & PCIE_PORT_LINKUP_V2), 20,
+ 100 * USEC_PER_MSEC);
+ if (err)
+ return -ETIMEDOUT;
+
+ /* Set INTx mask */
+ val = readl(port->base + PCIE_INT_MASK);
+ val &= ~INTX_MASK;
+ writel(val, port->base + PCIE_INT_MASK);
+
+ if (IS_ENABLED(CONFIG_PCI_MSI))
+ mtk_pcie_enable_msi(port);
+
+ /* Set AHB to PCIe translation windows */
+ size = mem->end - mem->start;
+ val = lower_32_bits(mem->start) | AHB2PCIE_SIZE(fls(size));
+ writel(val, port->base + PCIE_AHB_TRANS_BASE0_L);
+
+ val = upper_32_bits(mem->start);
+ writel(val, port->base + PCIE_AHB_TRANS_BASE0_H);
+
+ /* Set PCIe to AXI translation memory space.*/
+ val = fls(0xffffffff) | WIN_ENABLE;
+ writel(val, port->base + PCIE_AXI_WINDOW0);
+
+ return 0;
+}
+
+static void __iomem *mtk_pcie_map_bus(struct pci_bus *bus,
+ unsigned int devfn, int where)
+{
+ struct mtk_pcie *pcie = bus->sysdata;
+
+ writel(PCIE_CONF_ADDR(where, PCI_FUNC(devfn), PCI_SLOT(devfn),
+ bus->number), pcie->base + PCIE_CFG_ADDR);
+
+ return pcie->base + PCIE_CFG_DATA + (where & 3);
+}
+
+static struct pci_ops mtk_pcie_ops = {
+ .map_bus = mtk_pcie_map_bus,
+ .read = pci_generic_config_read,
+ .write = pci_generic_config_write,
+};
+
+static int mtk_pcie_startup_port(struct mtk_pcie_port *port)
+{
+ struct mtk_pcie *pcie = port->pcie;
+ u32 func = PCI_FUNC(port->slot << 3);
+ u32 slot = PCI_SLOT(port->slot << 3);
+ u32 val;
+ int err;
+
+ /* assert port PERST_N */
+ val = readl(pcie->base + PCIE_SYS_CFG);
+ val |= PCIE_PORT_PERST(port->slot);
+ writel(val, pcie->base + PCIE_SYS_CFG);
+
+ /* de-assert port PERST_N */
+ val = readl(pcie->base + PCIE_SYS_CFG);
+ val &= ~PCIE_PORT_PERST(port->slot);
+ writel(val, pcie->base + PCIE_SYS_CFG);
+
+ /* 100ms timeout value should be enough for Gen1/2 training */
+ err = readl_poll_timeout(port->base + PCIE_LINK_STATUS, val,
+ !!(val & PCIE_PORT_LINKUP), 20,
+ 100 * USEC_PER_MSEC);
+ if (err)
+ return -ETIMEDOUT;
+
+ /* enable interrupt */
+ val = readl(pcie->base + PCIE_INT_ENABLE);
+ val |= PCIE_PORT_INT_EN(port->slot);
+ writel(val, pcie->base + PCIE_INT_ENABLE);
+
+ /* map to all DDR region. We need to set it before cfg operation. */
+ writel(PCIE_BAR_MAP_MAX | PCIE_BAR_ENABLE,
+ port->base + PCIE_BAR0_SETUP);
+
+ /* configure class code and revision ID */
+ writel(PCIE_CLASS_CODE | PCIE_REVISION_ID, port->base + PCIE_CLASS);
+
+ /* configure FC credit */
+ writel(PCIE_CONF_ADDR(PCIE_FC_CREDIT, func, slot, 0),
+ pcie->base + PCIE_CFG_ADDR);
+ val = readl(pcie->base + PCIE_CFG_DATA);
+ val &= ~PCIE_FC_CREDIT_MASK;
+ val |= PCIE_FC_CREDIT_VAL(0x806c);
+ writel(PCIE_CONF_ADDR(PCIE_FC_CREDIT, func, slot, 0),
+ pcie->base + PCIE_CFG_ADDR);
+ writel(val, pcie->base + PCIE_CFG_DATA);
+
+ /* configure RC FTS number to 250 when it leaves L0s */
+ writel(PCIE_CONF_ADDR(PCIE_FTS_NUM, func, slot, 0),
+ pcie->base + PCIE_CFG_ADDR);
+ val = readl(pcie->base + PCIE_CFG_DATA);
+ val &= ~PCIE_FTS_NUM_MASK;
+ val |= PCIE_FTS_NUM_L0(0x50);
+ writel(PCIE_CONF_ADDR(PCIE_FTS_NUM, func, slot, 0),
+ pcie->base + PCIE_CFG_ADDR);
+ writel(val, pcie->base + PCIE_CFG_DATA);
+
+ return 0;
+}
+
+static void mtk_pcie_enable_port(struct mtk_pcie_port *port)
+{
+ struct mtk_pcie *pcie = port->pcie;
+ struct device *dev = pcie->dev;
+ int err;
+
+ err = clk_prepare_enable(port->sys_ck);
+ if (err) {
+ dev_err(dev, "failed to enable sys_ck%d clock\n", port->slot);
+ goto err_sys_clk;
+ }
+
+ err = clk_prepare_enable(port->ahb_ck);
+ if (err) {
+ dev_err(dev, "failed to enable ahb_ck%d\n", port->slot);
+ goto err_ahb_clk;
+ }
+
+ err = clk_prepare_enable(port->aux_ck);
+ if (err) {
+ dev_err(dev, "failed to enable aux_ck%d\n", port->slot);
+ goto err_aux_clk;
+ }
+
+ err = clk_prepare_enable(port->axi_ck);
+ if (err) {
+ dev_err(dev, "failed to enable axi_ck%d\n", port->slot);
+ goto err_axi_clk;
+ }
+
+ err = clk_prepare_enable(port->obff_ck);
+ if (err) {
+ dev_err(dev, "failed to enable obff_ck%d\n", port->slot);
+ goto err_obff_clk;
+ }
+
+ err = clk_prepare_enable(port->pipe_ck);
+ if (err) {
+ dev_err(dev, "failed to enable pipe_ck%d\n", port->slot);
+ goto err_pipe_clk;
+ }
+
+ reset_control_assert(port->reset);
+ reset_control_deassert(port->reset);
+
+ err = phy_init(port->phy);
+ if (err) {
+ dev_err(dev, "failed to initialize port%d phy\n", port->slot);
+ goto err_phy_init;
+ }
+
+ err = phy_power_on(port->phy);
+ if (err) {
+ dev_err(dev, "failed to power on port%d phy\n", port->slot);
+ goto err_phy_on;
+ }
+
+ err = phy_set_mode(port->phy, PHY_MODE_PCIE);
+ if (err) {
+ dev_err(dev, "failed to set mode on port%d phy\n", port->slot);
+ goto err_phy_on;
+ }
+
+ if (!pcie->soc->startup(port))
+ return;
+
+ dev_info(dev, "Port%d link down\n", port->slot);
+
+ phy_power_off(port->phy);
+err_phy_on:
+ phy_exit(port->phy);
+err_phy_init:
+ clk_disable_unprepare(port->pipe_ck);
+err_pipe_clk:
+ clk_disable_unprepare(port->obff_ck);
+err_obff_clk:
+ clk_disable_unprepare(port->axi_ck);
+err_axi_clk:
+ clk_disable_unprepare(port->aux_ck);
+err_aux_clk:
+ clk_disable_unprepare(port->ahb_ck);
+err_ahb_clk:
+ clk_disable_unprepare(port->sys_ck);
+err_sys_clk:
+ mtk_pcie_port_free(port);
+}
+
+static int mtk_pcie_parse_port(struct mtk_pcie *pcie,
+ struct device_node *node,
+ int slot)
+{
+ struct mtk_pcie_port *port;
+ struct resource *regs;
+ struct device *dev = pcie->dev;
+ struct platform_device *pdev = to_platform_device(dev);
+ char name[10];
+ int err;
+
+ port = devm_kzalloc(dev, sizeof(*port), GFP_KERNEL);
+ if (!port)
+ return -ENOMEM;
+
+ snprintf(name, sizeof(name), "port%d", slot);
+ regs = platform_get_resource_byname(pdev, IORESOURCE_MEM, name);
+ port->base = devm_ioremap_resource(dev, regs);
+ if (IS_ERR(port->base)) {
+ dev_err(dev, "failed to map port%d base\n", slot);
+ return PTR_ERR(port->base);
+ }
+
+ snprintf(name, sizeof(name), "sys_ck%d", slot);
+ port->sys_ck = devm_clk_get(dev, name);
+ if (IS_ERR(port->sys_ck)) {
+ dev_err(dev, "failed to get sys_ck%d clock\n", slot);
+ return PTR_ERR(port->sys_ck);
+ }
+
+ /* sys_ck might be divided into the following parts in some chips */
+ snprintf(name, sizeof(name), "ahb_ck%d", slot);
+ port->ahb_ck = devm_clk_get(dev, name);
+ if (IS_ERR(port->ahb_ck)) {
+ if (PTR_ERR(port->ahb_ck) == -EPROBE_DEFER)
+ return -EPROBE_DEFER;
+
+ port->ahb_ck = NULL;
+ }
+
+ snprintf(name, sizeof(name), "axi_ck%d", slot);
+ port->axi_ck = devm_clk_get(dev, name);
+ if (IS_ERR(port->axi_ck)) {
+ if (PTR_ERR(port->axi_ck) == -EPROBE_DEFER)
+ return -EPROBE_DEFER;
+
+ port->axi_ck = NULL;
+ }
+
+ snprintf(name, sizeof(name), "aux_ck%d", slot);
+ port->aux_ck = devm_clk_get(dev, name);
+ if (IS_ERR(port->aux_ck)) {
+ if (PTR_ERR(port->aux_ck) == -EPROBE_DEFER)
+ return -EPROBE_DEFER;
+
+ port->aux_ck = NULL;
+ }
+
+ snprintf(name, sizeof(name), "obff_ck%d", slot);
+ port->obff_ck = devm_clk_get(dev, name);
+ if (IS_ERR(port->obff_ck)) {
+ if (PTR_ERR(port->obff_ck) == -EPROBE_DEFER)
+ return -EPROBE_DEFER;
+
+ port->obff_ck = NULL;
+ }
+
+ snprintf(name, sizeof(name), "pipe_ck%d", slot);
+ port->pipe_ck = devm_clk_get(dev, name);
+ if (IS_ERR(port->pipe_ck)) {
+ if (PTR_ERR(port->pipe_ck) == -EPROBE_DEFER)
+ return -EPROBE_DEFER;
+
+ port->pipe_ck = NULL;
+ }
+
+ snprintf(name, sizeof(name), "pcie-rst%d", slot);
+ port->reset = devm_reset_control_get_optional_exclusive(dev, name);
+ if (PTR_ERR(port->reset) == -EPROBE_DEFER)
+ return PTR_ERR(port->reset);
+
+ /* some platforms may use default PHY setting */
+ snprintf(name, sizeof(name), "pcie-phy%d", slot);
+ port->phy = devm_phy_optional_get(dev, name);
+ if (IS_ERR(port->phy))
+ return PTR_ERR(port->phy);
+
+ port->slot = slot;
+ port->pcie = pcie;
+
+ if (pcie->soc->setup_irq) {
+ err = pcie->soc->setup_irq(port, node);
+ if (err)
+ return err;
+ }
+
+ INIT_LIST_HEAD(&port->list);
+ list_add_tail(&port->list, &pcie->ports);
+
+ return 0;
+}
+
+static int mtk_pcie_subsys_powerup(struct mtk_pcie *pcie)
+{
+ struct device *dev = pcie->dev;
+ struct platform_device *pdev = to_platform_device(dev);
+ struct resource *regs;
+ int err;
+
+ /* get shared registers, which are optional */
+ regs = platform_get_resource_byname(pdev, IORESOURCE_MEM, "subsys");
+ if (regs) {
+ pcie->base = devm_ioremap_resource(dev, regs);
+ if (IS_ERR(pcie->base)) {
+ dev_err(dev, "failed to map shared register\n");
+ return PTR_ERR(pcie->base);
+ }
+ }
+
+ pcie->free_ck = devm_clk_get(dev, "free_ck");
+ if (IS_ERR(pcie->free_ck)) {
+ if (PTR_ERR(pcie->free_ck) == -EPROBE_DEFER)
+ return -EPROBE_DEFER;
+
+ pcie->free_ck = NULL;
+ }
+
+ pm_runtime_enable(dev);
+ pm_runtime_get_sync(dev);
+
+ /* enable top level clock */
+ err = clk_prepare_enable(pcie->free_ck);
+ if (err) {
+ dev_err(dev, "failed to enable free_ck\n");
+ goto err_free_ck;
+ }
+
+ return 0;
+
+err_free_ck:
+ pm_runtime_put_sync(dev);
+ pm_runtime_disable(dev);
+
+ return err;
+}
+
+static int mtk_pcie_setup(struct mtk_pcie *pcie)
+{
+ struct device *dev = pcie->dev;
+ struct device_node *node = dev->of_node, *child;
+ struct mtk_pcie_port *port, *tmp;
+ struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie);
+ struct list_head *windows = &host->windows;
+ struct resource_entry *win, *tmp_win;
+ resource_size_t io_base;
+ int err;
+
+ err = of_pci_get_host_bridge_resources(node, 0, 0xff,
+ windows, &io_base);
+ if (err)
+ return err;
+
+ err = devm_request_pci_bus_resources(dev, windows);
+ if (err < 0)
+ return err;
+
+ /* Get the I/O and memory ranges from DT */
+ resource_list_for_each_entry_safe(win, tmp_win, windows) {
+ switch (resource_type(win->res)) {
+ case IORESOURCE_IO:
+ err = devm_pci_remap_iospace(dev, win->res, io_base);
+ if (err) {
+ dev_warn(dev, "error %d: failed to map resource %pR\n",
+ err, win->res);
+ resource_list_destroy_entry(win);
+ }
+ break;
+ case IORESOURCE_MEM:
+ memcpy(&pcie->mem, win->res, sizeof(*win->res));
+ pcie->mem.name = "non-prefetchable";
+ break;
+ case IORESOURCE_BUS:
+ pcie->busnr = win->res->start;
+ break;
+ }
+ }
+
+ for_each_available_child_of_node(node, child) {
+ int slot;
+
+ err = of_pci_get_devfn(child);
+ if (err < 0) {
+ dev_err(dev, "failed to parse devfn: %d\n", err);
+ return err;
+ }
+
+ slot = PCI_SLOT(err);
+
+ err = mtk_pcie_parse_port(pcie, child, slot);
+ if (err)
+ return err;
+ }
+
+ err = mtk_pcie_subsys_powerup(pcie);
+ if (err)
+ return err;
+
+ /* enable each port, and then check link status */
+ list_for_each_entry_safe(port, tmp, &pcie->ports, list)
+ mtk_pcie_enable_port(port);
+
+ /* power down PCIe subsys if slots are all empty (link down) */
+ if (list_empty(&pcie->ports))
+ mtk_pcie_subsys_powerdown(pcie);
+
+ return 0;
+}
+
+static int mtk_pcie_probe(struct platform_device *pdev)
+{
+ struct device *dev = &pdev->dev;
+ struct mtk_pcie *pcie;
+ struct pci_host_bridge *host;
+ int err;
+
+ host = devm_pci_alloc_host_bridge(dev, sizeof(*pcie));
+ if (!host)
+ return -ENOMEM;
+
+ pcie = pci_host_bridge_priv(host);
+
+ pcie->dev = dev;
+ pcie->soc = of_device_get_match_data(dev);
+ platform_set_drvdata(pdev, pcie);
+ INIT_LIST_HEAD(&pcie->ports);
+
+ err = mtk_pcie_setup(pcie);
+ if (err)
+ return err;
+
+ host->busnr = pcie->busnr;
+ host->dev.parent = pcie->dev;
+ host->ops = pcie->soc->ops;
+ host->map_irq = of_irq_parse_and_map_pci;
+ host->swizzle_irq = pci_common_swizzle;
+ host->sysdata = pcie;
+
+ err = pci_host_probe(host);
+ if (err)
+ goto put_resources;
+
+ return 0;
+
+put_resources:
+ if (!list_empty(&pcie->ports))
+ mtk_pcie_put_resources(pcie);
+
+ return err;
+}
+
+static void mtk_pcie_free_resources(struct mtk_pcie *pcie)
+{
+ struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie);
+ struct list_head *windows = &host->windows;
+
+ pci_free_resource_list(windows);
+}
+
+static int mtk_pcie_remove(struct platform_device *pdev)
+{
+ struct mtk_pcie *pcie = platform_get_drvdata(pdev);
+ struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie);
+
+ pci_stop_root_bus(host->bus);
+ pci_remove_root_bus(host->bus);
+ mtk_pcie_free_resources(pcie);
+
+ mtk_pcie_irq_teardown(pcie);
+
+ mtk_pcie_put_resources(pcie);
+
+ return 0;
+}
+
+static int __maybe_unused mtk_pcie_suspend_noirq(struct device *dev)
+{
+ struct mtk_pcie *pcie = dev_get_drvdata(dev);
+ struct mtk_pcie_port *port;
+ int ret;
+
+ if (list_empty(&pcie->ports))
+ return 0;
+
+ list_for_each_entry(port, &pcie->ports, list) {
+ clk_disable_unprepare(port->pipe_ck);
+ clk_disable_unprepare(port->obff_ck);
+ clk_disable_unprepare(port->axi_ck);
+ clk_disable_unprepare(port->aux_ck);
+ clk_disable_unprepare(port->ahb_ck);
+ clk_disable_unprepare(port->sys_ck);
+ phy_power_off(port->phy);
+ phy_exit(port->phy);
+ }
+
+ clk_disable_unprepare(pcie->free_ck);
+
+ ret = pinctrl_pm_select_sleep_state(dev);
+ if (ret)
+ dev_info(dev, "Failed to set pin sleep state (%d)\n", ret);
+
+ return 0;
+}
+
+static int __maybe_unused mtk_pcie_resume_noirq(struct device *dev)
+{
+ struct mtk_pcie *pcie = dev_get_drvdata(dev);
+ struct mtk_pcie_port *port, *tmp;
+ int ret;
+
+ if (list_empty(&pcie->ports))
+ return 0;
+
+ ret = pinctrl_pm_select_default_state(dev);
+ if (ret) {
+ dev_info(dev, "Failed to set pin default state (%d)\n", ret);
+ return ret;
+ }
+
+ clk_prepare_enable(pcie->free_ck);
+
+ list_for_each_entry_safe(port, tmp, &pcie->ports, list)
+ mtk_pcie_enable_port(port);
+
+ /* In case of EP was removed while system suspend. */
+ if (list_empty(&pcie->ports))
+ clk_disable_unprepare(pcie->free_ck);
+
+ return 0;
+}
+
+static const struct dev_pm_ops mtk_pcie_pm_ops = {
+ SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(mtk_pcie_suspend_noirq,
+ mtk_pcie_resume_noirq)
+};
+
+static const struct mtk_pcie_soc mtk_pcie_soc_v1 = {
+ .ops = &mtk_pcie_ops,
+ .startup = mtk_pcie_startup_port,
+};
+
+static const struct mtk_pcie_soc mtk_pcie_soc_v2_common = {
+ .ops = &mtk_pcie_ops_v2,
+ .startup = mtk_pcie_startup_port_v2,
+ .setup_irq = mtk_pcie_setup_irq,
+};
+
+static const struct mtk_pcie_soc mtk_pcie_soc_mt7622 = {
+ .need_fix_class_id = true,
+ .ops = &mtk_pcie_ops_v2,
+ .startup = mtk_pcie_startup_port_v2,
+ .setup_irq = mtk_pcie_setup_irq,
+};
+
+static const struct of_device_id mtk_pcie_ids[] = {
+ {
+ .compatible = "mediatek,mt2701-pcie",
+ .data = &mtk_pcie_soc_v1
+ },
+ {
+ .compatible = "mediatek,mt7623-pcie",
+ .data = &mtk_pcie_soc_v1
+ },
+ {
+ .compatible = "mediatek,mt2712-pcie",
+ .data = &mtk_pcie_soc_v2_common
+ },
+ {
+ .compatible = "mediatek,mt2731-pcie",
+ .data = &mtk_pcie_soc_v2_common
+ },
+ {
+ .compatible = "mediatek,mt7622-pcie",
+ .data = &mtk_pcie_soc_mt7622
+ },
+ {},
+};
+
+static struct platform_driver mtk_pcie_driver = {
+ .probe = mtk_pcie_probe,
+ .remove = mtk_pcie_remove,
+ .driver = {
+ .name = "mtk-pcie",
+ .of_match_table = mtk_pcie_ids,
+ .suppress_bind_attrs = true,
+ .pm = &mtk_pcie_pm_ops,
+ },
+};
+module_platform_driver(mtk_pcie_driver);
+MODULE_LICENSE("GPL v2");
+
+static void mtk_fixup_bar0(struct pci_dev *dev)
+{
+ struct resource *dev_res = &dev->resource[0];
+ /* 32bit resource length will calculate size to 0, set it smaller */
+ if ((dev->class >> 8) == PCI_CLASS_BRIDGE_PCI)
+ dev_res->end = 0xfffffffe;
+}
+DECLARE_PCI_FIXUP_HEADER(0x14c3, 0x2731, mtk_fixup_bar0);
diff --git a/src/kernel/linux/v4.14/drivers/pci/host/pcie-rcar.c b/src/kernel/linux/v4.14/drivers/pci/host/pcie-rcar.c
new file mode 100644
index 0000000..0077afc
--- /dev/null
+++ b/src/kernel/linux/v4.14/drivers/pci/host/pcie-rcar.c
@@ -0,0 +1,1220 @@
+/*
+ * PCIe driver for Renesas R-Car SoCs
+ * Copyright (C) 2014 Renesas Electronics Europe Ltd
+ *
+ * Based on:
+ * arch/sh/drivers/pci/pcie-sh7786.c
+ * arch/sh/drivers/pci/ops-sh7786.c
+ * Copyright (C) 2009 - 2011 Paul Mundt
+ *
+ * Author: Phil Edworthy <phil.edworthy@renesas.com>
+ *
+ * This file is licensed under the terms of the GNU General Public
+ * License version 2. This program is licensed "as is" without any
+ * warranty of any kind, whether express or implied.
+ */
+
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/interrupt.h>
+#include <linux/irq.h>
+#include <linux/irqdomain.h>
+#include <linux/kernel.h>
+#include <linux/init.h>
+#include <linux/msi.h>
+#include <linux/of_address.h>
+#include <linux/of_irq.h>
+#include <linux/of_pci.h>
+#include <linux/of_platform.h>
+#include <linux/pci.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/slab.h>
+
+#define PCIECAR 0x000010
+#define PCIECCTLR 0x000018
+#define CONFIG_SEND_ENABLE (1 << 31)
+#define TYPE0 (0 << 8)
+#define TYPE1 (1 << 8)
+#define PCIECDR 0x000020
+#define PCIEMSR 0x000028
+#define PCIEINTXR 0x000400
+#define PCIEMSITXR 0x000840
+
+/* Transfer control */
+#define PCIETCTLR 0x02000
+#define CFINIT 1
+#define PCIETSTR 0x02004
+#define DATA_LINK_ACTIVE 1
+#define PCIEERRFR 0x02020
+#define UNSUPPORTED_REQUEST (1 << 4)
+#define PCIEMSIFR 0x02044
+#define PCIEMSIALR 0x02048
+#define MSIFE 1
+#define PCIEMSIAUR 0x0204c
+#define PCIEMSIIER 0x02050
+
+/* root port address */
+#define PCIEPRAR(x) (0x02080 + ((x) * 0x4))
+
+/* local address reg & mask */
+#define PCIELAR(x) (0x02200 + ((x) * 0x20))
+#define PCIELAMR(x) (0x02208 + ((x) * 0x20))
+#define LAM_PREFETCH (1 << 3)
+#define LAM_64BIT (1 << 2)
+#define LAR_ENABLE (1 << 1)
+
+/* PCIe address reg & mask */
+#define PCIEPALR(x) (0x03400 + ((x) * 0x20))
+#define PCIEPAUR(x) (0x03404 + ((x) * 0x20))
+#define PCIEPAMR(x) (0x03408 + ((x) * 0x20))
+#define PCIEPTCTLR(x) (0x0340c + ((x) * 0x20))
+#define PAR_ENABLE (1 << 31)
+#define IO_SPACE (1 << 8)
+
+/* Configuration */
+#define PCICONF(x) (0x010000 + ((x) * 0x4))
+#define PMCAP(x) (0x010040 + ((x) * 0x4))
+#define EXPCAP(x) (0x010070 + ((x) * 0x4))
+#define VCCAP(x) (0x010100 + ((x) * 0x4))
+
+/* link layer */
+#define IDSETR1 0x011004
+#define TLCTLR 0x011048
+#define MACSR 0x011054
+#define SPCHGFIN (1 << 4)
+#define SPCHGFAIL (1 << 6)
+#define SPCHGSUC (1 << 7)
+#define LINK_SPEED (0xf << 16)
+#define LINK_SPEED_2_5GTS (1 << 16)
+#define LINK_SPEED_5_0GTS (2 << 16)
+#define MACCTLR 0x011058
+#define SPEED_CHANGE (1 << 24)
+#define SCRAMBLE_DISABLE (1 << 27)
+#define MACS2R 0x011078
+#define MACCGSPSETR 0x011084
+#define SPCNGRSN (1 << 31)
+
+/* R-Car H1 PHY */
+#define H1_PCIEPHYADRR 0x04000c
+#define WRITE_CMD (1 << 16)
+#define PHY_ACK (1 << 24)
+#define RATE_POS 12
+#define LANE_POS 8
+#define ADR_POS 0
+#define H1_PCIEPHYDOUTR 0x040014
+#define H1_PCIEPHYSR 0x040018
+
+/* R-Car Gen2 PHY */
+#define GEN2_PCIEPHYADDR 0x780
+#define GEN2_PCIEPHYDATA 0x784
+#define GEN2_PCIEPHYCTRL 0x78c
+
+#define INT_PCI_MSI_NR 32
+
+#define RCONF(x) (PCICONF(0)+(x))
+#define RPMCAP(x) (PMCAP(0)+(x))
+#define REXPCAP(x) (EXPCAP(0)+(x))
+#define RVCCAP(x) (VCCAP(0)+(x))
+
+#define PCIE_CONF_BUS(b) (((b) & 0xff) << 24)
+#define PCIE_CONF_DEV(d) (((d) & 0x1f) << 19)
+#define PCIE_CONF_FUNC(f) (((f) & 0x7) << 16)
+
+#define RCAR_PCI_MAX_RESOURCES 4
+#define MAX_NR_INBOUND_MAPS 6
+
+struct rcar_msi {
+ DECLARE_BITMAP(used, INT_PCI_MSI_NR);
+ struct irq_domain *domain;
+ struct msi_controller chip;
+ unsigned long pages;
+ struct mutex lock;
+ int irq1;
+ int irq2;
+};
+
+static inline struct rcar_msi *to_rcar_msi(struct msi_controller *chip)
+{
+ return container_of(chip, struct rcar_msi, chip);
+}
+
+/* Structure representing the PCIe interface */
+struct rcar_pcie {
+ struct device *dev;
+ void __iomem *base;
+ struct list_head resources;
+ int root_bus_nr;
+ struct clk *clk;
+ struct clk *bus_clk;
+ struct rcar_msi msi;
+};
+
+static void rcar_pci_write_reg(struct rcar_pcie *pcie, unsigned long val,
+ unsigned long reg)
+{
+ writel(val, pcie->base + reg);
+}
+
+static unsigned long rcar_pci_read_reg(struct rcar_pcie *pcie,
+ unsigned long reg)
+{
+ return readl(pcie->base + reg);
+}
+
+enum {
+ RCAR_PCI_ACCESS_READ,
+ RCAR_PCI_ACCESS_WRITE,
+};
+
+static void rcar_rmw32(struct rcar_pcie *pcie, int where, u32 mask, u32 data)
+{
+ int shift = 8 * (where & 3);
+ u32 val = rcar_pci_read_reg(pcie, where & ~3);
+
+ val &= ~(mask << shift);
+ val |= data << shift;
+ rcar_pci_write_reg(pcie, val, where & ~3);
+}
+
+static u32 rcar_read_conf(struct rcar_pcie *pcie, int where)
+{
+ int shift = 8 * (where & 3);
+ u32 val = rcar_pci_read_reg(pcie, where & ~3);
+
+ return val >> shift;
+}
+
+/* Serialization is provided by 'pci_lock' in drivers/pci/access.c */
+static int rcar_pcie_config_access(struct rcar_pcie *pcie,
+ unsigned char access_type, struct pci_bus *bus,
+ unsigned int devfn, int where, u32 *data)
+{
+ int dev, func, reg, index;
+
+ dev = PCI_SLOT(devfn);
+ func = PCI_FUNC(devfn);
+ reg = where & ~3;
+ index = reg / 4;
+
+ /*
+ * While each channel has its own memory-mapped extended config
+ * space, it's generally only accessible when in endpoint mode.
+ * When in root complex mode, the controller is unable to target
+ * itself with either type 0 or type 1 accesses, and indeed, any
+ * controller initiated target transfer to its own config space
+ * result in a completer abort.
+ *
+ * Each channel effectively only supports a single device, but as
+ * the same channel <-> device access works for any PCI_SLOT()
+ * value, we cheat a bit here and bind the controller's config
+ * space to devfn 0 in order to enable self-enumeration. In this
+ * case the regular ECAR/ECDR path is sidelined and the mangled
+ * config access itself is initiated as an internal bus transaction.
+ */
+ if (pci_is_root_bus(bus)) {
+ if (dev != 0)
+ return PCIBIOS_DEVICE_NOT_FOUND;
+
+ if (access_type == RCAR_PCI_ACCESS_READ) {
+ *data = rcar_pci_read_reg(pcie, PCICONF(index));
+ } else {
+ /* Keep an eye out for changes to the root bus number */
+ if (pci_is_root_bus(bus) && (reg == PCI_PRIMARY_BUS))
+ pcie->root_bus_nr = *data & 0xff;
+
+ rcar_pci_write_reg(pcie, *data, PCICONF(index));
+ }
+
+ return PCIBIOS_SUCCESSFUL;
+ }
+
+ if (pcie->root_bus_nr < 0)
+ return PCIBIOS_DEVICE_NOT_FOUND;
+
+ /* Clear errors */
+ rcar_pci_write_reg(pcie, rcar_pci_read_reg(pcie, PCIEERRFR), PCIEERRFR);
+
+ /* Set the PIO address */
+ rcar_pci_write_reg(pcie, PCIE_CONF_BUS(bus->number) |
+ PCIE_CONF_DEV(dev) | PCIE_CONF_FUNC(func) | reg, PCIECAR);
+
+ /* Enable the configuration access */
+ if (bus->parent->number == pcie->root_bus_nr)
+ rcar_pci_write_reg(pcie, CONFIG_SEND_ENABLE | TYPE0, PCIECCTLR);
+ else
+ rcar_pci_write_reg(pcie, CONFIG_SEND_ENABLE | TYPE1, PCIECCTLR);
+
+ /* Check for errors */
+ if (rcar_pci_read_reg(pcie, PCIEERRFR) & UNSUPPORTED_REQUEST)
+ return PCIBIOS_DEVICE_NOT_FOUND;
+
+ /* Check for master and target aborts */
+ if (rcar_read_conf(pcie, RCONF(PCI_STATUS)) &
+ (PCI_STATUS_REC_MASTER_ABORT | PCI_STATUS_REC_TARGET_ABORT))
+ return PCIBIOS_DEVICE_NOT_FOUND;
+
+ if (access_type == RCAR_PCI_ACCESS_READ)
+ *data = rcar_pci_read_reg(pcie, PCIECDR);
+ else
+ rcar_pci_write_reg(pcie, *data, PCIECDR);
+
+ /* Disable the configuration access */
+ rcar_pci_write_reg(pcie, 0, PCIECCTLR);
+
+ return PCIBIOS_SUCCESSFUL;
+}
+
+static int rcar_pcie_read_conf(struct pci_bus *bus, unsigned int devfn,
+ int where, int size, u32 *val)
+{
+ struct rcar_pcie *pcie = bus->sysdata;
+ int ret;
+
+ ret = rcar_pcie_config_access(pcie, RCAR_PCI_ACCESS_READ,
+ bus, devfn, where, val);
+ if (ret != PCIBIOS_SUCCESSFUL) {
+ *val = 0xffffffff;
+ return ret;
+ }
+
+ if (size == 1)
+ *val = (*val >> (8 * (where & 3))) & 0xff;
+ else if (size == 2)
+ *val = (*val >> (8 * (where & 2))) & 0xffff;
+
+ dev_dbg(&bus->dev, "pcie-config-read: bus=%3d devfn=0x%04x where=0x%04x size=%d val=0x%08lx\n",
+ bus->number, devfn, where, size, (unsigned long)*val);
+
+ return ret;
+}
+
+/* Serialization is provided by 'pci_lock' in drivers/pci/access.c */
+static int rcar_pcie_write_conf(struct pci_bus *bus, unsigned int devfn,
+ int where, int size, u32 val)
+{
+ struct rcar_pcie *pcie = bus->sysdata;
+ int shift, ret;
+ u32 data;
+
+ ret = rcar_pcie_config_access(pcie, RCAR_PCI_ACCESS_READ,
+ bus, devfn, where, &data);
+ if (ret != PCIBIOS_SUCCESSFUL)
+ return ret;
+
+ dev_dbg(&bus->dev, "pcie-config-write: bus=%3d devfn=0x%04x where=0x%04x size=%d val=0x%08lx\n",
+ bus->number, devfn, where, size, (unsigned long)val);
+
+ if (size == 1) {
+ shift = 8 * (where & 3);
+ data &= ~(0xff << shift);
+ data |= ((val & 0xff) << shift);
+ } else if (size == 2) {
+ shift = 8 * (where & 2);
+ data &= ~(0xffff << shift);
+ data |= ((val & 0xffff) << shift);
+ } else
+ data = val;
+
+ ret = rcar_pcie_config_access(pcie, RCAR_PCI_ACCESS_WRITE,
+ bus, devfn, where, &data);
+
+ return ret;
+}
+
+static struct pci_ops rcar_pcie_ops = {
+ .read = rcar_pcie_read_conf,
+ .write = rcar_pcie_write_conf,
+};
+
+static void rcar_pcie_setup_window(int win, struct rcar_pcie *pcie,
+ struct resource_entry *window)
+{
+ /* Setup PCIe address space mappings for each resource */
+ resource_size_t size;
+ resource_size_t res_start;
+ struct resource *res = window->res;
+ u32 mask;
+
+ rcar_pci_write_reg(pcie, 0x00000000, PCIEPTCTLR(win));
+
+ /*
+ * The PAMR mask is calculated in units of 128Bytes, which
+ * keeps things pretty simple.
+ */
+ size = resource_size(res);
+ mask = (roundup_pow_of_two(size) / SZ_128) - 1;
+ rcar_pci_write_reg(pcie, mask << 7, PCIEPAMR(win));
+
+ if (res->flags & IORESOURCE_IO)
+ res_start = pci_pio_to_address(res->start) - window->offset;
+ else
+ res_start = res->start - window->offset;
+
+ rcar_pci_write_reg(pcie, upper_32_bits(res_start), PCIEPAUR(win));
+ rcar_pci_write_reg(pcie, lower_32_bits(res_start) & ~0x7F,
+ PCIEPALR(win));
+
+ /* First resource is for IO */
+ mask = PAR_ENABLE;
+ if (res->flags & IORESOURCE_IO)
+ mask |= IO_SPACE;
+
+ rcar_pci_write_reg(pcie, mask, PCIEPTCTLR(win));
+}
+
+static int rcar_pcie_setup(struct list_head *resource, struct rcar_pcie *pci)
+{
+ struct resource_entry *win;
+ int i = 0;
+
+ /* Setup PCI resources */
+ resource_list_for_each_entry(win, &pci->resources) {
+ struct resource *res = win->res;
+
+ if (!res->flags)
+ continue;
+
+ switch (resource_type(res)) {
+ case IORESOURCE_IO:
+ case IORESOURCE_MEM:
+ rcar_pcie_setup_window(i, pci, win);
+ i++;
+ break;
+ case IORESOURCE_BUS:
+ pci->root_bus_nr = res->start;
+ break;
+ default:
+ continue;
+ }
+
+ pci_add_resource(resource, res);
+ }
+
+ return 1;
+}
+
+static void rcar_pcie_force_speedup(struct rcar_pcie *pcie)
+{
+ struct device *dev = pcie->dev;
+ unsigned int timeout = 1000;
+ u32 macsr;
+
+ if ((rcar_pci_read_reg(pcie, MACS2R) & LINK_SPEED) != LINK_SPEED_5_0GTS)
+ return;
+
+ if (rcar_pci_read_reg(pcie, MACCTLR) & SPEED_CHANGE) {
+ dev_err(dev, "Speed change already in progress\n");
+ return;
+ }
+
+ macsr = rcar_pci_read_reg(pcie, MACSR);
+ if ((macsr & LINK_SPEED) == LINK_SPEED_5_0GTS)
+ goto done;
+
+ /* Set target link speed to 5.0 GT/s */
+ rcar_rmw32(pcie, EXPCAP(12), PCI_EXP_LNKSTA_CLS,
+ PCI_EXP_LNKSTA_CLS_5_0GB);
+
+ /* Set speed change reason as intentional factor */
+ rcar_rmw32(pcie, MACCGSPSETR, SPCNGRSN, 0);
+
+ /* Clear SPCHGFIN, SPCHGSUC, and SPCHGFAIL */
+ if (macsr & (SPCHGFIN | SPCHGSUC | SPCHGFAIL))
+ rcar_pci_write_reg(pcie, macsr, MACSR);
+
+ /* Start link speed change */
+ rcar_rmw32(pcie, MACCTLR, SPEED_CHANGE, SPEED_CHANGE);
+
+ while (timeout--) {
+ macsr = rcar_pci_read_reg(pcie, MACSR);
+ if (macsr & SPCHGFIN) {
+ /* Clear the interrupt bits */
+ rcar_pci_write_reg(pcie, macsr, MACSR);
+
+ if (macsr & SPCHGFAIL)
+ dev_err(dev, "Speed change failed\n");
+
+ goto done;
+ }
+
+ msleep(1);
+ };
+
+ dev_err(dev, "Speed change timed out\n");
+
+done:
+ dev_info(dev, "Current link speed is %s GT/s\n",
+ (macsr & LINK_SPEED) == LINK_SPEED_5_0GTS ? "5" : "2.5");
+}
+
+static int rcar_pcie_enable(struct rcar_pcie *pcie)
+{
+ struct device *dev = pcie->dev;
+ struct pci_host_bridge *bridge = pci_host_bridge_from_priv(pcie);
+ struct pci_bus *bus, *child;
+ int ret;
+
+ /* Try setting 5 GT/s link speed */
+ rcar_pcie_force_speedup(pcie);
+
+ rcar_pcie_setup(&bridge->windows, pcie);
+
+ pci_add_flags(PCI_REASSIGN_ALL_RSRC | PCI_REASSIGN_ALL_BUS);
+
+ bridge->dev.parent = dev;
+ bridge->sysdata = pcie;
+ bridge->busnr = pcie->root_bus_nr;
+ bridge->ops = &rcar_pcie_ops;
+ bridge->map_irq = of_irq_parse_and_map_pci;
+ bridge->swizzle_irq = pci_common_swizzle;
+ if (IS_ENABLED(CONFIG_PCI_MSI))
+ bridge->msi = &pcie->msi.chip;
+
+ ret = pci_scan_root_bus_bridge(bridge);
+ if (ret < 0)
+ return ret;
+
+ bus = bridge->bus;
+
+ pci_bus_size_bridges(bus);
+ pci_bus_assign_resources(bus);
+
+ list_for_each_entry(child, &bus->children, node)
+ pcie_bus_configure_settings(child);
+
+ pci_bus_add_devices(bus);
+
+ return 0;
+}
+
+static int phy_wait_for_ack(struct rcar_pcie *pcie)
+{
+ struct device *dev = pcie->dev;
+ unsigned int timeout = 100;
+
+ while (timeout--) {
+ if (rcar_pci_read_reg(pcie, H1_PCIEPHYADRR) & PHY_ACK)
+ return 0;
+
+ udelay(100);
+ }
+
+ dev_err(dev, "Access to PCIe phy timed out\n");
+
+ return -ETIMEDOUT;
+}
+
+static void phy_write_reg(struct rcar_pcie *pcie,
+ unsigned int rate, unsigned int addr,
+ unsigned int lane, unsigned int data)
+{
+ unsigned long phyaddr;
+
+ phyaddr = WRITE_CMD |
+ ((rate & 1) << RATE_POS) |
+ ((lane & 0xf) << LANE_POS) |
+ ((addr & 0xff) << ADR_POS);
+
+ /* Set write data */
+ rcar_pci_write_reg(pcie, data, H1_PCIEPHYDOUTR);
+ rcar_pci_write_reg(pcie, phyaddr, H1_PCIEPHYADRR);
+
+ /* Ignore errors as they will be dealt with if the data link is down */
+ phy_wait_for_ack(pcie);
+
+ /* Clear command */
+ rcar_pci_write_reg(pcie, 0, H1_PCIEPHYDOUTR);
+ rcar_pci_write_reg(pcie, 0, H1_PCIEPHYADRR);
+
+ /* Ignore errors as they will be dealt with if the data link is down */
+ phy_wait_for_ack(pcie);
+}
+
+static int rcar_pcie_wait_for_dl(struct rcar_pcie *pcie)
+{
+ unsigned int timeout = 10;
+
+ while (timeout--) {
+ if ((rcar_pci_read_reg(pcie, PCIETSTR) & DATA_LINK_ACTIVE))
+ return 0;
+
+ msleep(5);
+ }
+
+ return -ETIMEDOUT;
+}
+
+static int rcar_pcie_hw_init(struct rcar_pcie *pcie)
+{
+ int err;
+
+ /* Begin initialization */
+ rcar_pci_write_reg(pcie, 0, PCIETCTLR);
+
+ /* Set mode */
+ rcar_pci_write_reg(pcie, 1, PCIEMSR);
+
+ /*
+ * Initial header for port config space is type 1, set the device
+ * class to match. Hardware takes care of propagating the IDSETR
+ * settings, so there is no need to bother with a quirk.
+ */
+ rcar_pci_write_reg(pcie, PCI_CLASS_BRIDGE_PCI << 16, IDSETR1);
+
+ /*
+ * Setup Secondary Bus Number & Subordinate Bus Number, even though
+ * they aren't used, to avoid bridge being detected as broken.
+ */
+ rcar_rmw32(pcie, RCONF(PCI_SECONDARY_BUS), 0xff, 1);
+ rcar_rmw32(pcie, RCONF(PCI_SUBORDINATE_BUS), 0xff, 1);
+
+ /* Initialize default capabilities. */
+ rcar_rmw32(pcie, REXPCAP(0), 0xff, PCI_CAP_ID_EXP);
+ rcar_rmw32(pcie, REXPCAP(PCI_EXP_FLAGS),
+ PCI_EXP_FLAGS_TYPE, PCI_EXP_TYPE_ROOT_PORT << 4);
+ rcar_rmw32(pcie, RCONF(PCI_HEADER_TYPE), 0x7f,
+ PCI_HEADER_TYPE_BRIDGE);
+
+ /* Enable data link layer active state reporting */
+ rcar_rmw32(pcie, REXPCAP(PCI_EXP_LNKCAP), PCI_EXP_LNKCAP_DLLLARC,
+ PCI_EXP_LNKCAP_DLLLARC);
+
+ /* Write out the physical slot number = 0 */
+ rcar_rmw32(pcie, REXPCAP(PCI_EXP_SLTCAP), PCI_EXP_SLTCAP_PSN, 0);
+
+ /* Set the completion timer timeout to the maximum 50ms. */
+ rcar_rmw32(pcie, TLCTLR + 1, 0x3f, 50);
+
+ /* Terminate list of capabilities (Next Capability Offset=0) */
+ rcar_rmw32(pcie, RVCCAP(0), 0xfff00000, 0);
+
+ /* Enable MSI */
+ if (IS_ENABLED(CONFIG_PCI_MSI))
+ rcar_pci_write_reg(pcie, 0x801f0000, PCIEMSITXR);
+
+ /* Finish initialization - establish a PCI Express link */
+ rcar_pci_write_reg(pcie, CFINIT, PCIETCTLR);
+
+ /* This will timeout if we don't have a link. */
+ err = rcar_pcie_wait_for_dl(pcie);
+ if (err)
+ return err;
+
+ /* Enable INTx interrupts */
+ rcar_rmw32(pcie, PCIEINTXR, 0, 0xF << 8);
+
+ wmb();
+
+ return 0;
+}
+
+static int rcar_pcie_hw_init_h1(struct rcar_pcie *pcie)
+{
+ unsigned int timeout = 10;
+
+ /* Initialize the phy */
+ phy_write_reg(pcie, 0, 0x42, 0x1, 0x0EC34191);
+ phy_write_reg(pcie, 1, 0x42, 0x1, 0x0EC34180);
+ phy_write_reg(pcie, 0, 0x43, 0x1, 0x00210188);
+ phy_write_reg(pcie, 1, 0x43, 0x1, 0x00210188);
+ phy_write_reg(pcie, 0, 0x44, 0x1, 0x015C0014);
+ phy_write_reg(pcie, 1, 0x44, 0x1, 0x015C0014);
+ phy_write_reg(pcie, 1, 0x4C, 0x1, 0x786174A0);
+ phy_write_reg(pcie, 1, 0x4D, 0x1, 0x048000BB);
+ phy_write_reg(pcie, 0, 0x51, 0x1, 0x079EC062);
+ phy_write_reg(pcie, 0, 0x52, 0x1, 0x20000000);
+ phy_write_reg(pcie, 1, 0x52, 0x1, 0x20000000);
+ phy_write_reg(pcie, 1, 0x56, 0x1, 0x00003806);
+
+ phy_write_reg(pcie, 0, 0x60, 0x1, 0x004B03A5);
+ phy_write_reg(pcie, 0, 0x64, 0x1, 0x3F0F1F0F);
+ phy_write_reg(pcie, 0, 0x66, 0x1, 0x00008000);
+
+ while (timeout--) {
+ if (rcar_pci_read_reg(pcie, H1_PCIEPHYSR))
+ return rcar_pcie_hw_init(pcie);
+
+ msleep(5);
+ }
+
+ return -ETIMEDOUT;
+}
+
+static int rcar_pcie_hw_init_gen2(struct rcar_pcie *pcie)
+{
+ /*
+ * These settings come from the R-Car Series, 2nd Generation User's
+ * Manual, section 50.3.1 (2) Initialization of the physical layer.
+ */
+ rcar_pci_write_reg(pcie, 0x000f0030, GEN2_PCIEPHYADDR);
+ rcar_pci_write_reg(pcie, 0x00381203, GEN2_PCIEPHYDATA);
+ rcar_pci_write_reg(pcie, 0x00000001, GEN2_PCIEPHYCTRL);
+ rcar_pci_write_reg(pcie, 0x00000006, GEN2_PCIEPHYCTRL);
+
+ rcar_pci_write_reg(pcie, 0x000f0054, GEN2_PCIEPHYADDR);
+ /* The following value is for DC connection, no termination resistor */
+ rcar_pci_write_reg(pcie, 0x13802007, GEN2_PCIEPHYDATA);
+ rcar_pci_write_reg(pcie, 0x00000001, GEN2_PCIEPHYCTRL);
+ rcar_pci_write_reg(pcie, 0x00000006, GEN2_PCIEPHYCTRL);
+
+ return rcar_pcie_hw_init(pcie);
+}
+
+static int rcar_msi_alloc(struct rcar_msi *chip)
+{
+ int msi;
+
+ mutex_lock(&chip->lock);
+
+ msi = find_first_zero_bit(chip->used, INT_PCI_MSI_NR);
+ if (msi < INT_PCI_MSI_NR)
+ set_bit(msi, chip->used);
+ else
+ msi = -ENOSPC;
+
+ mutex_unlock(&chip->lock);
+
+ return msi;
+}
+
+static int rcar_msi_alloc_region(struct rcar_msi *chip, int no_irqs)
+{
+ int msi;
+
+ mutex_lock(&chip->lock);
+ msi = bitmap_find_free_region(chip->used, INT_PCI_MSI_NR,
+ order_base_2(no_irqs));
+ mutex_unlock(&chip->lock);
+
+ return msi;
+}
+
+static void rcar_msi_free(struct rcar_msi *chip, unsigned long irq)
+{
+ mutex_lock(&chip->lock);
+ clear_bit(irq, chip->used);
+ mutex_unlock(&chip->lock);
+}
+
+static irqreturn_t rcar_pcie_msi_irq(int irq, void *data)
+{
+ struct rcar_pcie *pcie = data;
+ struct rcar_msi *msi = &pcie->msi;
+ struct device *dev = pcie->dev;
+ unsigned long reg;
+
+ reg = rcar_pci_read_reg(pcie, PCIEMSIFR);
+
+ /* MSI & INTx share an interrupt - we only handle MSI here */
+ if (!reg)
+ return IRQ_NONE;
+
+ while (reg) {
+ unsigned int index = find_first_bit(®, 32);
+ unsigned int irq;
+
+ /* clear the interrupt */
+ rcar_pci_write_reg(pcie, 1 << index, PCIEMSIFR);
+
+ irq = irq_find_mapping(msi->domain, index);
+ if (irq) {
+ if (test_bit(index, msi->used))
+ generic_handle_irq(irq);
+ else
+ dev_info(dev, "unhandled MSI\n");
+ } else {
+ /* Unknown MSI, just clear it */
+ dev_dbg(dev, "unexpected MSI\n");
+ }
+
+ /* see if there's any more pending in this vector */
+ reg = rcar_pci_read_reg(pcie, PCIEMSIFR);
+ }
+
+ return IRQ_HANDLED;
+}
+
+static int rcar_msi_setup_irq(struct msi_controller *chip, struct pci_dev *pdev,
+ struct msi_desc *desc)
+{
+ struct rcar_msi *msi = to_rcar_msi(chip);
+ struct rcar_pcie *pcie = container_of(chip, struct rcar_pcie, msi.chip);
+ struct msi_msg msg;
+ unsigned int irq;
+ int hwirq;
+
+ hwirq = rcar_msi_alloc(msi);
+ if (hwirq < 0)
+ return hwirq;
+
+ irq = irq_find_mapping(msi->domain, hwirq);
+ if (!irq) {
+ rcar_msi_free(msi, hwirq);
+ return -EINVAL;
+ }
+
+ irq_set_msi_desc(irq, desc);
+
+ msg.address_lo = rcar_pci_read_reg(pcie, PCIEMSIALR) & ~MSIFE;
+ msg.address_hi = rcar_pci_read_reg(pcie, PCIEMSIAUR);
+ msg.data = hwirq;
+
+ pci_write_msi_msg(irq, &msg);
+
+ return 0;
+}
+
+static int rcar_msi_setup_irqs(struct msi_controller *chip,
+ struct pci_dev *pdev, int nvec, int type)
+{
+ struct rcar_pcie *pcie = container_of(chip, struct rcar_pcie, msi.chip);
+ struct rcar_msi *msi = to_rcar_msi(chip);
+ struct msi_desc *desc;
+ struct msi_msg msg;
+ unsigned int irq;
+ int hwirq;
+ int i;
+
+ /* MSI-X interrupts are not supported */
+ if (type == PCI_CAP_ID_MSIX)
+ return -EINVAL;
+
+ WARN_ON(!list_is_singular(&pdev->dev.msi_list));
+ desc = list_entry(pdev->dev.msi_list.next, struct msi_desc, list);
+
+ hwirq = rcar_msi_alloc_region(msi, nvec);
+ if (hwirq < 0)
+ return -ENOSPC;
+
+ irq = irq_find_mapping(msi->domain, hwirq);
+ if (!irq)
+ return -ENOSPC;
+
+ for (i = 0; i < nvec; i++) {
+ /*
+ * irq_create_mapping() called from rcar_pcie_probe() pre-
+ * allocates descs, so there is no need to allocate descs here.
+ * We can therefore assume that if irq_find_mapping() above
+ * returns non-zero, then the descs are also successfully
+ * allocated.
+ */
+ if (irq_set_msi_desc_off(irq, i, desc)) {
+ /* TODO: clear */
+ return -EINVAL;
+ }
+ }
+
+ desc->nvec_used = nvec;
+ desc->msi_attrib.multiple = order_base_2(nvec);
+
+ msg.address_lo = rcar_pci_read_reg(pcie, PCIEMSIALR) & ~MSIFE;
+ msg.address_hi = rcar_pci_read_reg(pcie, PCIEMSIAUR);
+ msg.data = hwirq;
+
+ pci_write_msi_msg(irq, &msg);
+
+ return 0;
+}
+
+static void rcar_msi_teardown_irq(struct msi_controller *chip, unsigned int irq)
+{
+ struct rcar_msi *msi = to_rcar_msi(chip);
+ struct irq_data *d = irq_get_irq_data(irq);
+
+ rcar_msi_free(msi, d->hwirq);
+}
+
+static struct irq_chip rcar_msi_irq_chip = {
+ .name = "R-Car PCIe MSI",
+ .irq_enable = pci_msi_unmask_irq,
+ .irq_disable = pci_msi_mask_irq,
+ .irq_mask = pci_msi_mask_irq,
+ .irq_unmask = pci_msi_unmask_irq,
+};
+
+static int rcar_msi_map(struct irq_domain *domain, unsigned int irq,
+ irq_hw_number_t hwirq)
+{
+ irq_set_chip_and_handler(irq, &rcar_msi_irq_chip, handle_simple_irq);
+ irq_set_chip_data(irq, domain->host_data);
+
+ return 0;
+}
+
+static const struct irq_domain_ops msi_domain_ops = {
+ .map = rcar_msi_map,
+};
+
+static int rcar_pcie_enable_msi(struct rcar_pcie *pcie)
+{
+ struct device *dev = pcie->dev;
+ struct rcar_msi *msi = &pcie->msi;
+ phys_addr_t base;
+ int err, i;
+
+ mutex_init(&msi->lock);
+
+ msi->chip.dev = dev;
+ msi->chip.setup_irq = rcar_msi_setup_irq;
+ msi->chip.setup_irqs = rcar_msi_setup_irqs;
+ msi->chip.teardown_irq = rcar_msi_teardown_irq;
+
+ msi->domain = irq_domain_add_linear(dev->of_node, INT_PCI_MSI_NR,
+ &msi_domain_ops, &msi->chip);
+ if (!msi->domain) {
+ dev_err(dev, "failed to create IRQ domain\n");
+ return -ENOMEM;
+ }
+
+ for (i = 0; i < INT_PCI_MSI_NR; i++)
+ irq_create_mapping(msi->domain, i);
+
+ /* Two irqs are for MSI, but they are also used for non-MSI irqs */
+ err = devm_request_irq(dev, msi->irq1, rcar_pcie_msi_irq,
+ IRQF_SHARED | IRQF_NO_THREAD,
+ rcar_msi_irq_chip.name, pcie);
+ if (err < 0) {
+ dev_err(dev, "failed to request IRQ: %d\n", err);
+ goto err;
+ }
+
+ err = devm_request_irq(dev, msi->irq2, rcar_pcie_msi_irq,
+ IRQF_SHARED | IRQF_NO_THREAD,
+ rcar_msi_irq_chip.name, pcie);
+ if (err < 0) {
+ dev_err(dev, "failed to request IRQ: %d\n", err);
+ goto err;
+ }
+
+ /* setup MSI data target */
+ msi->pages = __get_free_pages(GFP_KERNEL, 0);
+ if (!msi->pages) {
+ err = -ENOMEM;
+ goto err;
+ }
+ base = virt_to_phys((void *)msi->pages);
+
+ rcar_pci_write_reg(pcie, lower_32_bits(base) | MSIFE, PCIEMSIALR);
+ rcar_pci_write_reg(pcie, upper_32_bits(base), PCIEMSIAUR);
+
+ /* enable all MSI interrupts */
+ rcar_pci_write_reg(pcie, 0xffffffff, PCIEMSIIER);
+
+ return 0;
+
+err:
+ irq_domain_remove(msi->domain);
+ return err;
+}
+
+static int rcar_pcie_get_resources(struct rcar_pcie *pcie)
+{
+ struct device *dev = pcie->dev;
+ struct resource res;
+ int err, i;
+
+ err = of_address_to_resource(dev->of_node, 0, &res);
+ if (err)
+ return err;
+
+ pcie->base = devm_ioremap_resource(dev, &res);
+ if (IS_ERR(pcie->base))
+ return PTR_ERR(pcie->base);
+
+ pcie->clk = devm_clk_get(dev, "pcie");
+ if (IS_ERR(pcie->clk)) {
+ dev_err(dev, "cannot get platform clock\n");
+ return PTR_ERR(pcie->clk);
+ }
+ err = clk_prepare_enable(pcie->clk);
+ if (err)
+ return err;
+
+ pcie->bus_clk = devm_clk_get(dev, "pcie_bus");
+ if (IS_ERR(pcie->bus_clk)) {
+ dev_err(dev, "cannot get pcie bus clock\n");
+ err = PTR_ERR(pcie->bus_clk);
+ goto fail_clk;
+ }
+ err = clk_prepare_enable(pcie->bus_clk);
+ if (err)
+ goto fail_clk;
+
+ i = irq_of_parse_and_map(dev->of_node, 0);
+ if (!i) {
+ dev_err(dev, "cannot get platform resources for msi interrupt\n");
+ err = -ENOENT;
+ goto err_map_reg;
+ }
+ pcie->msi.irq1 = i;
+
+ i = irq_of_parse_and_map(dev->of_node, 1);
+ if (!i) {
+ dev_err(dev, "cannot get platform resources for msi interrupt\n");
+ err = -ENOENT;
+ goto err_map_reg;
+ }
+ pcie->msi.irq2 = i;
+
+ return 0;
+
+err_map_reg:
+ clk_disable_unprepare(pcie->bus_clk);
+fail_clk:
+ clk_disable_unprepare(pcie->clk);
+
+ return err;
+}
+
+static int rcar_pcie_inbound_ranges(struct rcar_pcie *pcie,
+ struct of_pci_range *range,
+ int *index)
+{
+ u64 restype = range->flags;
+ u64 cpu_addr = range->cpu_addr;
+ u64 cpu_end = range->cpu_addr + range->size;
+ u64 pci_addr = range->pci_addr;
+ u32 flags = LAM_64BIT | LAR_ENABLE;
+ u64 mask;
+ u64 size;
+ int idx = *index;
+
+ if (restype & IORESOURCE_PREFETCH)
+ flags |= LAM_PREFETCH;
+
+ /*
+ * If the size of the range is larger than the alignment of the start
+ * address, we have to use multiple entries to perform the mapping.
+ */
+ if (cpu_addr > 0) {
+ unsigned long nr_zeros = __ffs64(cpu_addr);
+ u64 alignment = 1ULL << nr_zeros;
+
+ size = min(range->size, alignment);
+ } else {
+ size = range->size;
+ }
+ /* Hardware supports max 4GiB inbound region */
+ size = min(size, 1ULL << 32);
+
+ mask = roundup_pow_of_two(size) - 1;
+ mask &= ~0xf;
+
+ while (cpu_addr < cpu_end) {
+ /*
+ * Set up 64-bit inbound regions as the range parser doesn't
+ * distinguish between 32 and 64-bit types.
+ */
+ rcar_pci_write_reg(pcie, lower_32_bits(pci_addr),
+ PCIEPRAR(idx));
+ rcar_pci_write_reg(pcie, lower_32_bits(cpu_addr), PCIELAR(idx));
+ rcar_pci_write_reg(pcie, lower_32_bits(mask) | flags,
+ PCIELAMR(idx));
+
+ rcar_pci_write_reg(pcie, upper_32_bits(pci_addr),
+ PCIEPRAR(idx + 1));
+ rcar_pci_write_reg(pcie, upper_32_bits(cpu_addr),
+ PCIELAR(idx + 1));
+ rcar_pci_write_reg(pcie, 0, PCIELAMR(idx + 1));
+
+ pci_addr += size;
+ cpu_addr += size;
+ idx += 2;
+
+ if (idx > MAX_NR_INBOUND_MAPS) {
+ dev_err(pcie->dev, "Failed to map inbound regions!\n");
+ return -EINVAL;
+ }
+ }
+ *index = idx;
+
+ return 0;
+}
+
+static int pci_dma_range_parser_init(struct of_pci_range_parser *parser,
+ struct device_node *node)
+{
+ const int na = 3, ns = 2;
+ int rlen;
+
+ parser->node = node;
+ parser->pna = of_n_addr_cells(node);
+ parser->np = parser->pna + na + ns;
+
+ parser->range = of_get_property(node, "dma-ranges", &rlen);
+ if (!parser->range)
+ return -ENOENT;
+
+ parser->end = parser->range + rlen / sizeof(__be32);
+ return 0;
+}
+
+static int rcar_pcie_parse_map_dma_ranges(struct rcar_pcie *pcie,
+ struct device_node *np)
+{
+ struct of_pci_range range;
+ struct of_pci_range_parser parser;
+ int index = 0;
+ int err;
+
+ if (pci_dma_range_parser_init(&parser, np))
+ return -EINVAL;
+
+ /* Get the dma-ranges from DT */
+ for_each_of_pci_range(&parser, &range) {
+ u64 end = range.cpu_addr + range.size - 1;
+
+ dev_dbg(pcie->dev, "0x%08x 0x%016llx..0x%016llx -> 0x%016llx\n",
+ range.flags, range.cpu_addr, end, range.pci_addr);
+
+ err = rcar_pcie_inbound_ranges(pcie, &range, &index);
+ if (err)
+ return err;
+ }
+
+ return 0;
+}
+
+static const struct of_device_id rcar_pcie_of_match[] = {
+ { .compatible = "renesas,pcie-r8a7779", .data = rcar_pcie_hw_init_h1 },
+ { .compatible = "renesas,pcie-r8a7790",
+ .data = rcar_pcie_hw_init_gen2 },
+ { .compatible = "renesas,pcie-r8a7791",
+ .data = rcar_pcie_hw_init_gen2 },
+ { .compatible = "renesas,pcie-rcar-gen2",
+ .data = rcar_pcie_hw_init_gen2 },
+ { .compatible = "renesas,pcie-r8a7795", .data = rcar_pcie_hw_init },
+ { .compatible = "renesas,pcie-rcar-gen3", .data = rcar_pcie_hw_init },
+ {},
+};
+
+static int rcar_pcie_parse_request_of_pci_ranges(struct rcar_pcie *pci)
+{
+ int err;
+ struct device *dev = pci->dev;
+ struct device_node *np = dev->of_node;
+ resource_size_t iobase;
+ struct resource_entry *win, *tmp;
+
+ err = of_pci_get_host_bridge_resources(np, 0, 0xff, &pci->resources,
+ &iobase);
+ if (err)
+ return err;
+
+ err = devm_request_pci_bus_resources(dev, &pci->resources);
+ if (err)
+ goto out_release_res;
+
+ resource_list_for_each_entry_safe(win, tmp, &pci->resources) {
+ struct resource *res = win->res;
+
+ if (resource_type(res) == IORESOURCE_IO) {
+ err = devm_pci_remap_iospace(dev, res, iobase);
+ if (err) {
+ dev_warn(dev, "error %d: failed to map resource %pR\n",
+ err, res);
+
+ resource_list_destroy_entry(win);
+ }
+ }
+ }
+
+ return 0;
+
+out_release_res:
+ pci_free_resource_list(&pci->resources);
+ return err;
+}
+
+static int rcar_pcie_probe(struct platform_device *pdev)
+{
+ struct device *dev = &pdev->dev;
+ struct rcar_pcie *pcie;
+ unsigned int data;
+ int err;
+ int (*hw_init_fn)(struct rcar_pcie *);
+ struct pci_host_bridge *bridge;
+
+ bridge = pci_alloc_host_bridge(sizeof(*pcie));
+ if (!bridge)
+ return -ENOMEM;
+
+ pcie = pci_host_bridge_priv(bridge);
+
+ pcie->dev = dev;
+
+ INIT_LIST_HEAD(&pcie->resources);
+
+ err = rcar_pcie_parse_request_of_pci_ranges(pcie);
+ if (err)
+ goto err_free_bridge;
+
+ err = rcar_pcie_get_resources(pcie);
+ if (err < 0) {
+ dev_err(dev, "failed to request resources: %d\n", err);
+ goto err_free_resource_list;
+ }
+
+ err = rcar_pcie_parse_map_dma_ranges(pcie, dev->of_node);
+ if (err)
+ goto err_free_resource_list;
+
+ pm_runtime_enable(dev);
+ err = pm_runtime_get_sync(dev);
+ if (err < 0) {
+ dev_err(dev, "pm_runtime_get_sync failed\n");
+ goto err_pm_disable;
+ }
+
+ /* Failure to get a link might just be that no cards are inserted */
+ hw_init_fn = of_device_get_match_data(dev);
+ err = hw_init_fn(pcie);
+ if (err) {
+ dev_info(dev, "PCIe link down\n");
+ err = -ENODEV;
+ goto err_pm_put;
+ }
+
+ data = rcar_pci_read_reg(pcie, MACSR);
+ dev_info(dev, "PCIe x%d: link up\n", (data >> 20) & 0x3f);
+
+ if (IS_ENABLED(CONFIG_PCI_MSI)) {
+ err = rcar_pcie_enable_msi(pcie);
+ if (err < 0) {
+ dev_err(dev,
+ "failed to enable MSI support: %d\n",
+ err);
+ goto err_pm_put;
+ }
+ }
+
+ err = rcar_pcie_enable(pcie);
+ if (err)
+ goto err_pm_put;
+
+ return 0;
+
+err_pm_put:
+ pm_runtime_put(dev);
+
+err_pm_disable:
+ pm_runtime_disable(dev);
+
+err_free_resource_list:
+ pci_free_resource_list(&pcie->resources);
+err_free_bridge:
+ pci_free_host_bridge(bridge);
+
+ return err;
+}
+
+static struct platform_driver rcar_pcie_driver = {
+ .driver = {
+ .name = "rcar-pcie",
+ .of_match_table = rcar_pcie_of_match,
+ .suppress_bind_attrs = true,
+ },
+ .probe = rcar_pcie_probe,
+};
+builtin_platform_driver(rcar_pcie_driver);
diff --git a/src/kernel/linux/v4.14/drivers/pci/host/pcie-rockchip.c b/src/kernel/linux/v4.14/drivers/pci/host/pcie-rockchip.c
new file mode 100644
index 0000000..d3f9e7d
--- /dev/null
+++ b/src/kernel/linux/v4.14/drivers/pci/host/pcie-rockchip.c
@@ -0,0 +1,1712 @@
+/*
+ * Rockchip AXI PCIe host controller driver
+ *
+ * Copyright (c) 2016 Rockchip, Inc.
+ *
+ * Author: Shawn Lin <shawn.lin@rock-chips.com>
+ * Wenrui Li <wenrui.li@rock-chips.com>
+ *
+ * Bits taken from Synopsys DesignWare Host controller driver and
+ * ARM PCI Host generic driver.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include <linux/bitrev.h>
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/gpio/consumer.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/iopoll.h>
+#include <linux/irq.h>
+#include <linux/irqchip/chained_irq.h>
+#include <linux/irqdomain.h>
+#include <linux/kernel.h>
+#include <linux/mfd/syscon.h>
+#include <linux/module.h>
+#include <linux/of_address.h>
+#include <linux/of_device.h>
+#include <linux/of_pci.h>
+#include <linux/of_platform.h>
+#include <linux/of_irq.h>
+#include <linux/pci.h>
+#include <linux/pci_ids.h>
+#include <linux/phy/phy.h>
+#include <linux/platform_device.h>
+#include <linux/reset.h>
+#include <linux/regmap.h>
+
+/*
+ * The upper 16 bits of PCIE_CLIENT_CONFIG are a write mask for the lower 16
+ * bits. This allows atomic updates of the register without locking.
+ */
+#define HIWORD_UPDATE(mask, val) (((mask) << 16) | (val))
+#define HIWORD_UPDATE_BIT(val) HIWORD_UPDATE(val, val)
+
+#define ENCODE_LANES(x) ((((x) >> 1) & 3) << 4)
+#define MAX_LANE_NUM 4
+
+#define PCIE_CLIENT_BASE 0x0
+#define PCIE_CLIENT_CONFIG (PCIE_CLIENT_BASE + 0x00)
+#define PCIE_CLIENT_CONF_ENABLE HIWORD_UPDATE_BIT(0x0001)
+#define PCIE_CLIENT_LINK_TRAIN_ENABLE HIWORD_UPDATE_BIT(0x0002)
+#define PCIE_CLIENT_ARI_ENABLE HIWORD_UPDATE_BIT(0x0008)
+#define PCIE_CLIENT_CONF_LANE_NUM(x) HIWORD_UPDATE(0x0030, ENCODE_LANES(x))
+#define PCIE_CLIENT_MODE_RC HIWORD_UPDATE_BIT(0x0040)
+#define PCIE_CLIENT_GEN_SEL_1 HIWORD_UPDATE(0x0080, 0)
+#define PCIE_CLIENT_GEN_SEL_2 HIWORD_UPDATE_BIT(0x0080)
+#define PCIE_CLIENT_DEBUG_OUT_0 (PCIE_CLIENT_BASE + 0x3c)
+#define PCIE_CLIENT_DEBUG_LTSSM_MASK GENMASK(5, 0)
+#define PCIE_CLIENT_DEBUG_LTSSM_L1 0x18
+#define PCIE_CLIENT_DEBUG_LTSSM_L2 0x19
+#define PCIE_CLIENT_BASIC_STATUS1 (PCIE_CLIENT_BASE + 0x48)
+#define PCIE_CLIENT_LINK_STATUS_UP 0x00300000
+#define PCIE_CLIENT_LINK_STATUS_MASK 0x00300000
+#define PCIE_CLIENT_INT_MASK (PCIE_CLIENT_BASE + 0x4c)
+#define PCIE_CLIENT_INT_STATUS (PCIE_CLIENT_BASE + 0x50)
+#define PCIE_CLIENT_INTR_MASK GENMASK(8, 5)
+#define PCIE_CLIENT_INTR_SHIFT 5
+#define PCIE_CLIENT_INT_LEGACY_DONE BIT(15)
+#define PCIE_CLIENT_INT_MSG BIT(14)
+#define PCIE_CLIENT_INT_HOT_RST BIT(13)
+#define PCIE_CLIENT_INT_DPA BIT(12)
+#define PCIE_CLIENT_INT_FATAL_ERR BIT(11)
+#define PCIE_CLIENT_INT_NFATAL_ERR BIT(10)
+#define PCIE_CLIENT_INT_CORR_ERR BIT(9)
+#define PCIE_CLIENT_INT_INTD BIT(8)
+#define PCIE_CLIENT_INT_INTC BIT(7)
+#define PCIE_CLIENT_INT_INTB BIT(6)
+#define PCIE_CLIENT_INT_INTA BIT(5)
+#define PCIE_CLIENT_INT_LOCAL BIT(4)
+#define PCIE_CLIENT_INT_UDMA BIT(3)
+#define PCIE_CLIENT_INT_PHY BIT(2)
+#define PCIE_CLIENT_INT_HOT_PLUG BIT(1)
+#define PCIE_CLIENT_INT_PWR_STCG BIT(0)
+
+#define PCIE_CLIENT_INT_LEGACY \
+ (PCIE_CLIENT_INT_INTA | PCIE_CLIENT_INT_INTB | \
+ PCIE_CLIENT_INT_INTC | PCIE_CLIENT_INT_INTD)
+
+#define PCIE_CLIENT_INT_CLI \
+ (PCIE_CLIENT_INT_CORR_ERR | PCIE_CLIENT_INT_NFATAL_ERR | \
+ PCIE_CLIENT_INT_FATAL_ERR | PCIE_CLIENT_INT_DPA | \
+ PCIE_CLIENT_INT_HOT_RST | PCIE_CLIENT_INT_MSG | \
+ PCIE_CLIENT_INT_LEGACY_DONE | PCIE_CLIENT_INT_LEGACY | \
+ PCIE_CLIENT_INT_PHY)
+
+#define PCIE_CORE_CTRL_MGMT_BASE 0x900000
+#define PCIE_CORE_CTRL (PCIE_CORE_CTRL_MGMT_BASE + 0x000)
+#define PCIE_CORE_PL_CONF_SPEED_5G 0x00000008
+#define PCIE_CORE_PL_CONF_SPEED_MASK 0x00000018
+#define PCIE_CORE_PL_CONF_LANE_MASK 0x00000006
+#define PCIE_CORE_PL_CONF_LANE_SHIFT 1
+#define PCIE_CORE_CTRL_PLC1 (PCIE_CORE_CTRL_MGMT_BASE + 0x004)
+#define PCIE_CORE_CTRL_PLC1_FTS_MASK GENMASK(23, 8)
+#define PCIE_CORE_CTRL_PLC1_FTS_SHIFT 8
+#define PCIE_CORE_CTRL_PLC1_FTS_CNT 0xffff
+#define PCIE_CORE_TXCREDIT_CFG1 (PCIE_CORE_CTRL_MGMT_BASE + 0x020)
+#define PCIE_CORE_TXCREDIT_CFG1_MUI_MASK 0xFFFF0000
+#define PCIE_CORE_TXCREDIT_CFG1_MUI_SHIFT 16
+#define PCIE_CORE_TXCREDIT_CFG1_MUI_ENCODE(x) \
+ (((x) >> 3) << PCIE_CORE_TXCREDIT_CFG1_MUI_SHIFT)
+#define PCIE_CORE_LANE_MAP (PCIE_CORE_CTRL_MGMT_BASE + 0x200)
+#define PCIE_CORE_LANE_MAP_MASK 0x0000000f
+#define PCIE_CORE_LANE_MAP_REVERSE BIT(16)
+#define PCIE_CORE_INT_STATUS (PCIE_CORE_CTRL_MGMT_BASE + 0x20c)
+#define PCIE_CORE_INT_PRFPE BIT(0)
+#define PCIE_CORE_INT_CRFPE BIT(1)
+#define PCIE_CORE_INT_RRPE BIT(2)
+#define PCIE_CORE_INT_PRFO BIT(3)
+#define PCIE_CORE_INT_CRFO BIT(4)
+#define PCIE_CORE_INT_RT BIT(5)
+#define PCIE_CORE_INT_RTR BIT(6)
+#define PCIE_CORE_INT_PE BIT(7)
+#define PCIE_CORE_INT_MTR BIT(8)
+#define PCIE_CORE_INT_UCR BIT(9)
+#define PCIE_CORE_INT_FCE BIT(10)
+#define PCIE_CORE_INT_CT BIT(11)
+#define PCIE_CORE_INT_UTC BIT(18)
+#define PCIE_CORE_INT_MMVC BIT(19)
+#define PCIE_CORE_CONFIG_VENDOR (PCIE_CORE_CTRL_MGMT_BASE + 0x44)
+#define PCIE_CORE_INT_MASK (PCIE_CORE_CTRL_MGMT_BASE + 0x210)
+#define PCIE_RC_BAR_CONF (PCIE_CORE_CTRL_MGMT_BASE + 0x300)
+
+#define PCIE_CORE_INT \
+ (PCIE_CORE_INT_PRFPE | PCIE_CORE_INT_CRFPE | \
+ PCIE_CORE_INT_RRPE | PCIE_CORE_INT_CRFO | \
+ PCIE_CORE_INT_RT | PCIE_CORE_INT_RTR | \
+ PCIE_CORE_INT_PE | PCIE_CORE_INT_MTR | \
+ PCIE_CORE_INT_UCR | PCIE_CORE_INT_FCE | \
+ PCIE_CORE_INT_CT | PCIE_CORE_INT_UTC | \
+ PCIE_CORE_INT_MMVC)
+
+#define PCIE_RC_CONFIG_NORMAL_BASE 0x800000
+#define PCIE_RC_CONFIG_BASE 0xa00000
+#define PCIE_RC_CONFIG_RID_CCR (PCIE_RC_CONFIG_BASE + 0x08)
+#define PCIE_RC_CONFIG_SCC_SHIFT 16
+#define PCIE_RC_CONFIG_DCR (PCIE_RC_CONFIG_BASE + 0xc4)
+#define PCIE_RC_CONFIG_DCR_CSPL_SHIFT 18
+#define PCIE_RC_CONFIG_DCR_CSPL_LIMIT 0xff
+#define PCIE_RC_CONFIG_DCR_CPLS_SHIFT 26
+#define PCIE_RC_CONFIG_DCSR (PCIE_RC_CONFIG_BASE + 0xc8)
+#define PCIE_RC_CONFIG_DCSR_MPS_MASK GENMASK(7, 5)
+#define PCIE_RC_CONFIG_DCSR_MPS_256 (0x1 << 5)
+#define PCIE_RC_CONFIG_LINK_CAP (PCIE_RC_CONFIG_BASE + 0xcc)
+#define PCIE_RC_CONFIG_LINK_CAP_L0S BIT(10)
+#define PCIE_RC_CONFIG_LCS (PCIE_RC_CONFIG_BASE + 0xd0)
+#define PCIE_RC_CONFIG_L1_SUBSTATE_CTRL2 (PCIE_RC_CONFIG_BASE + 0x90c)
+#define PCIE_RC_CONFIG_THP_CAP (PCIE_RC_CONFIG_BASE + 0x274)
+#define PCIE_RC_CONFIG_THP_CAP_NEXT_MASK GENMASK(31, 20)
+
+#define PCIE_CORE_AXI_CONF_BASE 0xc00000
+#define PCIE_CORE_OB_REGION_ADDR0 (PCIE_CORE_AXI_CONF_BASE + 0x0)
+#define PCIE_CORE_OB_REGION_ADDR0_NUM_BITS 0x3f
+#define PCIE_CORE_OB_REGION_ADDR0_LO_ADDR 0xffffff00
+#define PCIE_CORE_OB_REGION_ADDR1 (PCIE_CORE_AXI_CONF_BASE + 0x4)
+#define PCIE_CORE_OB_REGION_DESC0 (PCIE_CORE_AXI_CONF_BASE + 0x8)
+#define PCIE_CORE_OB_REGION_DESC1 (PCIE_CORE_AXI_CONF_BASE + 0xc)
+
+#define PCIE_CORE_AXI_INBOUND_BASE 0xc00800
+#define PCIE_RP_IB_ADDR0 (PCIE_CORE_AXI_INBOUND_BASE + 0x0)
+#define PCIE_CORE_IB_REGION_ADDR0_NUM_BITS 0x3f
+#define PCIE_CORE_IB_REGION_ADDR0_LO_ADDR 0xffffff00
+#define PCIE_RP_IB_ADDR1 (PCIE_CORE_AXI_INBOUND_BASE + 0x4)
+
+/* Size of one AXI Region (not Region 0) */
+#define AXI_REGION_SIZE BIT(20)
+/* Size of Region 0, equal to sum of sizes of other regions */
+#define AXI_REGION_0_SIZE (32 * (0x1 << 20))
+#define OB_REG_SIZE_SHIFT 5
+#define IB_ROOT_PORT_REG_SIZE_SHIFT 3
+#define AXI_WRAPPER_IO_WRITE 0x6
+#define AXI_WRAPPER_MEM_WRITE 0x2
+#define AXI_WRAPPER_TYPE0_CFG 0xa
+#define AXI_WRAPPER_TYPE1_CFG 0xb
+#define AXI_WRAPPER_NOR_MSG 0xc
+
+#define MAX_AXI_IB_ROOTPORT_REGION_NUM 3
+#define MIN_AXI_ADDR_BITS_PASSED 8
+#define PCIE_RC_SEND_PME_OFF 0x11960
+#define ROCKCHIP_VENDOR_ID 0x1d87
+#define PCIE_ECAM_BUS(x) (((x) & 0xff) << 20)
+#define PCIE_ECAM_DEV(x) (((x) & 0x1f) << 15)
+#define PCIE_ECAM_FUNC(x) (((x) & 0x7) << 12)
+#define PCIE_ECAM_REG(x) (((x) & 0xfff) << 0)
+#define PCIE_ECAM_ADDR(bus, dev, func, reg) \
+ (PCIE_ECAM_BUS(bus) | PCIE_ECAM_DEV(dev) | \
+ PCIE_ECAM_FUNC(func) | PCIE_ECAM_REG(reg))
+#define PCIE_LINK_IS_L2(x) \
+ (((x) & PCIE_CLIENT_DEBUG_LTSSM_MASK) == PCIE_CLIENT_DEBUG_LTSSM_L2)
+#define PCIE_LINK_UP(x) \
+ (((x) & PCIE_CLIENT_LINK_STATUS_MASK) == PCIE_CLIENT_LINK_STATUS_UP)
+#define PCIE_LINK_IS_GEN2(x) \
+ (((x) & PCIE_CORE_PL_CONF_SPEED_MASK) == PCIE_CORE_PL_CONF_SPEED_5G)
+
+#define RC_REGION_0_ADDR_TRANS_H 0x00000000
+#define RC_REGION_0_ADDR_TRANS_L 0x00000000
+#define RC_REGION_0_PASS_BITS (25 - 1)
+#define RC_REGION_0_TYPE_MASK GENMASK(3, 0)
+#define MAX_AXI_WRAPPER_REGION_NUM 33
+
+struct rockchip_pcie {
+ void __iomem *reg_base; /* DT axi-base */
+ void __iomem *apb_base; /* DT apb-base */
+ bool legacy_phy;
+ struct phy *phys[MAX_LANE_NUM];
+ struct reset_control *core_rst;
+ struct reset_control *mgmt_rst;
+ struct reset_control *mgmt_sticky_rst;
+ struct reset_control *pipe_rst;
+ struct reset_control *pm_rst;
+ struct reset_control *aclk_rst;
+ struct reset_control *pclk_rst;
+ struct clk *aclk_pcie;
+ struct clk *aclk_perf_pcie;
+ struct clk *hclk_pcie;
+ struct clk *clk_pcie_pm;
+ struct regulator *vpcie12v; /* 12V power supply */
+ struct regulator *vpcie3v3; /* 3.3V power supply */
+ struct regulator *vpcie1v8; /* 1.8V power supply */
+ struct regulator *vpcie0v9; /* 0.9V power supply */
+ struct gpio_desc *ep_gpio;
+ u32 lanes;
+ u8 lanes_map;
+ u8 root_bus_nr;
+ int link_gen;
+ struct device *dev;
+ struct irq_domain *irq_domain;
+ int offset;
+ struct pci_bus *root_bus;
+ struct resource *io;
+ phys_addr_t io_bus_addr;
+ u32 io_size;
+ void __iomem *msg_region;
+ u32 mem_size;
+ phys_addr_t msg_bus_addr;
+ phys_addr_t mem_bus_addr;
+};
+
+static u32 rockchip_pcie_read(struct rockchip_pcie *rockchip, u32 reg)
+{
+ return readl(rockchip->apb_base + reg);
+}
+
+static void rockchip_pcie_write(struct rockchip_pcie *rockchip, u32 val,
+ u32 reg)
+{
+ writel(val, rockchip->apb_base + reg);
+}
+
+static void rockchip_pcie_enable_bw_int(struct rockchip_pcie *rockchip)
+{
+ u32 status;
+
+ status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS);
+ status |= (PCI_EXP_LNKCTL_LBMIE | PCI_EXP_LNKCTL_LABIE);
+ rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS);
+}
+
+static void rockchip_pcie_clr_bw_int(struct rockchip_pcie *rockchip)
+{
+ u32 status;
+
+ status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS);
+ status |= (PCI_EXP_LNKSTA_LBMS | PCI_EXP_LNKSTA_LABS) << 16;
+ rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS);
+}
+
+static void rockchip_pcie_update_txcredit_mui(struct rockchip_pcie *rockchip)
+{
+ u32 val;
+
+ /* Update Tx credit maximum update interval */
+ val = rockchip_pcie_read(rockchip, PCIE_CORE_TXCREDIT_CFG1);
+ val &= ~PCIE_CORE_TXCREDIT_CFG1_MUI_MASK;
+ val |= PCIE_CORE_TXCREDIT_CFG1_MUI_ENCODE(24000); /* ns */
+ rockchip_pcie_write(rockchip, val, PCIE_CORE_TXCREDIT_CFG1);
+}
+
+static int rockchip_pcie_valid_device(struct rockchip_pcie *rockchip,
+ struct pci_bus *bus, int dev)
+{
+ /* access only one slot on each root port */
+ if (bus->number == rockchip->root_bus_nr && dev > 0)
+ return 0;
+
+ /*
+ * do not read more than one device on the bus directly attached
+ * to RC's downstream side.
+ */
+ if (bus->primary == rockchip->root_bus_nr && dev > 0)
+ return 0;
+
+ return 1;
+}
+
+static u8 rockchip_pcie_lane_map(struct rockchip_pcie *rockchip)
+{
+ u32 val;
+ u8 map;
+
+ if (rockchip->legacy_phy)
+ return GENMASK(MAX_LANE_NUM - 1, 0);
+
+ val = rockchip_pcie_read(rockchip, PCIE_CORE_LANE_MAP);
+ map = val & PCIE_CORE_LANE_MAP_MASK;
+
+ /* The link may be using a reverse-indexed mapping. */
+ if (val & PCIE_CORE_LANE_MAP_REVERSE)
+ map = bitrev8(map) >> 4;
+
+ return map;
+}
+
+static int rockchip_pcie_rd_own_conf(struct rockchip_pcie *rockchip,
+ int where, int size, u32 *val)
+{
+ void __iomem *addr;
+
+ addr = rockchip->apb_base + PCIE_RC_CONFIG_NORMAL_BASE + where;
+
+ if (!IS_ALIGNED((uintptr_t)addr, size)) {
+ *val = 0;
+ return PCIBIOS_BAD_REGISTER_NUMBER;
+ }
+
+ if (size == 4) {
+ *val = readl(addr);
+ } else if (size == 2) {
+ *val = readw(addr);
+ } else if (size == 1) {
+ *val = readb(addr);
+ } else {
+ *val = 0;
+ return PCIBIOS_BAD_REGISTER_NUMBER;
+ }
+ return PCIBIOS_SUCCESSFUL;
+}
+
+static int rockchip_pcie_wr_own_conf(struct rockchip_pcie *rockchip,
+ int where, int size, u32 val)
+{
+ u32 mask, tmp, offset;
+ void __iomem *addr;
+
+ offset = where & ~0x3;
+ addr = rockchip->apb_base + PCIE_RC_CONFIG_NORMAL_BASE + offset;
+
+ if (size == 4) {
+ writel(val, addr);
+ return PCIBIOS_SUCCESSFUL;
+ }
+
+ mask = ~(((1 << (size * 8)) - 1) << ((where & 0x3) * 8));
+
+ /*
+ * N.B. This read/modify/write isn't safe in general because it can
+ * corrupt RW1C bits in adjacent registers. But the hardware
+ * doesn't support smaller writes.
+ */
+ tmp = readl(addr) & mask;
+ tmp |= val << ((where & 0x3) * 8);
+ writel(tmp, addr);
+
+ return PCIBIOS_SUCCESSFUL;
+}
+
+static void rockchip_pcie_cfg_configuration_accesses(
+ struct rockchip_pcie *rockchip, u32 type)
+{
+ u32 ob_desc_0;
+
+ /* Configuration Accesses for region 0 */
+ rockchip_pcie_write(rockchip, 0x0, PCIE_RC_BAR_CONF);
+
+ rockchip_pcie_write(rockchip,
+ (RC_REGION_0_ADDR_TRANS_L + RC_REGION_0_PASS_BITS),
+ PCIE_CORE_OB_REGION_ADDR0);
+ rockchip_pcie_write(rockchip, RC_REGION_0_ADDR_TRANS_H,
+ PCIE_CORE_OB_REGION_ADDR1);
+ ob_desc_0 = rockchip_pcie_read(rockchip, PCIE_CORE_OB_REGION_DESC0);
+ ob_desc_0 &= ~(RC_REGION_0_TYPE_MASK);
+ ob_desc_0 |= (type | (0x1 << 23));
+ rockchip_pcie_write(rockchip, ob_desc_0, PCIE_CORE_OB_REGION_DESC0);
+ rockchip_pcie_write(rockchip, 0x0, PCIE_CORE_OB_REGION_DESC1);
+}
+
+static int rockchip_pcie_rd_other_conf(struct rockchip_pcie *rockchip,
+ struct pci_bus *bus, u32 devfn,
+ int where, int size, u32 *val)
+{
+ u32 busdev;
+
+ busdev = PCIE_ECAM_ADDR(bus->number, PCI_SLOT(devfn),
+ PCI_FUNC(devfn), where);
+
+ if (!IS_ALIGNED(busdev, size)) {
+ *val = 0;
+ return PCIBIOS_BAD_REGISTER_NUMBER;
+ }
+
+ if (bus->parent->number == rockchip->root_bus_nr)
+ rockchip_pcie_cfg_configuration_accesses(rockchip,
+ AXI_WRAPPER_TYPE0_CFG);
+ else
+ rockchip_pcie_cfg_configuration_accesses(rockchip,
+ AXI_WRAPPER_TYPE1_CFG);
+
+ if (size == 4) {
+ *val = readl(rockchip->reg_base + busdev);
+ } else if (size == 2) {
+ *val = readw(rockchip->reg_base + busdev);
+ } else if (size == 1) {
+ *val = readb(rockchip->reg_base + busdev);
+ } else {
+ *val = 0;
+ return PCIBIOS_BAD_REGISTER_NUMBER;
+ }
+ return PCIBIOS_SUCCESSFUL;
+}
+
+static int rockchip_pcie_wr_other_conf(struct rockchip_pcie *rockchip,
+ struct pci_bus *bus, u32 devfn,
+ int where, int size, u32 val)
+{
+ u32 busdev;
+
+ busdev = PCIE_ECAM_ADDR(bus->number, PCI_SLOT(devfn),
+ PCI_FUNC(devfn), where);
+ if (!IS_ALIGNED(busdev, size))
+ return PCIBIOS_BAD_REGISTER_NUMBER;
+
+ if (bus->parent->number == rockchip->root_bus_nr)
+ rockchip_pcie_cfg_configuration_accesses(rockchip,
+ AXI_WRAPPER_TYPE0_CFG);
+ else
+ rockchip_pcie_cfg_configuration_accesses(rockchip,
+ AXI_WRAPPER_TYPE1_CFG);
+
+ if (size == 4)
+ writel(val, rockchip->reg_base + busdev);
+ else if (size == 2)
+ writew(val, rockchip->reg_base + busdev);
+ else if (size == 1)
+ writeb(val, rockchip->reg_base + busdev);
+ else
+ return PCIBIOS_BAD_REGISTER_NUMBER;
+
+ return PCIBIOS_SUCCESSFUL;
+}
+
+static int rockchip_pcie_rd_conf(struct pci_bus *bus, u32 devfn, int where,
+ int size, u32 *val)
+{
+ struct rockchip_pcie *rockchip = bus->sysdata;
+
+ if (!rockchip_pcie_valid_device(rockchip, bus, PCI_SLOT(devfn))) {
+ *val = 0xffffffff;
+ return PCIBIOS_DEVICE_NOT_FOUND;
+ }
+
+ if (bus->number == rockchip->root_bus_nr)
+ return rockchip_pcie_rd_own_conf(rockchip, where, size, val);
+
+ return rockchip_pcie_rd_other_conf(rockchip, bus, devfn, where, size, val);
+}
+
+static int rockchip_pcie_wr_conf(struct pci_bus *bus, u32 devfn,
+ int where, int size, u32 val)
+{
+ struct rockchip_pcie *rockchip = bus->sysdata;
+
+ if (!rockchip_pcie_valid_device(rockchip, bus, PCI_SLOT(devfn)))
+ return PCIBIOS_DEVICE_NOT_FOUND;
+
+ if (bus->number == rockchip->root_bus_nr)
+ return rockchip_pcie_wr_own_conf(rockchip, where, size, val);
+
+ return rockchip_pcie_wr_other_conf(rockchip, bus, devfn, where, size, val);
+}
+
+static struct pci_ops rockchip_pcie_ops = {
+ .read = rockchip_pcie_rd_conf,
+ .write = rockchip_pcie_wr_conf,
+};
+
+static void rockchip_pcie_set_power_limit(struct rockchip_pcie *rockchip)
+{
+ int curr;
+ u32 status, scale, power;
+
+ if (IS_ERR(rockchip->vpcie3v3))
+ return;
+
+ /*
+ * Set RC's captured slot power limit and scale if
+ * vpcie3v3 available. The default values are both zero
+ * which means the software should set these two according
+ * to the actual power supply.
+ */
+ curr = regulator_get_current_limit(rockchip->vpcie3v3);
+ if (curr <= 0)
+ return;
+
+ scale = 3; /* 0.001x */
+ curr = curr / 1000; /* convert to mA */
+ power = (curr * 3300) / 1000; /* milliwatt */
+ while (power > PCIE_RC_CONFIG_DCR_CSPL_LIMIT) {
+ if (!scale) {
+ dev_warn(rockchip->dev, "invalid power supply\n");
+ return;
+ }
+ scale--;
+ power = power / 10;
+ }
+
+ status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_DCR);
+ status |= (power << PCIE_RC_CONFIG_DCR_CSPL_SHIFT) |
+ (scale << PCIE_RC_CONFIG_DCR_CPLS_SHIFT);
+ rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_DCR);
+}
+
+/**
+ * rockchip_pcie_init_port - Initialize hardware
+ * @rockchip: PCIe port information
+ */
+static int rockchip_pcie_init_port(struct rockchip_pcie *rockchip)
+{
+ struct device *dev = rockchip->dev;
+ int err, i;
+ u32 status;
+
+ gpiod_set_value_cansleep(rockchip->ep_gpio, 0);
+
+ err = reset_control_assert(rockchip->aclk_rst);
+ if (err) {
+ dev_err(dev, "assert aclk_rst err %d\n", err);
+ return err;
+ }
+
+ err = reset_control_assert(rockchip->pclk_rst);
+ if (err) {
+ dev_err(dev, "assert pclk_rst err %d\n", err);
+ return err;
+ }
+
+ err = reset_control_assert(rockchip->pm_rst);
+ if (err) {
+ dev_err(dev, "assert pm_rst err %d\n", err);
+ return err;
+ }
+
+ for (i = 0; i < MAX_LANE_NUM; i++) {
+ err = phy_init(rockchip->phys[i]);
+ if (err) {
+ dev_err(dev, "init phy%d err %d\n", i, err);
+ goto err_exit_phy;
+ }
+ }
+
+ err = reset_control_assert(rockchip->core_rst);
+ if (err) {
+ dev_err(dev, "assert core_rst err %d\n", err);
+ goto err_exit_phy;
+ }
+
+ err = reset_control_assert(rockchip->mgmt_rst);
+ if (err) {
+ dev_err(dev, "assert mgmt_rst err %d\n", err);
+ goto err_exit_phy;
+ }
+
+ err = reset_control_assert(rockchip->mgmt_sticky_rst);
+ if (err) {
+ dev_err(dev, "assert mgmt_sticky_rst err %d\n", err);
+ goto err_exit_phy;
+ }
+
+ err = reset_control_assert(rockchip->pipe_rst);
+ if (err) {
+ dev_err(dev, "assert pipe_rst err %d\n", err);
+ goto err_exit_phy;
+ }
+
+ udelay(10);
+
+ err = reset_control_deassert(rockchip->pm_rst);
+ if (err) {
+ dev_err(dev, "deassert pm_rst err %d\n", err);
+ goto err_exit_phy;
+ }
+
+ err = reset_control_deassert(rockchip->aclk_rst);
+ if (err) {
+ dev_err(dev, "deassert aclk_rst err %d\n", err);
+ goto err_exit_phy;
+ }
+
+ err = reset_control_deassert(rockchip->pclk_rst);
+ if (err) {
+ dev_err(dev, "deassert pclk_rst err %d\n", err);
+ goto err_exit_phy;
+ }
+
+ if (rockchip->link_gen == 2)
+ rockchip_pcie_write(rockchip, PCIE_CLIENT_GEN_SEL_2,
+ PCIE_CLIENT_CONFIG);
+ else
+ rockchip_pcie_write(rockchip, PCIE_CLIENT_GEN_SEL_1,
+ PCIE_CLIENT_CONFIG);
+
+ rockchip_pcie_write(rockchip,
+ PCIE_CLIENT_CONF_ENABLE |
+ PCIE_CLIENT_LINK_TRAIN_ENABLE |
+ PCIE_CLIENT_ARI_ENABLE |
+ PCIE_CLIENT_CONF_LANE_NUM(rockchip->lanes) |
+ PCIE_CLIENT_MODE_RC,
+ PCIE_CLIENT_CONFIG);
+
+ for (i = 0; i < MAX_LANE_NUM; i++) {
+ err = phy_power_on(rockchip->phys[i]);
+ if (err) {
+ dev_err(dev, "power on phy%d err %d\n", i, err);
+ goto err_power_off_phy;
+ }
+ }
+
+ /*
+ * Please don't reorder the deassert sequence of the following
+ * four reset pins.
+ */
+ err = reset_control_deassert(rockchip->mgmt_sticky_rst);
+ if (err) {
+ dev_err(dev, "deassert mgmt_sticky_rst err %d\n", err);
+ goto err_power_off_phy;
+ }
+
+ err = reset_control_deassert(rockchip->core_rst);
+ if (err) {
+ dev_err(dev, "deassert core_rst err %d\n", err);
+ goto err_power_off_phy;
+ }
+
+ err = reset_control_deassert(rockchip->mgmt_rst);
+ if (err) {
+ dev_err(dev, "deassert mgmt_rst err %d\n", err);
+ goto err_power_off_phy;
+ }
+
+ err = reset_control_deassert(rockchip->pipe_rst);
+ if (err) {
+ dev_err(dev, "deassert pipe_rst err %d\n", err);
+ goto err_power_off_phy;
+ }
+
+ /* Fix the transmitted FTS count desired to exit from L0s. */
+ status = rockchip_pcie_read(rockchip, PCIE_CORE_CTRL_PLC1);
+ status = (status & ~PCIE_CORE_CTRL_PLC1_FTS_MASK) |
+ (PCIE_CORE_CTRL_PLC1_FTS_CNT << PCIE_CORE_CTRL_PLC1_FTS_SHIFT);
+ rockchip_pcie_write(rockchip, status, PCIE_CORE_CTRL_PLC1);
+
+ rockchip_pcie_set_power_limit(rockchip);
+
+ /* Set RC's clock architecture as common clock */
+ status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS);
+ status |= PCI_EXP_LNKSTA_SLC << 16;
+ rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS);
+
+ /* Set RC's RCB to 128 */
+ status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS);
+ status |= PCI_EXP_LNKCTL_RCB;
+ rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS);
+
+ /* Enable Gen1 training */
+ rockchip_pcie_write(rockchip, PCIE_CLIENT_LINK_TRAIN_ENABLE,
+ PCIE_CLIENT_CONFIG);
+
+ gpiod_set_value_cansleep(rockchip->ep_gpio, 1);
+
+ /* 500ms timeout value should be enough for Gen1/2 training */
+ err = readl_poll_timeout(rockchip->apb_base + PCIE_CLIENT_BASIC_STATUS1,
+ status, PCIE_LINK_UP(status), 20,
+ 500 * USEC_PER_MSEC);
+ if (err) {
+ dev_err(dev, "PCIe link training gen1 timeout!\n");
+ goto err_power_off_phy;
+ }
+
+ if (rockchip->link_gen == 2) {
+ /*
+ * Enable retrain for gen2. This should be configured only after
+ * gen1 finished.
+ */
+ status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS);
+ status |= PCI_EXP_LNKCTL_RL;
+ rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS);
+
+ err = readl_poll_timeout(rockchip->apb_base + PCIE_CORE_CTRL,
+ status, PCIE_LINK_IS_GEN2(status), 20,
+ 500 * USEC_PER_MSEC);
+ if (err)
+ dev_dbg(dev, "PCIe link training gen2 timeout, fall back to gen1!\n");
+ }
+
+ /* Check the final link width from negotiated lane counter from MGMT */
+ status = rockchip_pcie_read(rockchip, PCIE_CORE_CTRL);
+ status = 0x1 << ((status & PCIE_CORE_PL_CONF_LANE_MASK) >>
+ PCIE_CORE_PL_CONF_LANE_SHIFT);
+ dev_dbg(dev, "current link width is x%d\n", status);
+
+ /* Power off unused lane(s) */
+ rockchip->lanes_map = rockchip_pcie_lane_map(rockchip);
+ for (i = 0; i < MAX_LANE_NUM; i++) {
+ if (!(rockchip->lanes_map & BIT(i))) {
+ dev_dbg(dev, "idling lane %d\n", i);
+ phy_power_off(rockchip->phys[i]);
+ }
+ }
+
+ rockchip_pcie_write(rockchip, ROCKCHIP_VENDOR_ID,
+ PCIE_CORE_CONFIG_VENDOR);
+ rockchip_pcie_write(rockchip,
+ PCI_CLASS_BRIDGE_PCI << PCIE_RC_CONFIG_SCC_SHIFT,
+ PCIE_RC_CONFIG_RID_CCR);
+
+ /* Clear THP cap's next cap pointer to remove L1 substate cap */
+ status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_THP_CAP);
+ status &= ~PCIE_RC_CONFIG_THP_CAP_NEXT_MASK;
+ rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_THP_CAP);
+
+ /* Clear L0s from RC's link cap */
+ if (of_property_read_bool(dev->of_node, "aspm-no-l0s")) {
+ status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LINK_CAP);
+ status &= ~PCIE_RC_CONFIG_LINK_CAP_L0S;
+ rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LINK_CAP);
+ }
+
+ status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_DCSR);
+ status &= ~PCIE_RC_CONFIG_DCSR_MPS_MASK;
+ status |= PCIE_RC_CONFIG_DCSR_MPS_256;
+ rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_DCSR);
+
+ return 0;
+err_power_off_phy:
+ while (i--)
+ phy_power_off(rockchip->phys[i]);
+ i = MAX_LANE_NUM;
+err_exit_phy:
+ while (i--)
+ phy_exit(rockchip->phys[i]);
+ return err;
+}
+
+static void rockchip_pcie_deinit_phys(struct rockchip_pcie *rockchip)
+{
+ int i;
+
+ for (i = 0; i < MAX_LANE_NUM; i++) {
+ /* inactive lanes are already powered off */
+ if (rockchip->lanes_map & BIT(i))
+ phy_power_off(rockchip->phys[i]);
+ phy_exit(rockchip->phys[i]);
+ }
+}
+
+static irqreturn_t rockchip_pcie_subsys_irq_handler(int irq, void *arg)
+{
+ struct rockchip_pcie *rockchip = arg;
+ struct device *dev = rockchip->dev;
+ u32 reg;
+ u32 sub_reg;
+
+ reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS);
+ if (reg & PCIE_CLIENT_INT_LOCAL) {
+ dev_dbg(dev, "local interrupt received\n");
+ sub_reg = rockchip_pcie_read(rockchip, PCIE_CORE_INT_STATUS);
+ if (sub_reg & PCIE_CORE_INT_PRFPE)
+ dev_dbg(dev, "parity error detected while reading from the PNP receive FIFO RAM\n");
+
+ if (sub_reg & PCIE_CORE_INT_CRFPE)
+ dev_dbg(dev, "parity error detected while reading from the Completion Receive FIFO RAM\n");
+
+ if (sub_reg & PCIE_CORE_INT_RRPE)
+ dev_dbg(dev, "parity error detected while reading from replay buffer RAM\n");
+
+ if (sub_reg & PCIE_CORE_INT_PRFO)
+ dev_dbg(dev, "overflow occurred in the PNP receive FIFO\n");
+
+ if (sub_reg & PCIE_CORE_INT_CRFO)
+ dev_dbg(dev, "overflow occurred in the completion receive FIFO\n");
+
+ if (sub_reg & PCIE_CORE_INT_RT)
+ dev_dbg(dev, "replay timer timed out\n");
+
+ if (sub_reg & PCIE_CORE_INT_RTR)
+ dev_dbg(dev, "replay timer rolled over after 4 transmissions of the same TLP\n");
+
+ if (sub_reg & PCIE_CORE_INT_PE)
+ dev_dbg(dev, "phy error detected on receive side\n");
+
+ if (sub_reg & PCIE_CORE_INT_MTR)
+ dev_dbg(dev, "malformed TLP received from the link\n");
+
+ if (sub_reg & PCIE_CORE_INT_UCR)
+ dev_dbg(dev, "malformed TLP received from the link\n");
+
+ if (sub_reg & PCIE_CORE_INT_FCE)
+ dev_dbg(dev, "an error was observed in the flow control advertisements from the other side\n");
+
+ if (sub_reg & PCIE_CORE_INT_CT)
+ dev_dbg(dev, "a request timed out waiting for completion\n");
+
+ if (sub_reg & PCIE_CORE_INT_UTC)
+ dev_dbg(dev, "unmapped TC error\n");
+
+ if (sub_reg & PCIE_CORE_INT_MMVC)
+ dev_dbg(dev, "MSI mask register changes\n");
+
+ rockchip_pcie_write(rockchip, sub_reg, PCIE_CORE_INT_STATUS);
+ } else if (reg & PCIE_CLIENT_INT_PHY) {
+ dev_dbg(dev, "phy link changes\n");
+ rockchip_pcie_update_txcredit_mui(rockchip);
+ rockchip_pcie_clr_bw_int(rockchip);
+ }
+
+ rockchip_pcie_write(rockchip, reg & PCIE_CLIENT_INT_LOCAL,
+ PCIE_CLIENT_INT_STATUS);
+
+ return IRQ_HANDLED;
+}
+
+static irqreturn_t rockchip_pcie_client_irq_handler(int irq, void *arg)
+{
+ struct rockchip_pcie *rockchip = arg;
+ struct device *dev = rockchip->dev;
+ u32 reg;
+
+ reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS);
+ if (reg & PCIE_CLIENT_INT_LEGACY_DONE)
+ dev_dbg(dev, "legacy done interrupt received\n");
+
+ if (reg & PCIE_CLIENT_INT_MSG)
+ dev_dbg(dev, "message done interrupt received\n");
+
+ if (reg & PCIE_CLIENT_INT_HOT_RST)
+ dev_dbg(dev, "hot reset interrupt received\n");
+
+ if (reg & PCIE_CLIENT_INT_DPA)
+ dev_dbg(dev, "dpa interrupt received\n");
+
+ if (reg & PCIE_CLIENT_INT_FATAL_ERR)
+ dev_dbg(dev, "fatal error interrupt received\n");
+
+ if (reg & PCIE_CLIENT_INT_NFATAL_ERR)
+ dev_dbg(dev, "no fatal error interrupt received\n");
+
+ if (reg & PCIE_CLIENT_INT_CORR_ERR)
+ dev_dbg(dev, "correctable error interrupt received\n");
+
+ if (reg & PCIE_CLIENT_INT_PHY)
+ dev_dbg(dev, "phy interrupt received\n");
+
+ rockchip_pcie_write(rockchip, reg & (PCIE_CLIENT_INT_LEGACY_DONE |
+ PCIE_CLIENT_INT_MSG | PCIE_CLIENT_INT_HOT_RST |
+ PCIE_CLIENT_INT_DPA | PCIE_CLIENT_INT_FATAL_ERR |
+ PCIE_CLIENT_INT_NFATAL_ERR |
+ PCIE_CLIENT_INT_CORR_ERR |
+ PCIE_CLIENT_INT_PHY),
+ PCIE_CLIENT_INT_STATUS);
+
+ return IRQ_HANDLED;
+}
+
+static void rockchip_pcie_legacy_int_handler(struct irq_desc *desc)
+{
+ struct irq_chip *chip = irq_desc_get_chip(desc);
+ struct rockchip_pcie *rockchip = irq_desc_get_handler_data(desc);
+ struct device *dev = rockchip->dev;
+ u32 reg;
+ u32 hwirq;
+ u32 virq;
+
+ chained_irq_enter(chip, desc);
+
+ reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS);
+ reg = (reg & PCIE_CLIENT_INTR_MASK) >> PCIE_CLIENT_INTR_SHIFT;
+
+ while (reg) {
+ hwirq = ffs(reg) - 1;
+ reg &= ~BIT(hwirq);
+
+ virq = irq_find_mapping(rockchip->irq_domain, hwirq);
+ if (virq)
+ generic_handle_irq(virq);
+ else
+ dev_err(dev, "unexpected IRQ, INT%d\n", hwirq);
+ }
+
+ chained_irq_exit(chip, desc);
+}
+
+static int rockchip_pcie_get_phys(struct rockchip_pcie *rockchip)
+{
+ struct device *dev = rockchip->dev;
+ struct phy *phy;
+ char *name;
+ u32 i;
+
+ phy = devm_phy_get(dev, "pcie-phy");
+ if (!IS_ERR(phy)) {
+ rockchip->legacy_phy = true;
+ rockchip->phys[0] = phy;
+ dev_warn(dev, "legacy phy model is deprecated!\n");
+ return 0;
+ }
+
+ if (PTR_ERR(phy) == -EPROBE_DEFER)
+ return PTR_ERR(phy);
+
+ dev_dbg(dev, "missing legacy phy; search for per-lane PHY\n");
+
+ for (i = 0; i < MAX_LANE_NUM; i++) {
+ name = kasprintf(GFP_KERNEL, "pcie-phy-%u", i);
+ if (!name)
+ return -ENOMEM;
+
+ phy = devm_of_phy_get(dev, dev->of_node, name);
+ kfree(name);
+
+ if (IS_ERR(phy)) {
+ if (PTR_ERR(phy) != -EPROBE_DEFER)
+ dev_err(dev, "missing phy for lane %d: %ld\n",
+ i, PTR_ERR(phy));
+ return PTR_ERR(phy);
+ }
+
+ rockchip->phys[i] = phy;
+ }
+
+ return 0;
+}
+
+static int rockchip_pcie_setup_irq(struct rockchip_pcie *rockchip)
+{
+ int irq, err;
+ struct device *dev = rockchip->dev;
+ struct platform_device *pdev = to_platform_device(dev);
+
+ irq = platform_get_irq_byname(pdev, "sys");
+ if (irq < 0) {
+ dev_err(dev, "missing sys IRQ resource\n");
+ return irq;
+ }
+
+ err = devm_request_irq(dev, irq, rockchip_pcie_subsys_irq_handler,
+ IRQF_SHARED, "pcie-sys", rockchip);
+ if (err) {
+ dev_err(dev, "failed to request PCIe subsystem IRQ\n");
+ return err;
+ }
+
+ irq = platform_get_irq_byname(pdev, "legacy");
+ if (irq < 0) {
+ dev_err(dev, "missing legacy IRQ resource\n");
+ return irq;
+ }
+
+ irq_set_chained_handler_and_data(irq,
+ rockchip_pcie_legacy_int_handler,
+ rockchip);
+
+ irq = platform_get_irq_byname(pdev, "client");
+ if (irq < 0) {
+ dev_err(dev, "missing client IRQ resource\n");
+ return irq;
+ }
+
+ err = devm_request_irq(dev, irq, rockchip_pcie_client_irq_handler,
+ IRQF_SHARED, "pcie-client", rockchip);
+ if (err) {
+ dev_err(dev, "failed to request PCIe client IRQ\n");
+ return err;
+ }
+
+ return 0;
+}
+
+/**
+ * rockchip_pcie_parse_dt - Parse Device Tree
+ * @rockchip: PCIe port information
+ *
+ * Return: '0' on success and error value on failure
+ */
+static int rockchip_pcie_parse_dt(struct rockchip_pcie *rockchip)
+{
+ struct device *dev = rockchip->dev;
+ struct platform_device *pdev = to_platform_device(dev);
+ struct device_node *node = dev->of_node;
+ struct resource *regs;
+ int err;
+
+ regs = platform_get_resource_byname(pdev,
+ IORESOURCE_MEM,
+ "axi-base");
+ rockchip->reg_base = devm_pci_remap_cfg_resource(dev, regs);
+ if (IS_ERR(rockchip->reg_base))
+ return PTR_ERR(rockchip->reg_base);
+
+ regs = platform_get_resource_byname(pdev,
+ IORESOURCE_MEM,
+ "apb-base");
+ rockchip->apb_base = devm_ioremap_resource(dev, regs);
+ if (IS_ERR(rockchip->apb_base))
+ return PTR_ERR(rockchip->apb_base);
+
+ err = rockchip_pcie_get_phys(rockchip);
+ if (err)
+ return err;
+
+ rockchip->lanes = 1;
+ err = of_property_read_u32(node, "num-lanes", &rockchip->lanes);
+ if (!err && (rockchip->lanes == 0 ||
+ rockchip->lanes == 3 ||
+ rockchip->lanes > 4)) {
+ dev_warn(dev, "invalid num-lanes, default to use one lane\n");
+ rockchip->lanes = 1;
+ }
+
+ rockchip->link_gen = of_pci_get_max_link_speed(node);
+ if (rockchip->link_gen < 0 || rockchip->link_gen > 2)
+ rockchip->link_gen = 2;
+
+ rockchip->core_rst = devm_reset_control_get_exclusive(dev, "core");
+ if (IS_ERR(rockchip->core_rst)) {
+ if (PTR_ERR(rockchip->core_rst) != -EPROBE_DEFER)
+ dev_err(dev, "missing core reset property in node\n");
+ return PTR_ERR(rockchip->core_rst);
+ }
+
+ rockchip->mgmt_rst = devm_reset_control_get_exclusive(dev, "mgmt");
+ if (IS_ERR(rockchip->mgmt_rst)) {
+ if (PTR_ERR(rockchip->mgmt_rst) != -EPROBE_DEFER)
+ dev_err(dev, "missing mgmt reset property in node\n");
+ return PTR_ERR(rockchip->mgmt_rst);
+ }
+
+ rockchip->mgmt_sticky_rst = devm_reset_control_get_exclusive(dev,
+ "mgmt-sticky");
+ if (IS_ERR(rockchip->mgmt_sticky_rst)) {
+ if (PTR_ERR(rockchip->mgmt_sticky_rst) != -EPROBE_DEFER)
+ dev_err(dev, "missing mgmt-sticky reset property in node\n");
+ return PTR_ERR(rockchip->mgmt_sticky_rst);
+ }
+
+ rockchip->pipe_rst = devm_reset_control_get_exclusive(dev, "pipe");
+ if (IS_ERR(rockchip->pipe_rst)) {
+ if (PTR_ERR(rockchip->pipe_rst) != -EPROBE_DEFER)
+ dev_err(dev, "missing pipe reset property in node\n");
+ return PTR_ERR(rockchip->pipe_rst);
+ }
+
+ rockchip->pm_rst = devm_reset_control_get_exclusive(dev, "pm");
+ if (IS_ERR(rockchip->pm_rst)) {
+ if (PTR_ERR(rockchip->pm_rst) != -EPROBE_DEFER)
+ dev_err(dev, "missing pm reset property in node\n");
+ return PTR_ERR(rockchip->pm_rst);
+ }
+
+ rockchip->pclk_rst = devm_reset_control_get_exclusive(dev, "pclk");
+ if (IS_ERR(rockchip->pclk_rst)) {
+ if (PTR_ERR(rockchip->pclk_rst) != -EPROBE_DEFER)
+ dev_err(dev, "missing pclk reset property in node\n");
+ return PTR_ERR(rockchip->pclk_rst);
+ }
+
+ rockchip->aclk_rst = devm_reset_control_get_exclusive(dev, "aclk");
+ if (IS_ERR(rockchip->aclk_rst)) {
+ if (PTR_ERR(rockchip->aclk_rst) != -EPROBE_DEFER)
+ dev_err(dev, "missing aclk reset property in node\n");
+ return PTR_ERR(rockchip->aclk_rst);
+ }
+
+ rockchip->ep_gpio = devm_gpiod_get(dev, "ep", GPIOD_OUT_HIGH);
+ if (IS_ERR(rockchip->ep_gpio)) {
+ dev_err(dev, "missing ep-gpios property in node\n");
+ return PTR_ERR(rockchip->ep_gpio);
+ }
+
+ rockchip->aclk_pcie = devm_clk_get(dev, "aclk");
+ if (IS_ERR(rockchip->aclk_pcie)) {
+ dev_err(dev, "aclk clock not found\n");
+ return PTR_ERR(rockchip->aclk_pcie);
+ }
+
+ rockchip->aclk_perf_pcie = devm_clk_get(dev, "aclk-perf");
+ if (IS_ERR(rockchip->aclk_perf_pcie)) {
+ dev_err(dev, "aclk_perf clock not found\n");
+ return PTR_ERR(rockchip->aclk_perf_pcie);
+ }
+
+ rockchip->hclk_pcie = devm_clk_get(dev, "hclk");
+ if (IS_ERR(rockchip->hclk_pcie)) {
+ dev_err(dev, "hclk clock not found\n");
+ return PTR_ERR(rockchip->hclk_pcie);
+ }
+
+ rockchip->clk_pcie_pm = devm_clk_get(dev, "pm");
+ if (IS_ERR(rockchip->clk_pcie_pm)) {
+ dev_err(dev, "pm clock not found\n");
+ return PTR_ERR(rockchip->clk_pcie_pm);
+ }
+
+ err = rockchip_pcie_setup_irq(rockchip);
+ if (err)
+ return err;
+
+ rockchip->vpcie12v = devm_regulator_get_optional(dev, "vpcie12v");
+ if (IS_ERR(rockchip->vpcie12v)) {
+ if (PTR_ERR(rockchip->vpcie12v) != -ENODEV)
+ return PTR_ERR(rockchip->vpcie12v);
+ dev_info(dev, "no vpcie12v regulator found\n");
+ }
+
+ rockchip->vpcie3v3 = devm_regulator_get_optional(dev, "vpcie3v3");
+ if (IS_ERR(rockchip->vpcie3v3)) {
+ if (PTR_ERR(rockchip->vpcie3v3) != -ENODEV)
+ return PTR_ERR(rockchip->vpcie3v3);
+ dev_info(dev, "no vpcie3v3 regulator found\n");
+ }
+
+ rockchip->vpcie1v8 = devm_regulator_get_optional(dev, "vpcie1v8");
+ if (IS_ERR(rockchip->vpcie1v8)) {
+ if (PTR_ERR(rockchip->vpcie1v8) != -ENODEV)
+ return PTR_ERR(rockchip->vpcie1v8);
+ dev_info(dev, "no vpcie1v8 regulator found\n");
+ }
+
+ rockchip->vpcie0v9 = devm_regulator_get_optional(dev, "vpcie0v9");
+ if (IS_ERR(rockchip->vpcie0v9)) {
+ if (PTR_ERR(rockchip->vpcie0v9) != -ENODEV)
+ return PTR_ERR(rockchip->vpcie0v9);
+ dev_info(dev, "no vpcie0v9 regulator found\n");
+ }
+
+ return 0;
+}
+
+static int rockchip_pcie_set_vpcie(struct rockchip_pcie *rockchip)
+{
+ struct device *dev = rockchip->dev;
+ int err;
+
+ if (!IS_ERR(rockchip->vpcie12v)) {
+ err = regulator_enable(rockchip->vpcie12v);
+ if (err) {
+ dev_err(dev, "fail to enable vpcie12v regulator\n");
+ goto err_out;
+ }
+ }
+
+ if (!IS_ERR(rockchip->vpcie3v3)) {
+ err = regulator_enable(rockchip->vpcie3v3);
+ if (err) {
+ dev_err(dev, "fail to enable vpcie3v3 regulator\n");
+ goto err_disable_12v;
+ }
+ }
+
+ if (!IS_ERR(rockchip->vpcie1v8)) {
+ err = regulator_enable(rockchip->vpcie1v8);
+ if (err) {
+ dev_err(dev, "fail to enable vpcie1v8 regulator\n");
+ goto err_disable_3v3;
+ }
+ }
+
+ if (!IS_ERR(rockchip->vpcie0v9)) {
+ err = regulator_enable(rockchip->vpcie0v9);
+ if (err) {
+ dev_err(dev, "fail to enable vpcie0v9 regulator\n");
+ goto err_disable_1v8;
+ }
+ }
+
+ return 0;
+
+err_disable_1v8:
+ if (!IS_ERR(rockchip->vpcie1v8))
+ regulator_disable(rockchip->vpcie1v8);
+err_disable_3v3:
+ if (!IS_ERR(rockchip->vpcie3v3))
+ regulator_disable(rockchip->vpcie3v3);
+err_disable_12v:
+ if (!IS_ERR(rockchip->vpcie12v))
+ regulator_disable(rockchip->vpcie12v);
+err_out:
+ return err;
+}
+
+static void rockchip_pcie_enable_interrupts(struct rockchip_pcie *rockchip)
+{
+ rockchip_pcie_write(rockchip, (PCIE_CLIENT_INT_CLI << 16) &
+ (~PCIE_CLIENT_INT_CLI), PCIE_CLIENT_INT_MASK);
+ rockchip_pcie_write(rockchip, (u32)(~PCIE_CORE_INT),
+ PCIE_CORE_INT_MASK);
+
+ rockchip_pcie_enable_bw_int(rockchip);
+}
+
+static int rockchip_pcie_intx_map(struct irq_domain *domain, unsigned int irq,
+ irq_hw_number_t hwirq)
+{
+ irq_set_chip_and_handler(irq, &dummy_irq_chip, handle_simple_irq);
+ irq_set_chip_data(irq, domain->host_data);
+
+ return 0;
+}
+
+static const struct irq_domain_ops intx_domain_ops = {
+ .map = rockchip_pcie_intx_map,
+};
+
+static int rockchip_pcie_init_irq_domain(struct rockchip_pcie *rockchip)
+{
+ struct device *dev = rockchip->dev;
+ struct device_node *intc = of_get_next_child(dev->of_node, NULL);
+
+ if (!intc) {
+ dev_err(dev, "missing child interrupt-controller node\n");
+ return -EINVAL;
+ }
+
+ rockchip->irq_domain = irq_domain_add_linear(intc, PCI_NUM_INTX,
+ &intx_domain_ops, rockchip);
+ if (!rockchip->irq_domain) {
+ dev_err(dev, "failed to get a INTx IRQ domain\n");
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static int rockchip_pcie_prog_ob_atu(struct rockchip_pcie *rockchip,
+ int region_no, int type, u8 num_pass_bits,
+ u32 lower_addr, u32 upper_addr)
+{
+ u32 ob_addr_0;
+ u32 ob_addr_1;
+ u32 ob_desc_0;
+ u32 aw_offset;
+
+ if (region_no >= MAX_AXI_WRAPPER_REGION_NUM)
+ return -EINVAL;
+ if (num_pass_bits + 1 < 8)
+ return -EINVAL;
+ if (num_pass_bits > 63)
+ return -EINVAL;
+ if (region_no == 0) {
+ if (AXI_REGION_0_SIZE < (2ULL << num_pass_bits))
+ return -EINVAL;
+ }
+ if (region_no != 0) {
+ if (AXI_REGION_SIZE < (2ULL << num_pass_bits))
+ return -EINVAL;
+ }
+
+ aw_offset = (region_no << OB_REG_SIZE_SHIFT);
+
+ ob_addr_0 = num_pass_bits & PCIE_CORE_OB_REGION_ADDR0_NUM_BITS;
+ ob_addr_0 |= lower_addr & PCIE_CORE_OB_REGION_ADDR0_LO_ADDR;
+ ob_addr_1 = upper_addr;
+ ob_desc_0 = (1 << 23 | type);
+
+ rockchip_pcie_write(rockchip, ob_addr_0,
+ PCIE_CORE_OB_REGION_ADDR0 + aw_offset);
+ rockchip_pcie_write(rockchip, ob_addr_1,
+ PCIE_CORE_OB_REGION_ADDR1 + aw_offset);
+ rockchip_pcie_write(rockchip, ob_desc_0,
+ PCIE_CORE_OB_REGION_DESC0 + aw_offset);
+ rockchip_pcie_write(rockchip, 0,
+ PCIE_CORE_OB_REGION_DESC1 + aw_offset);
+
+ return 0;
+}
+
+static int rockchip_pcie_prog_ib_atu(struct rockchip_pcie *rockchip,
+ int region_no, u8 num_pass_bits,
+ u32 lower_addr, u32 upper_addr)
+{
+ u32 ib_addr_0;
+ u32 ib_addr_1;
+ u32 aw_offset;
+
+ if (region_no > MAX_AXI_IB_ROOTPORT_REGION_NUM)
+ return -EINVAL;
+ if (num_pass_bits + 1 < MIN_AXI_ADDR_BITS_PASSED)
+ return -EINVAL;
+ if (num_pass_bits > 63)
+ return -EINVAL;
+
+ aw_offset = (region_no << IB_ROOT_PORT_REG_SIZE_SHIFT);
+
+ ib_addr_0 = num_pass_bits & PCIE_CORE_IB_REGION_ADDR0_NUM_BITS;
+ ib_addr_0 |= (lower_addr << 8) & PCIE_CORE_IB_REGION_ADDR0_LO_ADDR;
+ ib_addr_1 = upper_addr;
+
+ rockchip_pcie_write(rockchip, ib_addr_0, PCIE_RP_IB_ADDR0 + aw_offset);
+ rockchip_pcie_write(rockchip, ib_addr_1, PCIE_RP_IB_ADDR1 + aw_offset);
+
+ return 0;
+}
+
+static int rockchip_pcie_cfg_atu(struct rockchip_pcie *rockchip)
+{
+ struct device *dev = rockchip->dev;
+ int offset;
+ int err;
+ int reg_no;
+
+ rockchip_pcie_cfg_configuration_accesses(rockchip,
+ AXI_WRAPPER_TYPE0_CFG);
+
+ for (reg_no = 0; reg_no < (rockchip->mem_size >> 20); reg_no++) {
+ err = rockchip_pcie_prog_ob_atu(rockchip, reg_no + 1,
+ AXI_WRAPPER_MEM_WRITE,
+ 20 - 1,
+ rockchip->mem_bus_addr +
+ (reg_no << 20),
+ 0);
+ if (err) {
+ dev_err(dev, "program RC mem outbound ATU failed\n");
+ return err;
+ }
+ }
+
+ err = rockchip_pcie_prog_ib_atu(rockchip, 2, 32 - 1, 0x0, 0);
+ if (err) {
+ dev_err(dev, "program RC mem inbound ATU failed\n");
+ return err;
+ }
+
+ offset = rockchip->mem_size >> 20;
+ for (reg_no = 0; reg_no < (rockchip->io_size >> 20); reg_no++) {
+ err = rockchip_pcie_prog_ob_atu(rockchip,
+ reg_no + 1 + offset,
+ AXI_WRAPPER_IO_WRITE,
+ 20 - 1,
+ rockchip->io_bus_addr +
+ (reg_no << 20),
+ 0);
+ if (err) {
+ dev_err(dev, "program RC io outbound ATU failed\n");
+ return err;
+ }
+ }
+
+ /* assign message regions */
+ rockchip_pcie_prog_ob_atu(rockchip, reg_no + 1 + offset,
+ AXI_WRAPPER_NOR_MSG,
+ 20 - 1, 0, 0);
+
+ rockchip->msg_bus_addr = rockchip->mem_bus_addr +
+ ((reg_no + offset) << 20);
+ return err;
+}
+
+static int rockchip_pcie_wait_l2(struct rockchip_pcie *rockchip)
+{
+ u32 value;
+ int err;
+
+ /* send PME_TURN_OFF message */
+ writel(0x0, rockchip->msg_region + PCIE_RC_SEND_PME_OFF);
+
+ /* read LTSSM and wait for falling into L2 link state */
+ err = readl_poll_timeout(rockchip->apb_base + PCIE_CLIENT_DEBUG_OUT_0,
+ value, PCIE_LINK_IS_L2(value), 20,
+ jiffies_to_usecs(5 * HZ));
+ if (err) {
+ dev_err(rockchip->dev, "PCIe link enter L2 timeout!\n");
+ return err;
+ }
+
+ return 0;
+}
+
+static int rockchip_pcie_enable_clocks(struct rockchip_pcie *rockchip)
+{
+ struct device *dev = rockchip->dev;
+ int err;
+
+ err = clk_prepare_enable(rockchip->aclk_pcie);
+ if (err) {
+ dev_err(dev, "unable to enable aclk_pcie clock\n");
+ return err;
+ }
+
+ err = clk_prepare_enable(rockchip->aclk_perf_pcie);
+ if (err) {
+ dev_err(dev, "unable to enable aclk_perf_pcie clock\n");
+ goto err_aclk_perf_pcie;
+ }
+
+ err = clk_prepare_enable(rockchip->hclk_pcie);
+ if (err) {
+ dev_err(dev, "unable to enable hclk_pcie clock\n");
+ goto err_hclk_pcie;
+ }
+
+ err = clk_prepare_enable(rockchip->clk_pcie_pm);
+ if (err) {
+ dev_err(dev, "unable to enable clk_pcie_pm clock\n");
+ goto err_clk_pcie_pm;
+ }
+
+ return 0;
+
+err_clk_pcie_pm:
+ clk_disable_unprepare(rockchip->hclk_pcie);
+err_hclk_pcie:
+ clk_disable_unprepare(rockchip->aclk_perf_pcie);
+err_aclk_perf_pcie:
+ clk_disable_unprepare(rockchip->aclk_pcie);
+ return err;
+}
+
+static void rockchip_pcie_disable_clocks(void *data)
+{
+ struct rockchip_pcie *rockchip = data;
+
+ clk_disable_unprepare(rockchip->clk_pcie_pm);
+ clk_disable_unprepare(rockchip->hclk_pcie);
+ clk_disable_unprepare(rockchip->aclk_perf_pcie);
+ clk_disable_unprepare(rockchip->aclk_pcie);
+}
+
+static int __maybe_unused rockchip_pcie_suspend_noirq(struct device *dev)
+{
+ struct rockchip_pcie *rockchip = dev_get_drvdata(dev);
+ int ret;
+
+ /* disable core and cli int since we don't need to ack PME_ACK */
+ rockchip_pcie_write(rockchip, (PCIE_CLIENT_INT_CLI << 16) |
+ PCIE_CLIENT_INT_CLI, PCIE_CLIENT_INT_MASK);
+ rockchip_pcie_write(rockchip, (u32)PCIE_CORE_INT, PCIE_CORE_INT_MASK);
+
+ ret = rockchip_pcie_wait_l2(rockchip);
+ if (ret) {
+ rockchip_pcie_enable_interrupts(rockchip);
+ return ret;
+ }
+
+ rockchip_pcie_deinit_phys(rockchip);
+
+ rockchip_pcie_disable_clocks(rockchip);
+
+ if (!IS_ERR(rockchip->vpcie0v9))
+ regulator_disable(rockchip->vpcie0v9);
+
+ return ret;
+}
+
+static int __maybe_unused rockchip_pcie_resume_noirq(struct device *dev)
+{
+ struct rockchip_pcie *rockchip = dev_get_drvdata(dev);
+ int err;
+
+ if (!IS_ERR(rockchip->vpcie0v9)) {
+ err = regulator_enable(rockchip->vpcie0v9);
+ if (err) {
+ dev_err(dev, "fail to enable vpcie0v9 regulator\n");
+ return err;
+ }
+ }
+
+ err = rockchip_pcie_enable_clocks(rockchip);
+ if (err)
+ goto err_disable_0v9;
+
+ err = rockchip_pcie_init_port(rockchip);
+ if (err)
+ goto err_pcie_resume;
+
+ err = rockchip_pcie_cfg_atu(rockchip);
+ if (err)
+ goto err_err_deinit_port;
+
+ /* Need this to enter L1 again */
+ rockchip_pcie_update_txcredit_mui(rockchip);
+ rockchip_pcie_enable_interrupts(rockchip);
+
+ return 0;
+
+err_err_deinit_port:
+ rockchip_pcie_deinit_phys(rockchip);
+err_pcie_resume:
+ rockchip_pcie_disable_clocks(rockchip);
+err_disable_0v9:
+ if (!IS_ERR(rockchip->vpcie0v9))
+ regulator_disable(rockchip->vpcie0v9);
+ return err;
+}
+
+static int rockchip_pcie_probe(struct platform_device *pdev)
+{
+ struct rockchip_pcie *rockchip;
+ struct device *dev = &pdev->dev;
+ struct pci_bus *bus, *child;
+ struct pci_host_bridge *bridge;
+ struct resource_entry *win;
+ resource_size_t io_base;
+ struct resource *mem;
+ struct resource *io;
+ int err;
+
+ LIST_HEAD(res);
+
+ if (!dev->of_node)
+ return -ENODEV;
+
+ bridge = devm_pci_alloc_host_bridge(dev, sizeof(*rockchip));
+ if (!bridge)
+ return -ENOMEM;
+
+ rockchip = pci_host_bridge_priv(bridge);
+
+ platform_set_drvdata(pdev, rockchip);
+ rockchip->dev = dev;
+
+ err = rockchip_pcie_parse_dt(rockchip);
+ if (err)
+ return err;
+
+ err = rockchip_pcie_enable_clocks(rockchip);
+ if (err)
+ return err;
+
+ err = rockchip_pcie_set_vpcie(rockchip);
+ if (err) {
+ dev_err(dev, "failed to set vpcie regulator\n");
+ goto err_set_vpcie;
+ }
+
+ err = rockchip_pcie_init_port(rockchip);
+ if (err)
+ goto err_vpcie;
+
+ rockchip_pcie_enable_interrupts(rockchip);
+
+ err = rockchip_pcie_init_irq_domain(rockchip);
+ if (err < 0)
+ goto err_deinit_port;
+
+ err = of_pci_get_host_bridge_resources(dev->of_node, 0, 0xff,
+ &res, &io_base);
+ if (err)
+ goto err_remove_irq_domain;
+
+ err = devm_request_pci_bus_resources(dev, &res);
+ if (err)
+ goto err_free_res;
+
+ /* Get the I/O and memory ranges from DT */
+ resource_list_for_each_entry(win, &res) {
+ switch (resource_type(win->res)) {
+ case IORESOURCE_IO:
+ io = win->res;
+ io->name = "I/O";
+ rockchip->io_size = resource_size(io);
+ rockchip->io_bus_addr = io->start - win->offset;
+ err = pci_remap_iospace(io, io_base);
+ if (err) {
+ dev_warn(dev, "error %d: failed to map resource %pR\n",
+ err, io);
+ continue;
+ }
+ rockchip->io = io;
+ break;
+ case IORESOURCE_MEM:
+ mem = win->res;
+ mem->name = "MEM";
+ rockchip->mem_size = resource_size(mem);
+ rockchip->mem_bus_addr = mem->start - win->offset;
+ break;
+ case IORESOURCE_BUS:
+ rockchip->root_bus_nr = win->res->start;
+ break;
+ default:
+ continue;
+ }
+ }
+
+ err = rockchip_pcie_cfg_atu(rockchip);
+ if (err)
+ goto err_unmap_iospace;
+
+ rockchip->msg_region = devm_ioremap(dev, rockchip->msg_bus_addr, SZ_1M);
+ if (!rockchip->msg_region) {
+ err = -ENOMEM;
+ goto err_unmap_iospace;
+ }
+
+ list_splice_init(&res, &bridge->windows);
+ bridge->dev.parent = dev;
+ bridge->sysdata = rockchip;
+ bridge->busnr = 0;
+ bridge->ops = &rockchip_pcie_ops;
+ bridge->map_irq = of_irq_parse_and_map_pci;
+ bridge->swizzle_irq = pci_common_swizzle;
+
+ err = pci_scan_root_bus_bridge(bridge);
+ if (err < 0)
+ goto err_unmap_iospace;
+
+ bus = bridge->bus;
+
+ rockchip->root_bus = bus;
+
+ pci_bus_size_bridges(bus);
+ pci_bus_assign_resources(bus);
+ list_for_each_entry(child, &bus->children, node)
+ pcie_bus_configure_settings(child);
+
+ pci_bus_add_devices(bus);
+ return 0;
+
+err_unmap_iospace:
+ pci_unmap_iospace(rockchip->io);
+err_free_res:
+ pci_free_resource_list(&res);
+err_remove_irq_domain:
+ irq_domain_remove(rockchip->irq_domain);
+err_deinit_port:
+ rockchip_pcie_deinit_phys(rockchip);
+err_vpcie:
+ if (!IS_ERR(rockchip->vpcie12v))
+ regulator_disable(rockchip->vpcie12v);
+ if (!IS_ERR(rockchip->vpcie3v3))
+ regulator_disable(rockchip->vpcie3v3);
+ if (!IS_ERR(rockchip->vpcie1v8))
+ regulator_disable(rockchip->vpcie1v8);
+ if (!IS_ERR(rockchip->vpcie0v9))
+ regulator_disable(rockchip->vpcie0v9);
+err_set_vpcie:
+ rockchip_pcie_disable_clocks(rockchip);
+ return err;
+}
+
+static int rockchip_pcie_remove(struct platform_device *pdev)
+{
+ struct device *dev = &pdev->dev;
+ struct rockchip_pcie *rockchip = dev_get_drvdata(dev);
+
+ pci_stop_root_bus(rockchip->root_bus);
+ pci_remove_root_bus(rockchip->root_bus);
+ pci_unmap_iospace(rockchip->io);
+ irq_domain_remove(rockchip->irq_domain);
+
+ rockchip_pcie_deinit_phys(rockchip);
+
+ rockchip_pcie_disable_clocks(rockchip);
+
+ if (!IS_ERR(rockchip->vpcie12v))
+ regulator_disable(rockchip->vpcie12v);
+ if (!IS_ERR(rockchip->vpcie3v3))
+ regulator_disable(rockchip->vpcie3v3);
+ if (!IS_ERR(rockchip->vpcie1v8))
+ regulator_disable(rockchip->vpcie1v8);
+ if (!IS_ERR(rockchip->vpcie0v9))
+ regulator_disable(rockchip->vpcie0v9);
+
+ return 0;
+}
+
+static const struct dev_pm_ops rockchip_pcie_pm_ops = {
+ SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(rockchip_pcie_suspend_noirq,
+ rockchip_pcie_resume_noirq)
+};
+
+static const struct of_device_id rockchip_pcie_of_match[] = {
+ { .compatible = "rockchip,rk3399-pcie", },
+ {}
+};
+MODULE_DEVICE_TABLE(of, rockchip_pcie_of_match);
+
+static struct platform_driver rockchip_pcie_driver = {
+ .driver = {
+ .name = "rockchip-pcie",
+ .of_match_table = rockchip_pcie_of_match,
+ .pm = &rockchip_pcie_pm_ops,
+ },
+ .probe = rockchip_pcie_probe,
+ .remove = rockchip_pcie_remove,
+};
+module_platform_driver(rockchip_pcie_driver);
+
+MODULE_AUTHOR("Rockchip Inc");
+MODULE_DESCRIPTION("Rockchip AXI PCIe driver");
+MODULE_LICENSE("GPL v2");
diff --git a/src/kernel/linux/v4.14/drivers/pci/host/pcie-tango.c b/src/kernel/linux/v4.14/drivers/pci/host/pcie-tango.c
new file mode 100644
index 0000000..95049de
--- /dev/null
+++ b/src/kernel/linux/v4.14/drivers/pci/host/pcie-tango.c
@@ -0,0 +1,142 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <linux/pci-ecam.h>
+#include <linux/delay.h>
+#include <linux/of.h>
+
+#define SMP8759_MUX 0x48
+#define SMP8759_TEST_OUT 0x74
+
+struct tango_pcie {
+ void __iomem *base;
+};
+
+static int smp8759_config_read(struct pci_bus *bus, unsigned int devfn,
+ int where, int size, u32 *val)
+{
+ struct pci_config_window *cfg = bus->sysdata;
+ struct tango_pcie *pcie = dev_get_drvdata(cfg->parent);
+ int ret;
+
+ /* Reads in configuration space outside devfn 0 return garbage */
+ if (devfn != 0)
+ return PCIBIOS_FUNC_NOT_SUPPORTED;
+
+ /*
+ * PCI config and MMIO accesses are muxed. Linux doesn't have a
+ * mutual exclusion mechanism for config vs. MMIO accesses, so
+ * concurrent accesses may cause corruption.
+ */
+ writel_relaxed(1, pcie->base + SMP8759_MUX);
+ ret = pci_generic_config_read(bus, devfn, where, size, val);
+ writel_relaxed(0, pcie->base + SMP8759_MUX);
+
+ return ret;
+}
+
+static int smp8759_config_write(struct pci_bus *bus, unsigned int devfn,
+ int where, int size, u32 val)
+{
+ struct pci_config_window *cfg = bus->sysdata;
+ struct tango_pcie *pcie = dev_get_drvdata(cfg->parent);
+ int ret;
+
+ writel_relaxed(1, pcie->base + SMP8759_MUX);
+ ret = pci_generic_config_write(bus, devfn, where, size, val);
+ writel_relaxed(0, pcie->base + SMP8759_MUX);
+
+ return ret;
+}
+
+static struct pci_ecam_ops smp8759_ecam_ops = {
+ .bus_shift = 20,
+ .pci_ops = {
+ .map_bus = pci_ecam_map_bus,
+ .read = smp8759_config_read,
+ .write = smp8759_config_write,
+ }
+};
+
+static int tango_pcie_link_up(struct tango_pcie *pcie)
+{
+ void __iomem *test_out = pcie->base + SMP8759_TEST_OUT;
+ int i;
+
+ writel_relaxed(16, test_out);
+ for (i = 0; i < 10; ++i) {
+ u32 ltssm_state = readl_relaxed(test_out) >> 8;
+ if ((ltssm_state & 0x1f) == 0xf) /* L0 */
+ return 1;
+ usleep_range(3000, 4000);
+ }
+
+ return 0;
+}
+
+static int tango_pcie_probe(struct platform_device *pdev)
+{
+ struct device *dev = &pdev->dev;
+ struct tango_pcie *pcie;
+ struct resource *res;
+ int ret;
+
+ dev_warn(dev, "simultaneous PCI config and MMIO accesses may cause data corruption\n");
+ add_taint(TAINT_CRAP, LOCKDEP_STILL_OK);
+
+ pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL);
+ if (!pcie)
+ return -ENOMEM;
+
+ res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
+ pcie->base = devm_ioremap_resource(dev, res);
+ if (IS_ERR(pcie->base))
+ return PTR_ERR(pcie->base);
+
+ platform_set_drvdata(pdev, pcie);
+
+ if (!tango_pcie_link_up(pcie))
+ return -ENODEV;
+
+ return pci_host_common_probe(pdev, &smp8759_ecam_ops);
+}
+
+static const struct of_device_id tango_pcie_ids[] = {
+ { .compatible = "sigma,smp8759-pcie" },
+ { },
+};
+
+static struct platform_driver tango_pcie_driver = {
+ .probe = tango_pcie_probe,
+ .driver = {
+ .name = KBUILD_MODNAME,
+ .of_match_table = tango_pcie_ids,
+ .suppress_bind_attrs = true,
+ },
+};
+builtin_platform_driver(tango_pcie_driver);
+
+/*
+ * The root complex advertises the wrong device class.
+ * Header Type 1 is for PCI-to-PCI bridges.
+ */
+static void tango_fixup_class(struct pci_dev *dev)
+{
+ dev->class = PCI_CLASS_BRIDGE_PCI << 8;
+}
+DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_SIGMA, 0x0024, tango_fixup_class);
+DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_SIGMA, 0x0028, tango_fixup_class);
+
+/*
+ * The root complex exposes a "fake" BAR, which is used to filter
+ * bus-to-system accesses. Only accesses within the range defined by this
+ * BAR are forwarded to the host, others are ignored.
+ *
+ * By default, the DMA framework expects an identity mapping, and DRAM0 is
+ * mapped at 0x80000000.
+ */
+static void tango_fixup_bar(struct pci_dev *dev)
+{
+ dev->non_compliant_bars = true;
+ pci_write_config_dword(dev, PCI_BASE_ADDRESS_0, 0x80000000);
+}
+DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_SIGMA, 0x0024, tango_fixup_bar);
+DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_SIGMA, 0x0028, tango_fixup_bar);
diff --git a/src/kernel/linux/v4.14/drivers/pci/host/pcie-xilinx-nwl.c b/src/kernel/linux/v4.14/drivers/pci/host/pcie-xilinx-nwl.c
new file mode 100644
index 0000000..981a519
--- /dev/null
+++ b/src/kernel/linux/v4.14/drivers/pci/host/pcie-xilinx-nwl.c
@@ -0,0 +1,918 @@
+/*
+ * PCIe host controller driver for NWL PCIe Bridge
+ * Based on pcie-xilinx.c, pci-tegra.c
+ *
+ * (C) Copyright 2014 - 2015, Xilinx, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include <linux/delay.h>
+#include <linux/interrupt.h>
+#include <linux/irq.h>
+#include <linux/irqdomain.h>
+#include <linux/kernel.h>
+#include <linux/init.h>
+#include <linux/msi.h>
+#include <linux/of_address.h>
+#include <linux/of_pci.h>
+#include <linux/of_platform.h>
+#include <linux/of_irq.h>
+#include <linux/pci.h>
+#include <linux/platform_device.h>
+#include <linux/irqchip/chained_irq.h>
+
+/* Bridge core config registers */
+#define BRCFG_PCIE_RX0 0x00000000
+#define BRCFG_INTERRUPT 0x00000010
+#define BRCFG_PCIE_RX_MSG_FILTER 0x00000020
+
+/* Egress - Bridge translation registers */
+#define E_BREG_CAPABILITIES 0x00000200
+#define E_BREG_CONTROL 0x00000208
+#define E_BREG_BASE_LO 0x00000210
+#define E_BREG_BASE_HI 0x00000214
+#define E_ECAM_CAPABILITIES 0x00000220
+#define E_ECAM_CONTROL 0x00000228
+#define E_ECAM_BASE_LO 0x00000230
+#define E_ECAM_BASE_HI 0x00000234
+
+/* Ingress - address translations */
+#define I_MSII_CAPABILITIES 0x00000300
+#define I_MSII_CONTROL 0x00000308
+#define I_MSII_BASE_LO 0x00000310
+#define I_MSII_BASE_HI 0x00000314
+
+#define I_ISUB_CONTROL 0x000003E8
+#define SET_ISUB_CONTROL BIT(0)
+/* Rxed msg fifo - Interrupt status registers */
+#define MSGF_MISC_STATUS 0x00000400
+#define MSGF_MISC_MASK 0x00000404
+#define MSGF_LEG_STATUS 0x00000420
+#define MSGF_LEG_MASK 0x00000424
+#define MSGF_MSI_STATUS_LO 0x00000440
+#define MSGF_MSI_STATUS_HI 0x00000444
+#define MSGF_MSI_MASK_LO 0x00000448
+#define MSGF_MSI_MASK_HI 0x0000044C
+
+/* Msg filter mask bits */
+#define CFG_ENABLE_PM_MSG_FWD BIT(1)
+#define CFG_ENABLE_INT_MSG_FWD BIT(2)
+#define CFG_ENABLE_ERR_MSG_FWD BIT(3)
+#define CFG_ENABLE_MSG_FILTER_MASK (CFG_ENABLE_PM_MSG_FWD | \
+ CFG_ENABLE_INT_MSG_FWD | \
+ CFG_ENABLE_ERR_MSG_FWD)
+
+/* Misc interrupt status mask bits */
+#define MSGF_MISC_SR_RXMSG_AVAIL BIT(0)
+#define MSGF_MISC_SR_RXMSG_OVER BIT(1)
+#define MSGF_MISC_SR_SLAVE_ERR BIT(4)
+#define MSGF_MISC_SR_MASTER_ERR BIT(5)
+#define MSGF_MISC_SR_I_ADDR_ERR BIT(6)
+#define MSGF_MISC_SR_E_ADDR_ERR BIT(7)
+#define MSGF_MISC_SR_FATAL_AER BIT(16)
+#define MSGF_MISC_SR_NON_FATAL_AER BIT(17)
+#define MSGF_MISC_SR_CORR_AER BIT(18)
+#define MSGF_MISC_SR_UR_DETECT BIT(20)
+#define MSGF_MISC_SR_NON_FATAL_DEV BIT(22)
+#define MSGF_MISC_SR_FATAL_DEV BIT(23)
+#define MSGF_MISC_SR_LINK_DOWN BIT(24)
+#define MSGF_MSIC_SR_LINK_AUTO_BWIDTH BIT(25)
+#define MSGF_MSIC_SR_LINK_BWIDTH BIT(26)
+
+#define MSGF_MISC_SR_MASKALL (MSGF_MISC_SR_RXMSG_AVAIL | \
+ MSGF_MISC_SR_RXMSG_OVER | \
+ MSGF_MISC_SR_SLAVE_ERR | \
+ MSGF_MISC_SR_MASTER_ERR | \
+ MSGF_MISC_SR_I_ADDR_ERR | \
+ MSGF_MISC_SR_E_ADDR_ERR | \
+ MSGF_MISC_SR_FATAL_AER | \
+ MSGF_MISC_SR_NON_FATAL_AER | \
+ MSGF_MISC_SR_CORR_AER | \
+ MSGF_MISC_SR_UR_DETECT | \
+ MSGF_MISC_SR_NON_FATAL_DEV | \
+ MSGF_MISC_SR_FATAL_DEV | \
+ MSGF_MISC_SR_LINK_DOWN | \
+ MSGF_MSIC_SR_LINK_AUTO_BWIDTH | \
+ MSGF_MSIC_SR_LINK_BWIDTH)
+
+/* Legacy interrupt status mask bits */
+#define MSGF_LEG_SR_INTA BIT(0)
+#define MSGF_LEG_SR_INTB BIT(1)
+#define MSGF_LEG_SR_INTC BIT(2)
+#define MSGF_LEG_SR_INTD BIT(3)
+#define MSGF_LEG_SR_MASKALL (MSGF_LEG_SR_INTA | MSGF_LEG_SR_INTB | \
+ MSGF_LEG_SR_INTC | MSGF_LEG_SR_INTD)
+
+/* MSI interrupt status mask bits */
+#define MSGF_MSI_SR_LO_MASK GENMASK(31, 0)
+#define MSGF_MSI_SR_HI_MASK GENMASK(31, 0)
+
+#define MSII_PRESENT BIT(0)
+#define MSII_ENABLE BIT(0)
+#define MSII_STATUS_ENABLE BIT(15)
+
+/* Bridge config interrupt mask */
+#define BRCFG_INTERRUPT_MASK BIT(0)
+#define BREG_PRESENT BIT(0)
+#define BREG_ENABLE BIT(0)
+#define BREG_ENABLE_FORCE BIT(1)
+
+/* E_ECAM status mask bits */
+#define E_ECAM_PRESENT BIT(0)
+#define E_ECAM_CR_ENABLE BIT(0)
+#define E_ECAM_SIZE_LOC GENMASK(20, 16)
+#define E_ECAM_SIZE_SHIFT 16
+#define ECAM_BUS_LOC_SHIFT 20
+#define ECAM_DEV_LOC_SHIFT 12
+#define NWL_ECAM_VALUE_DEFAULT 12
+
+#define CFG_DMA_REG_BAR GENMASK(2, 0)
+
+#define INT_PCI_MSI_NR (2 * 32)
+
+/* Readin the PS_LINKUP */
+#define PS_LINKUP_OFFSET 0x00000238
+#define PCIE_PHY_LINKUP_BIT BIT(0)
+#define PHY_RDY_LINKUP_BIT BIT(1)
+
+/* Parameters for the waiting for link up routine */
+#define LINK_WAIT_MAX_RETRIES 10
+#define LINK_WAIT_USLEEP_MIN 90000
+#define LINK_WAIT_USLEEP_MAX 100000
+
+struct nwl_msi { /* MSI information */
+ struct irq_domain *msi_domain;
+ unsigned long *bitmap;
+ struct irq_domain *dev_domain;
+ struct mutex lock; /* protect bitmap variable */
+ int irq_msi0;
+ int irq_msi1;
+};
+
+struct nwl_pcie {
+ struct device *dev;
+ void __iomem *breg_base;
+ void __iomem *pcireg_base;
+ void __iomem *ecam_base;
+ phys_addr_t phys_breg_base; /* Physical Bridge Register Base */
+ phys_addr_t phys_pcie_reg_base; /* Physical PCIe Controller Base */
+ phys_addr_t phys_ecam_base; /* Physical Configuration Base */
+ u32 breg_size;
+ u32 pcie_reg_size;
+ u32 ecam_size;
+ int irq_intx;
+ int irq_misc;
+ u32 ecam_value;
+ u8 last_busno;
+ u8 root_busno;
+ struct nwl_msi msi;
+ struct irq_domain *legacy_irq_domain;
+ raw_spinlock_t leg_mask_lock;
+};
+
+static inline u32 nwl_bridge_readl(struct nwl_pcie *pcie, u32 off)
+{
+ return readl(pcie->breg_base + off);
+}
+
+static inline void nwl_bridge_writel(struct nwl_pcie *pcie, u32 val, u32 off)
+{
+ writel(val, pcie->breg_base + off);
+}
+
+static bool nwl_pcie_link_up(struct nwl_pcie *pcie)
+{
+ if (readl(pcie->pcireg_base + PS_LINKUP_OFFSET) & PCIE_PHY_LINKUP_BIT)
+ return true;
+ return false;
+}
+
+static bool nwl_phy_link_up(struct nwl_pcie *pcie)
+{
+ if (readl(pcie->pcireg_base + PS_LINKUP_OFFSET) & PHY_RDY_LINKUP_BIT)
+ return true;
+ return false;
+}
+
+static int nwl_wait_for_link(struct nwl_pcie *pcie)
+{
+ struct device *dev = pcie->dev;
+ int retries;
+
+ /* check if the link is up or not */
+ for (retries = 0; retries < LINK_WAIT_MAX_RETRIES; retries++) {
+ if (nwl_phy_link_up(pcie))
+ return 0;
+ usleep_range(LINK_WAIT_USLEEP_MIN, LINK_WAIT_USLEEP_MAX);
+ }
+
+ dev_err(dev, "PHY link never came up\n");
+ return -ETIMEDOUT;
+}
+
+static bool nwl_pcie_valid_device(struct pci_bus *bus, unsigned int devfn)
+{
+ struct nwl_pcie *pcie = bus->sysdata;
+
+ /* Check link before accessing downstream ports */
+ if (bus->number != pcie->root_busno) {
+ if (!nwl_pcie_link_up(pcie))
+ return false;
+ }
+
+ /* Only one device down on each root port */
+ if (bus->number == pcie->root_busno && devfn > 0)
+ return false;
+
+ return true;
+}
+
+/**
+ * nwl_pcie_map_bus - Get configuration base
+ *
+ * @bus: Bus structure of current bus
+ * @devfn: Device/function
+ * @where: Offset from base
+ *
+ * Return: Base address of the configuration space needed to be
+ * accessed.
+ */
+static void __iomem *nwl_pcie_map_bus(struct pci_bus *bus, unsigned int devfn,
+ int where)
+{
+ struct nwl_pcie *pcie = bus->sysdata;
+ int relbus;
+
+ if (!nwl_pcie_valid_device(bus, devfn))
+ return NULL;
+
+ relbus = (bus->number << ECAM_BUS_LOC_SHIFT) |
+ (devfn << ECAM_DEV_LOC_SHIFT);
+
+ return pcie->ecam_base + relbus + where;
+}
+
+/* PCIe operations */
+static struct pci_ops nwl_pcie_ops = {
+ .map_bus = nwl_pcie_map_bus,
+ .read = pci_generic_config_read,
+ .write = pci_generic_config_write,
+};
+
+static irqreturn_t nwl_pcie_misc_handler(int irq, void *data)
+{
+ struct nwl_pcie *pcie = data;
+ struct device *dev = pcie->dev;
+ u32 misc_stat;
+
+ /* Checking for misc interrupts */
+ misc_stat = nwl_bridge_readl(pcie, MSGF_MISC_STATUS) &
+ MSGF_MISC_SR_MASKALL;
+ if (!misc_stat)
+ return IRQ_NONE;
+
+ if (misc_stat & MSGF_MISC_SR_RXMSG_OVER)
+ dev_err(dev, "Received Message FIFO Overflow\n");
+
+ if (misc_stat & MSGF_MISC_SR_SLAVE_ERR)
+ dev_err(dev, "Slave error\n");
+
+ if (misc_stat & MSGF_MISC_SR_MASTER_ERR)
+ dev_err(dev, "Master error\n");
+
+ if (misc_stat & MSGF_MISC_SR_I_ADDR_ERR)
+ dev_err(dev, "In Misc Ingress address translation error\n");
+
+ if (misc_stat & MSGF_MISC_SR_E_ADDR_ERR)
+ dev_err(dev, "In Misc Egress address translation error\n");
+
+ if (misc_stat & MSGF_MISC_SR_FATAL_AER)
+ dev_err(dev, "Fatal Error in AER Capability\n");
+
+ if (misc_stat & MSGF_MISC_SR_NON_FATAL_AER)
+ dev_err(dev, "Non-Fatal Error in AER Capability\n");
+
+ if (misc_stat & MSGF_MISC_SR_CORR_AER)
+ dev_err(dev, "Correctable Error in AER Capability\n");
+
+ if (misc_stat & MSGF_MISC_SR_UR_DETECT)
+ dev_err(dev, "Unsupported request Detected\n");
+
+ if (misc_stat & MSGF_MISC_SR_NON_FATAL_DEV)
+ dev_err(dev, "Non-Fatal Error Detected\n");
+
+ if (misc_stat & MSGF_MISC_SR_FATAL_DEV)
+ dev_err(dev, "Fatal Error Detected\n");
+
+ if (misc_stat & MSGF_MSIC_SR_LINK_AUTO_BWIDTH)
+ dev_info(dev, "Link Autonomous Bandwidth Management Status bit set\n");
+
+ if (misc_stat & MSGF_MSIC_SR_LINK_BWIDTH)
+ dev_info(dev, "Link Bandwidth Management Status bit set\n");
+
+ /* Clear misc interrupt status */
+ nwl_bridge_writel(pcie, misc_stat, MSGF_MISC_STATUS);
+
+ return IRQ_HANDLED;
+}
+
+static void nwl_pcie_leg_handler(struct irq_desc *desc)
+{
+ struct irq_chip *chip = irq_desc_get_chip(desc);
+ struct nwl_pcie *pcie;
+ unsigned long status;
+ u32 bit;
+ u32 virq;
+
+ chained_irq_enter(chip, desc);
+ pcie = irq_desc_get_handler_data(desc);
+
+ while ((status = nwl_bridge_readl(pcie, MSGF_LEG_STATUS) &
+ MSGF_LEG_SR_MASKALL) != 0) {
+ for_each_set_bit(bit, &status, PCI_NUM_INTX) {
+ virq = irq_find_mapping(pcie->legacy_irq_domain, bit);
+ if (virq)
+ generic_handle_irq(virq);
+ }
+ }
+
+ chained_irq_exit(chip, desc);
+}
+
+static void nwl_pcie_handle_msi_irq(struct nwl_pcie *pcie, u32 status_reg)
+{
+ struct nwl_msi *msi;
+ unsigned long status;
+ u32 bit;
+ u32 virq;
+
+ msi = &pcie->msi;
+
+ while ((status = nwl_bridge_readl(pcie, status_reg)) != 0) {
+ for_each_set_bit(bit, &status, 32) {
+ nwl_bridge_writel(pcie, 1 << bit, status_reg);
+ virq = irq_find_mapping(msi->dev_domain, bit);
+ if (virq)
+ generic_handle_irq(virq);
+ }
+ }
+}
+
+static void nwl_pcie_msi_handler_high(struct irq_desc *desc)
+{
+ struct irq_chip *chip = irq_desc_get_chip(desc);
+ struct nwl_pcie *pcie = irq_desc_get_handler_data(desc);
+
+ chained_irq_enter(chip, desc);
+ nwl_pcie_handle_msi_irq(pcie, MSGF_MSI_STATUS_HI);
+ chained_irq_exit(chip, desc);
+}
+
+static void nwl_pcie_msi_handler_low(struct irq_desc *desc)
+{
+ struct irq_chip *chip = irq_desc_get_chip(desc);
+ struct nwl_pcie *pcie = irq_desc_get_handler_data(desc);
+
+ chained_irq_enter(chip, desc);
+ nwl_pcie_handle_msi_irq(pcie, MSGF_MSI_STATUS_LO);
+ chained_irq_exit(chip, desc);
+}
+
+static void nwl_mask_leg_irq(struct irq_data *data)
+{
+ struct irq_desc *desc = irq_to_desc(data->irq);
+ struct nwl_pcie *pcie;
+ unsigned long flags;
+ u32 mask;
+ u32 val;
+
+ pcie = irq_desc_get_chip_data(desc);
+ mask = 1 << (data->hwirq - 1);
+ raw_spin_lock_irqsave(&pcie->leg_mask_lock, flags);
+ val = nwl_bridge_readl(pcie, MSGF_LEG_MASK);
+ nwl_bridge_writel(pcie, (val & (~mask)), MSGF_LEG_MASK);
+ raw_spin_unlock_irqrestore(&pcie->leg_mask_lock, flags);
+}
+
+static void nwl_unmask_leg_irq(struct irq_data *data)
+{
+ struct irq_desc *desc = irq_to_desc(data->irq);
+ struct nwl_pcie *pcie;
+ unsigned long flags;
+ u32 mask;
+ u32 val;
+
+ pcie = irq_desc_get_chip_data(desc);
+ mask = 1 << (data->hwirq - 1);
+ raw_spin_lock_irqsave(&pcie->leg_mask_lock, flags);
+ val = nwl_bridge_readl(pcie, MSGF_LEG_MASK);
+ nwl_bridge_writel(pcie, (val | mask), MSGF_LEG_MASK);
+ raw_spin_unlock_irqrestore(&pcie->leg_mask_lock, flags);
+}
+
+static struct irq_chip nwl_leg_irq_chip = {
+ .name = "nwl_pcie:legacy",
+ .irq_enable = nwl_unmask_leg_irq,
+ .irq_disable = nwl_mask_leg_irq,
+ .irq_mask = nwl_mask_leg_irq,
+ .irq_unmask = nwl_unmask_leg_irq,
+};
+
+static int nwl_legacy_map(struct irq_domain *domain, unsigned int irq,
+ irq_hw_number_t hwirq)
+{
+ irq_set_chip_and_handler(irq, &nwl_leg_irq_chip, handle_level_irq);
+ irq_set_chip_data(irq, domain->host_data);
+ irq_set_status_flags(irq, IRQ_LEVEL);
+
+ return 0;
+}
+
+static const struct irq_domain_ops legacy_domain_ops = {
+ .map = nwl_legacy_map,
+ .xlate = pci_irqd_intx_xlate,
+};
+
+#ifdef CONFIG_PCI_MSI
+static struct irq_chip nwl_msi_irq_chip = {
+ .name = "nwl_pcie:msi",
+ .irq_enable = unmask_msi_irq,
+ .irq_disable = mask_msi_irq,
+ .irq_mask = mask_msi_irq,
+ .irq_unmask = unmask_msi_irq,
+
+};
+
+static struct msi_domain_info nwl_msi_domain_info = {
+ .flags = (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS |
+ MSI_FLAG_MULTI_PCI_MSI),
+ .chip = &nwl_msi_irq_chip,
+};
+#endif
+
+static void nwl_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
+{
+ struct nwl_pcie *pcie = irq_data_get_irq_chip_data(data);
+ phys_addr_t msi_addr = pcie->phys_pcie_reg_base;
+
+ msg->address_lo = lower_32_bits(msi_addr);
+ msg->address_hi = upper_32_bits(msi_addr);
+ msg->data = data->hwirq;
+}
+
+static int nwl_msi_set_affinity(struct irq_data *irq_data,
+ const struct cpumask *mask, bool force)
+{
+ return -EINVAL;
+}
+
+static struct irq_chip nwl_irq_chip = {
+ .name = "Xilinx MSI",
+ .irq_compose_msi_msg = nwl_compose_msi_msg,
+ .irq_set_affinity = nwl_msi_set_affinity,
+};
+
+static int nwl_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
+ unsigned int nr_irqs, void *args)
+{
+ struct nwl_pcie *pcie = domain->host_data;
+ struct nwl_msi *msi = &pcie->msi;
+ int bit;
+ int i;
+
+ mutex_lock(&msi->lock);
+ bit = bitmap_find_free_region(msi->bitmap, INT_PCI_MSI_NR,
+ get_count_order(nr_irqs));
+ if (bit < 0) {
+ mutex_unlock(&msi->lock);
+ return -ENOSPC;
+ }
+
+ for (i = 0; i < nr_irqs; i++) {
+ irq_domain_set_info(domain, virq + i, bit + i, &nwl_irq_chip,
+ domain->host_data, handle_simple_irq,
+ NULL, NULL);
+ }
+ mutex_unlock(&msi->lock);
+ return 0;
+}
+
+static void nwl_irq_domain_free(struct irq_domain *domain, unsigned int virq,
+ unsigned int nr_irqs)
+{
+ struct irq_data *data = irq_domain_get_irq_data(domain, virq);
+ struct nwl_pcie *pcie = irq_data_get_irq_chip_data(data);
+ struct nwl_msi *msi = &pcie->msi;
+
+ mutex_lock(&msi->lock);
+ bitmap_release_region(msi->bitmap, data->hwirq,
+ get_count_order(nr_irqs));
+ mutex_unlock(&msi->lock);
+}
+
+static const struct irq_domain_ops dev_msi_domain_ops = {
+ .alloc = nwl_irq_domain_alloc,
+ .free = nwl_irq_domain_free,
+};
+
+static int nwl_pcie_init_msi_irq_domain(struct nwl_pcie *pcie)
+{
+#ifdef CONFIG_PCI_MSI
+ struct device *dev = pcie->dev;
+ struct fwnode_handle *fwnode = of_node_to_fwnode(dev->of_node);
+ struct nwl_msi *msi = &pcie->msi;
+
+ msi->dev_domain = irq_domain_add_linear(NULL, INT_PCI_MSI_NR,
+ &dev_msi_domain_ops, pcie);
+ if (!msi->dev_domain) {
+ dev_err(dev, "failed to create dev IRQ domain\n");
+ return -ENOMEM;
+ }
+ msi->msi_domain = pci_msi_create_irq_domain(fwnode,
+ &nwl_msi_domain_info,
+ msi->dev_domain);
+ if (!msi->msi_domain) {
+ dev_err(dev, "failed to create msi IRQ domain\n");
+ irq_domain_remove(msi->dev_domain);
+ return -ENOMEM;
+ }
+#endif
+ return 0;
+}
+
+static int nwl_pcie_init_irq_domain(struct nwl_pcie *pcie)
+{
+ struct device *dev = pcie->dev;
+ struct device_node *node = dev->of_node;
+ struct device_node *legacy_intc_node;
+
+ legacy_intc_node = of_get_next_child(node, NULL);
+ if (!legacy_intc_node) {
+ dev_err(dev, "No legacy intc node found\n");
+ return -EINVAL;
+ }
+
+ pcie->legacy_irq_domain = irq_domain_add_linear(legacy_intc_node,
+ PCI_NUM_INTX,
+ &legacy_domain_ops,
+ pcie);
+ of_node_put(legacy_intc_node);
+ if (!pcie->legacy_irq_domain) {
+ dev_err(dev, "failed to create IRQ domain\n");
+ return -ENOMEM;
+ }
+
+ raw_spin_lock_init(&pcie->leg_mask_lock);
+ nwl_pcie_init_msi_irq_domain(pcie);
+ return 0;
+}
+
+static int nwl_pcie_enable_msi(struct nwl_pcie *pcie)
+{
+ struct device *dev = pcie->dev;
+ struct platform_device *pdev = to_platform_device(dev);
+ struct nwl_msi *msi = &pcie->msi;
+ unsigned long base;
+ int ret;
+ int size = BITS_TO_LONGS(INT_PCI_MSI_NR) * sizeof(long);
+
+ mutex_init(&msi->lock);
+
+ msi->bitmap = kzalloc(size, GFP_KERNEL);
+ if (!msi->bitmap)
+ return -ENOMEM;
+
+ /* Get msi_1 IRQ number */
+ msi->irq_msi1 = platform_get_irq_byname(pdev, "msi1");
+ if (msi->irq_msi1 < 0) {
+ dev_err(dev, "failed to get IRQ#%d\n", msi->irq_msi1);
+ ret = -EINVAL;
+ goto err;
+ }
+
+ irq_set_chained_handler_and_data(msi->irq_msi1,
+ nwl_pcie_msi_handler_high, pcie);
+
+ /* Get msi_0 IRQ number */
+ msi->irq_msi0 = platform_get_irq_byname(pdev, "msi0");
+ if (msi->irq_msi0 < 0) {
+ dev_err(dev, "failed to get IRQ#%d\n", msi->irq_msi0);
+ ret = -EINVAL;
+ goto err;
+ }
+
+ irq_set_chained_handler_and_data(msi->irq_msi0,
+ nwl_pcie_msi_handler_low, pcie);
+
+ /* Check for msii_present bit */
+ ret = nwl_bridge_readl(pcie, I_MSII_CAPABILITIES) & MSII_PRESENT;
+ if (!ret) {
+ dev_err(dev, "MSI not present\n");
+ ret = -EIO;
+ goto err;
+ }
+
+ /* Enable MSII */
+ nwl_bridge_writel(pcie, nwl_bridge_readl(pcie, I_MSII_CONTROL) |
+ MSII_ENABLE, I_MSII_CONTROL);
+
+ /* Enable MSII status */
+ nwl_bridge_writel(pcie, nwl_bridge_readl(pcie, I_MSII_CONTROL) |
+ MSII_STATUS_ENABLE, I_MSII_CONTROL);
+
+ /* setup AFI/FPCI range */
+ base = pcie->phys_pcie_reg_base;
+ nwl_bridge_writel(pcie, lower_32_bits(base), I_MSII_BASE_LO);
+ nwl_bridge_writel(pcie, upper_32_bits(base), I_MSII_BASE_HI);
+
+ /*
+ * For high range MSI interrupts: disable, clear any pending,
+ * and enable
+ */
+ nwl_bridge_writel(pcie, (u32)~MSGF_MSI_SR_HI_MASK, MSGF_MSI_MASK_HI);
+
+ nwl_bridge_writel(pcie, nwl_bridge_readl(pcie, MSGF_MSI_STATUS_HI) &
+ MSGF_MSI_SR_HI_MASK, MSGF_MSI_STATUS_HI);
+
+ nwl_bridge_writel(pcie, MSGF_MSI_SR_HI_MASK, MSGF_MSI_MASK_HI);
+
+ /*
+ * For low range MSI interrupts: disable, clear any pending,
+ * and enable
+ */
+ nwl_bridge_writel(pcie, (u32)~MSGF_MSI_SR_LO_MASK, MSGF_MSI_MASK_LO);
+
+ nwl_bridge_writel(pcie, nwl_bridge_readl(pcie, MSGF_MSI_STATUS_LO) &
+ MSGF_MSI_SR_LO_MASK, MSGF_MSI_STATUS_LO);
+
+ nwl_bridge_writel(pcie, MSGF_MSI_SR_LO_MASK, MSGF_MSI_MASK_LO);
+
+ return 0;
+err:
+ kfree(msi->bitmap);
+ msi->bitmap = NULL;
+ return ret;
+}
+
+static int nwl_pcie_bridge_init(struct nwl_pcie *pcie)
+{
+ struct device *dev = pcie->dev;
+ struct platform_device *pdev = to_platform_device(dev);
+ u32 breg_val, ecam_val, first_busno = 0;
+ int err;
+
+ breg_val = nwl_bridge_readl(pcie, E_BREG_CAPABILITIES) & BREG_PRESENT;
+ if (!breg_val) {
+ dev_err(dev, "BREG is not present\n");
+ return breg_val;
+ }
+
+ /* Write bridge_off to breg base */
+ nwl_bridge_writel(pcie, lower_32_bits(pcie->phys_breg_base),
+ E_BREG_BASE_LO);
+ nwl_bridge_writel(pcie, upper_32_bits(pcie->phys_breg_base),
+ E_BREG_BASE_HI);
+
+ /* Enable BREG */
+ nwl_bridge_writel(pcie, ~BREG_ENABLE_FORCE & BREG_ENABLE,
+ E_BREG_CONTROL);
+
+ /* Disable DMA channel registers */
+ nwl_bridge_writel(pcie, nwl_bridge_readl(pcie, BRCFG_PCIE_RX0) |
+ CFG_DMA_REG_BAR, BRCFG_PCIE_RX0);
+
+ /* Enable Ingress subtractive decode translation */
+ nwl_bridge_writel(pcie, SET_ISUB_CONTROL, I_ISUB_CONTROL);
+
+ /* Enable msg filtering details */
+ nwl_bridge_writel(pcie, CFG_ENABLE_MSG_FILTER_MASK,
+ BRCFG_PCIE_RX_MSG_FILTER);
+
+ err = nwl_wait_for_link(pcie);
+ if (err)
+ return err;
+
+ ecam_val = nwl_bridge_readl(pcie, E_ECAM_CAPABILITIES) & E_ECAM_PRESENT;
+ if (!ecam_val) {
+ dev_err(dev, "ECAM is not present\n");
+ return ecam_val;
+ }
+
+ /* Enable ECAM */
+ nwl_bridge_writel(pcie, nwl_bridge_readl(pcie, E_ECAM_CONTROL) |
+ E_ECAM_CR_ENABLE, E_ECAM_CONTROL);
+
+ nwl_bridge_writel(pcie, nwl_bridge_readl(pcie, E_ECAM_CONTROL) |
+ (pcie->ecam_value << E_ECAM_SIZE_SHIFT),
+ E_ECAM_CONTROL);
+
+ nwl_bridge_writel(pcie, lower_32_bits(pcie->phys_ecam_base),
+ E_ECAM_BASE_LO);
+ nwl_bridge_writel(pcie, upper_32_bits(pcie->phys_ecam_base),
+ E_ECAM_BASE_HI);
+
+ /* Get bus range */
+ ecam_val = nwl_bridge_readl(pcie, E_ECAM_CONTROL);
+ pcie->last_busno = (ecam_val & E_ECAM_SIZE_LOC) >> E_ECAM_SIZE_SHIFT;
+ /* Write primary, secondary and subordinate bus numbers */
+ ecam_val = first_busno;
+ ecam_val |= (first_busno + 1) << 8;
+ ecam_val |= (pcie->last_busno << E_ECAM_SIZE_SHIFT);
+ writel(ecam_val, (pcie->ecam_base + PCI_PRIMARY_BUS));
+
+ if (nwl_pcie_link_up(pcie))
+ dev_info(dev, "Link is UP\n");
+ else
+ dev_info(dev, "Link is DOWN\n");
+
+ /* Get misc IRQ number */
+ pcie->irq_misc = platform_get_irq_byname(pdev, "misc");
+ if (pcie->irq_misc < 0) {
+ dev_err(dev, "failed to get misc IRQ %d\n",
+ pcie->irq_misc);
+ return -EINVAL;
+ }
+
+ err = devm_request_irq(dev, pcie->irq_misc,
+ nwl_pcie_misc_handler, IRQF_SHARED,
+ "nwl_pcie:misc", pcie);
+ if (err) {
+ dev_err(dev, "fail to register misc IRQ#%d\n",
+ pcie->irq_misc);
+ return err;
+ }
+
+ /* Disable all misc interrupts */
+ nwl_bridge_writel(pcie, (u32)~MSGF_MISC_SR_MASKALL, MSGF_MISC_MASK);
+
+ /* Clear pending misc interrupts */
+ nwl_bridge_writel(pcie, nwl_bridge_readl(pcie, MSGF_MISC_STATUS) &
+ MSGF_MISC_SR_MASKALL, MSGF_MISC_STATUS);
+
+ /* Enable all misc interrupts */
+ nwl_bridge_writel(pcie, MSGF_MISC_SR_MASKALL, MSGF_MISC_MASK);
+
+
+ /* Disable all legacy interrupts */
+ nwl_bridge_writel(pcie, (u32)~MSGF_LEG_SR_MASKALL, MSGF_LEG_MASK);
+
+ /* Clear pending legacy interrupts */
+ nwl_bridge_writel(pcie, nwl_bridge_readl(pcie, MSGF_LEG_STATUS) &
+ MSGF_LEG_SR_MASKALL, MSGF_LEG_STATUS);
+
+ /* Enable all legacy interrupts */
+ nwl_bridge_writel(pcie, MSGF_LEG_SR_MASKALL, MSGF_LEG_MASK);
+
+ /* Enable the bridge config interrupt */
+ nwl_bridge_writel(pcie, nwl_bridge_readl(pcie, BRCFG_INTERRUPT) |
+ BRCFG_INTERRUPT_MASK, BRCFG_INTERRUPT);
+
+ return 0;
+}
+
+static int nwl_pcie_parse_dt(struct nwl_pcie *pcie,
+ struct platform_device *pdev)
+{
+ struct device *dev = pcie->dev;
+ struct device_node *node = dev->of_node;
+ struct resource *res;
+ const char *type;
+
+ /* Check for device type */
+ type = of_get_property(node, "device_type", NULL);
+ if (!type || strcmp(type, "pci")) {
+ dev_err(dev, "invalid \"device_type\" %s\n", type);
+ return -EINVAL;
+ }
+
+ res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "breg");
+ pcie->breg_base = devm_ioremap_resource(dev, res);
+ if (IS_ERR(pcie->breg_base))
+ return PTR_ERR(pcie->breg_base);
+ pcie->phys_breg_base = res->start;
+
+ res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "pcireg");
+ pcie->pcireg_base = devm_ioremap_resource(dev, res);
+ if (IS_ERR(pcie->pcireg_base))
+ return PTR_ERR(pcie->pcireg_base);
+ pcie->phys_pcie_reg_base = res->start;
+
+ res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "cfg");
+ pcie->ecam_base = devm_pci_remap_cfg_resource(dev, res);
+ if (IS_ERR(pcie->ecam_base))
+ return PTR_ERR(pcie->ecam_base);
+ pcie->phys_ecam_base = res->start;
+
+ /* Get intx IRQ number */
+ pcie->irq_intx = platform_get_irq_byname(pdev, "intx");
+ if (pcie->irq_intx < 0) {
+ dev_err(dev, "failed to get intx IRQ %d\n", pcie->irq_intx);
+ return pcie->irq_intx;
+ }
+
+ irq_set_chained_handler_and_data(pcie->irq_intx,
+ nwl_pcie_leg_handler, pcie);
+
+ return 0;
+}
+
+static const struct of_device_id nwl_pcie_of_match[] = {
+ { .compatible = "xlnx,nwl-pcie-2.11", },
+ {}
+};
+
+static int nwl_pcie_probe(struct platform_device *pdev)
+{
+ struct device *dev = &pdev->dev;
+ struct device_node *node = dev->of_node;
+ struct nwl_pcie *pcie;
+ struct pci_bus *bus;
+ struct pci_bus *child;
+ struct pci_host_bridge *bridge;
+ int err;
+ resource_size_t iobase = 0;
+ LIST_HEAD(res);
+
+ bridge = devm_pci_alloc_host_bridge(dev, sizeof(*pcie));
+ if (!bridge)
+ return -ENODEV;
+
+ pcie = pci_host_bridge_priv(bridge);
+
+ pcie->dev = dev;
+ pcie->ecam_value = NWL_ECAM_VALUE_DEFAULT;
+
+ err = nwl_pcie_parse_dt(pcie, pdev);
+ if (err) {
+ dev_err(dev, "Parsing DT failed\n");
+ return err;
+ }
+
+ err = nwl_pcie_bridge_init(pcie);
+ if (err) {
+ dev_err(dev, "HW Initialization failed\n");
+ return err;
+ }
+
+ err = of_pci_get_host_bridge_resources(node, 0, 0xff, &res, &iobase);
+ if (err) {
+ dev_err(dev, "Getting bridge resources failed\n");
+ return err;
+ }
+
+ err = devm_request_pci_bus_resources(dev, &res);
+ if (err)
+ goto error;
+
+ err = nwl_pcie_init_irq_domain(pcie);
+ if (err) {
+ dev_err(dev, "Failed creating IRQ Domain\n");
+ goto error;
+ }
+
+ list_splice_init(&res, &bridge->windows);
+ bridge->dev.parent = dev;
+ bridge->sysdata = pcie;
+ bridge->busnr = pcie->root_busno;
+ bridge->ops = &nwl_pcie_ops;
+ bridge->map_irq = of_irq_parse_and_map_pci;
+ bridge->swizzle_irq = pci_common_swizzle;
+
+ if (IS_ENABLED(CONFIG_PCI_MSI)) {
+ err = nwl_pcie_enable_msi(pcie);
+ if (err < 0) {
+ dev_err(dev, "failed to enable MSI support: %d\n", err);
+ goto error;
+ }
+ }
+
+ err = pci_scan_root_bus_bridge(bridge);
+ if (err)
+ goto error;
+
+ bus = bridge->bus;
+
+ pci_assign_unassigned_bus_resources(bus);
+ list_for_each_entry(child, &bus->children, node)
+ pcie_bus_configure_settings(child);
+ pci_bus_add_devices(bus);
+ return 0;
+
+error:
+ pci_free_resource_list(&res);
+ return err;
+}
+
+static struct platform_driver nwl_pcie_driver = {
+ .driver = {
+ .name = "nwl-pcie",
+ .suppress_bind_attrs = true,
+ .of_match_table = nwl_pcie_of_match,
+ },
+ .probe = nwl_pcie_probe,
+};
+builtin_platform_driver(nwl_pcie_driver);
diff --git a/src/kernel/linux/v4.14/drivers/pci/host/pcie-xilinx.c b/src/kernel/linux/v4.14/drivers/pci/host/pcie-xilinx.c
new file mode 100644
index 0000000..a8a44af
--- /dev/null
+++ b/src/kernel/linux/v4.14/drivers/pci/host/pcie-xilinx.c
@@ -0,0 +1,713 @@
+/*
+ * PCIe host controller driver for Xilinx AXI PCIe Bridge
+ *
+ * Copyright (c) 2012 - 2014 Xilinx, Inc.
+ *
+ * Based on the Tegra PCIe driver
+ *
+ * Bits taken from Synopsys DesignWare Host controller driver and
+ * ARM PCI Host generic driver.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include <linux/interrupt.h>
+#include <linux/irq.h>
+#include <linux/irqdomain.h>
+#include <linux/kernel.h>
+#include <linux/init.h>
+#include <linux/msi.h>
+#include <linux/of_address.h>
+#include <linux/of_pci.h>
+#include <linux/of_platform.h>
+#include <linux/of_irq.h>
+#include <linux/pci.h>
+#include <linux/platform_device.h>
+
+/* Register definitions */
+#define XILINX_PCIE_REG_BIR 0x00000130
+#define XILINX_PCIE_REG_IDR 0x00000138
+#define XILINX_PCIE_REG_IMR 0x0000013c
+#define XILINX_PCIE_REG_PSCR 0x00000144
+#define XILINX_PCIE_REG_RPSC 0x00000148
+#define XILINX_PCIE_REG_MSIBASE1 0x0000014c
+#define XILINX_PCIE_REG_MSIBASE2 0x00000150
+#define XILINX_PCIE_REG_RPEFR 0x00000154
+#define XILINX_PCIE_REG_RPIFR1 0x00000158
+#define XILINX_PCIE_REG_RPIFR2 0x0000015c
+
+/* Interrupt registers definitions */
+#define XILINX_PCIE_INTR_LINK_DOWN BIT(0)
+#define XILINX_PCIE_INTR_ECRC_ERR BIT(1)
+#define XILINX_PCIE_INTR_STR_ERR BIT(2)
+#define XILINX_PCIE_INTR_HOT_RESET BIT(3)
+#define XILINX_PCIE_INTR_CFG_TIMEOUT BIT(8)
+#define XILINX_PCIE_INTR_CORRECTABLE BIT(9)
+#define XILINX_PCIE_INTR_NONFATAL BIT(10)
+#define XILINX_PCIE_INTR_FATAL BIT(11)
+#define XILINX_PCIE_INTR_INTX BIT(16)
+#define XILINX_PCIE_INTR_MSI BIT(17)
+#define XILINX_PCIE_INTR_SLV_UNSUPP BIT(20)
+#define XILINX_PCIE_INTR_SLV_UNEXP BIT(21)
+#define XILINX_PCIE_INTR_SLV_COMPL BIT(22)
+#define XILINX_PCIE_INTR_SLV_ERRP BIT(23)
+#define XILINX_PCIE_INTR_SLV_CMPABT BIT(24)
+#define XILINX_PCIE_INTR_SLV_ILLBUR BIT(25)
+#define XILINX_PCIE_INTR_MST_DECERR BIT(26)
+#define XILINX_PCIE_INTR_MST_SLVERR BIT(27)
+#define XILINX_PCIE_INTR_MST_ERRP BIT(28)
+#define XILINX_PCIE_IMR_ALL_MASK 0x1FF30FED
+#define XILINX_PCIE_IMR_ENABLE_MASK 0x1FF30F0D
+#define XILINX_PCIE_IDR_ALL_MASK 0xFFFFFFFF
+
+/* Root Port Error FIFO Read Register definitions */
+#define XILINX_PCIE_RPEFR_ERR_VALID BIT(18)
+#define XILINX_PCIE_RPEFR_REQ_ID GENMASK(15, 0)
+#define XILINX_PCIE_RPEFR_ALL_MASK 0xFFFFFFFF
+
+/* Root Port Interrupt FIFO Read Register 1 definitions */
+#define XILINX_PCIE_RPIFR1_INTR_VALID BIT(31)
+#define XILINX_PCIE_RPIFR1_MSI_INTR BIT(30)
+#define XILINX_PCIE_RPIFR1_INTR_MASK GENMASK(28, 27)
+#define XILINX_PCIE_RPIFR1_ALL_MASK 0xFFFFFFFF
+#define XILINX_PCIE_RPIFR1_INTR_SHIFT 27
+
+/* Bridge Info Register definitions */
+#define XILINX_PCIE_BIR_ECAM_SZ_MASK GENMASK(18, 16)
+#define XILINX_PCIE_BIR_ECAM_SZ_SHIFT 16
+
+/* Root Port Interrupt FIFO Read Register 2 definitions */
+#define XILINX_PCIE_RPIFR2_MSG_DATA GENMASK(15, 0)
+
+/* Root Port Status/control Register definitions */
+#define XILINX_PCIE_REG_RPSC_BEN BIT(0)
+
+/* Phy Status/Control Register definitions */
+#define XILINX_PCIE_REG_PSCR_LNKUP BIT(11)
+
+/* ECAM definitions */
+#define ECAM_BUS_NUM_SHIFT 20
+#define ECAM_DEV_NUM_SHIFT 12
+
+/* Number of MSI IRQs */
+#define XILINX_NUM_MSI_IRQS 128
+
+/**
+ * struct xilinx_pcie_port - PCIe port information
+ * @reg_base: IO Mapped Register Base
+ * @irq: Interrupt number
+ * @msi_pages: MSI pages
+ * @root_busno: Root Bus number
+ * @dev: Device pointer
+ * @msi_domain: MSI IRQ domain pointer
+ * @leg_domain: Legacy IRQ domain pointer
+ * @resources: Bus Resources
+ */
+struct xilinx_pcie_port {
+ void __iomem *reg_base;
+ u32 irq;
+ unsigned long msi_pages;
+ u8 root_busno;
+ struct device *dev;
+ struct irq_domain *msi_domain;
+ struct irq_domain *leg_domain;
+ struct list_head resources;
+};
+
+static DECLARE_BITMAP(msi_irq_in_use, XILINX_NUM_MSI_IRQS);
+
+static inline u32 pcie_read(struct xilinx_pcie_port *port, u32 reg)
+{
+ return readl(port->reg_base + reg);
+}
+
+static inline void pcie_write(struct xilinx_pcie_port *port, u32 val, u32 reg)
+{
+ writel(val, port->reg_base + reg);
+}
+
+static inline bool xilinx_pcie_link_is_up(struct xilinx_pcie_port *port)
+{
+ return (pcie_read(port, XILINX_PCIE_REG_PSCR) &
+ XILINX_PCIE_REG_PSCR_LNKUP) ? 1 : 0;
+}
+
+/**
+ * xilinx_pcie_clear_err_interrupts - Clear Error Interrupts
+ * @port: PCIe port information
+ */
+static void xilinx_pcie_clear_err_interrupts(struct xilinx_pcie_port *port)
+{
+ struct device *dev = port->dev;
+ unsigned long val = pcie_read(port, XILINX_PCIE_REG_RPEFR);
+
+ if (val & XILINX_PCIE_RPEFR_ERR_VALID) {
+ dev_dbg(dev, "Requester ID %lu\n",
+ val & XILINX_PCIE_RPEFR_REQ_ID);
+ pcie_write(port, XILINX_PCIE_RPEFR_ALL_MASK,
+ XILINX_PCIE_REG_RPEFR);
+ }
+}
+
+/**
+ * xilinx_pcie_valid_device - Check if a valid device is present on bus
+ * @bus: PCI Bus structure
+ * @devfn: device/function
+ *
+ * Return: 'true' on success and 'false' if invalid device is found
+ */
+static bool xilinx_pcie_valid_device(struct pci_bus *bus, unsigned int devfn)
+{
+ struct xilinx_pcie_port *port = bus->sysdata;
+
+ /* Check if link is up when trying to access downstream ports */
+ if (bus->number != port->root_busno)
+ if (!xilinx_pcie_link_is_up(port))
+ return false;
+
+ /* Only one device down on each root port */
+ if (bus->number == port->root_busno && devfn > 0)
+ return false;
+
+ return true;
+}
+
+/**
+ * xilinx_pcie_map_bus - Get configuration base
+ * @bus: PCI Bus structure
+ * @devfn: Device/function
+ * @where: Offset from base
+ *
+ * Return: Base address of the configuration space needed to be
+ * accessed.
+ */
+static void __iomem *xilinx_pcie_map_bus(struct pci_bus *bus,
+ unsigned int devfn, int where)
+{
+ struct xilinx_pcie_port *port = bus->sysdata;
+ int relbus;
+
+ if (!xilinx_pcie_valid_device(bus, devfn))
+ return NULL;
+
+ relbus = (bus->number << ECAM_BUS_NUM_SHIFT) |
+ (devfn << ECAM_DEV_NUM_SHIFT);
+
+ return port->reg_base + relbus + where;
+}
+
+/* PCIe operations */
+static struct pci_ops xilinx_pcie_ops = {
+ .map_bus = xilinx_pcie_map_bus,
+ .read = pci_generic_config_read,
+ .write = pci_generic_config_write,
+};
+
+/* MSI functions */
+
+/**
+ * xilinx_pcie_destroy_msi - Free MSI number
+ * @irq: IRQ to be freed
+ */
+static void xilinx_pcie_destroy_msi(unsigned int irq)
+{
+ struct msi_desc *msi;
+ struct xilinx_pcie_port *port;
+ struct irq_data *d = irq_get_irq_data(irq);
+ irq_hw_number_t hwirq = irqd_to_hwirq(d);
+
+ if (!test_bit(hwirq, msi_irq_in_use)) {
+ msi = irq_get_msi_desc(irq);
+ port = msi_desc_to_pci_sysdata(msi);
+ dev_err(port->dev, "Trying to free unused MSI#%d\n", irq);
+ } else {
+ clear_bit(hwirq, msi_irq_in_use);
+ }
+}
+
+/**
+ * xilinx_pcie_assign_msi - Allocate MSI number
+ *
+ * Return: A valid IRQ on success and error value on failure.
+ */
+static int xilinx_pcie_assign_msi(void)
+{
+ int pos;
+
+ pos = find_first_zero_bit(msi_irq_in_use, XILINX_NUM_MSI_IRQS);
+ if (pos < XILINX_NUM_MSI_IRQS)
+ set_bit(pos, msi_irq_in_use);
+ else
+ return -ENOSPC;
+
+ return pos;
+}
+
+/**
+ * xilinx_msi_teardown_irq - Destroy the MSI
+ * @chip: MSI Chip descriptor
+ * @irq: MSI IRQ to destroy
+ */
+static void xilinx_msi_teardown_irq(struct msi_controller *chip,
+ unsigned int irq)
+{
+ xilinx_pcie_destroy_msi(irq);
+ irq_dispose_mapping(irq);
+}
+
+/**
+ * xilinx_pcie_msi_setup_irq - Setup MSI request
+ * @chip: MSI chip pointer
+ * @pdev: PCIe device pointer
+ * @desc: MSI descriptor pointer
+ *
+ * Return: '0' on success and error value on failure
+ */
+static int xilinx_pcie_msi_setup_irq(struct msi_controller *chip,
+ struct pci_dev *pdev,
+ struct msi_desc *desc)
+{
+ struct xilinx_pcie_port *port = pdev->bus->sysdata;
+ unsigned int irq;
+ int hwirq;
+ struct msi_msg msg;
+ phys_addr_t msg_addr;
+
+ hwirq = xilinx_pcie_assign_msi();
+ if (hwirq < 0)
+ return hwirq;
+
+ irq = irq_create_mapping(port->msi_domain, hwirq);
+ if (!irq)
+ return -EINVAL;
+
+ irq_set_msi_desc(irq, desc);
+
+ msg_addr = virt_to_phys((void *)port->msi_pages);
+
+ msg.address_hi = 0;
+ msg.address_lo = msg_addr;
+ msg.data = irq;
+
+ pci_write_msi_msg(irq, &msg);
+
+ return 0;
+}
+
+/* MSI Chip Descriptor */
+static struct msi_controller xilinx_pcie_msi_chip = {
+ .setup_irq = xilinx_pcie_msi_setup_irq,
+ .teardown_irq = xilinx_msi_teardown_irq,
+};
+
+/* HW Interrupt Chip Descriptor */
+static struct irq_chip xilinx_msi_irq_chip = {
+ .name = "Xilinx PCIe MSI",
+ .irq_enable = pci_msi_unmask_irq,
+ .irq_disable = pci_msi_mask_irq,
+ .irq_mask = pci_msi_mask_irq,
+ .irq_unmask = pci_msi_unmask_irq,
+};
+
+/**
+ * xilinx_pcie_msi_map - Set the handler for the MSI and mark IRQ as valid
+ * @domain: IRQ domain
+ * @irq: Virtual IRQ number
+ * @hwirq: HW interrupt number
+ *
+ * Return: Always returns 0.
+ */
+static int xilinx_pcie_msi_map(struct irq_domain *domain, unsigned int irq,
+ irq_hw_number_t hwirq)
+{
+ irq_set_chip_and_handler(irq, &xilinx_msi_irq_chip, handle_simple_irq);
+ irq_set_chip_data(irq, domain->host_data);
+
+ return 0;
+}
+
+/* IRQ Domain operations */
+static const struct irq_domain_ops msi_domain_ops = {
+ .map = xilinx_pcie_msi_map,
+};
+
+/**
+ * xilinx_pcie_enable_msi - Enable MSI support
+ * @port: PCIe port information
+ */
+static int xilinx_pcie_enable_msi(struct xilinx_pcie_port *port)
+{
+ phys_addr_t msg_addr;
+
+ port->msi_pages = __get_free_pages(GFP_KERNEL, 0);
+ if (!port->msi_pages)
+ return -ENOMEM;
+
+ msg_addr = virt_to_phys((void *)port->msi_pages);
+ pcie_write(port, 0x0, XILINX_PCIE_REG_MSIBASE1);
+ pcie_write(port, msg_addr, XILINX_PCIE_REG_MSIBASE2);
+
+ return 0;
+}
+
+/* INTx Functions */
+
+/**
+ * xilinx_pcie_intx_map - Set the handler for the INTx and mark IRQ as valid
+ * @domain: IRQ domain
+ * @irq: Virtual IRQ number
+ * @hwirq: HW interrupt number
+ *
+ * Return: Always returns 0.
+ */
+static int xilinx_pcie_intx_map(struct irq_domain *domain, unsigned int irq,
+ irq_hw_number_t hwirq)
+{
+ irq_set_chip_and_handler(irq, &dummy_irq_chip, handle_simple_irq);
+ irq_set_chip_data(irq, domain->host_data);
+
+ return 0;
+}
+
+/* INTx IRQ Domain operations */
+static const struct irq_domain_ops intx_domain_ops = {
+ .map = xilinx_pcie_intx_map,
+ .xlate = pci_irqd_intx_xlate,
+};
+
+/* PCIe HW Functions */
+
+/**
+ * xilinx_pcie_intr_handler - Interrupt Service Handler
+ * @irq: IRQ number
+ * @data: PCIe port information
+ *
+ * Return: IRQ_HANDLED on success and IRQ_NONE on failure
+ */
+static irqreturn_t xilinx_pcie_intr_handler(int irq, void *data)
+{
+ struct xilinx_pcie_port *port = (struct xilinx_pcie_port *)data;
+ struct device *dev = port->dev;
+ u32 val, mask, status;
+
+ /* Read interrupt decode and mask registers */
+ val = pcie_read(port, XILINX_PCIE_REG_IDR);
+ mask = pcie_read(port, XILINX_PCIE_REG_IMR);
+
+ status = val & mask;
+ if (!status)
+ return IRQ_NONE;
+
+ if (status & XILINX_PCIE_INTR_LINK_DOWN)
+ dev_warn(dev, "Link Down\n");
+
+ if (status & XILINX_PCIE_INTR_ECRC_ERR)
+ dev_warn(dev, "ECRC failed\n");
+
+ if (status & XILINX_PCIE_INTR_STR_ERR)
+ dev_warn(dev, "Streaming error\n");
+
+ if (status & XILINX_PCIE_INTR_HOT_RESET)
+ dev_info(dev, "Hot reset\n");
+
+ if (status & XILINX_PCIE_INTR_CFG_TIMEOUT)
+ dev_warn(dev, "ECAM access timeout\n");
+
+ if (status & XILINX_PCIE_INTR_CORRECTABLE) {
+ dev_warn(dev, "Correctable error message\n");
+ xilinx_pcie_clear_err_interrupts(port);
+ }
+
+ if (status & XILINX_PCIE_INTR_NONFATAL) {
+ dev_warn(dev, "Non fatal error message\n");
+ xilinx_pcie_clear_err_interrupts(port);
+ }
+
+ if (status & XILINX_PCIE_INTR_FATAL) {
+ dev_warn(dev, "Fatal error message\n");
+ xilinx_pcie_clear_err_interrupts(port);
+ }
+
+ if (status & (XILINX_PCIE_INTR_INTX | XILINX_PCIE_INTR_MSI)) {
+ val = pcie_read(port, XILINX_PCIE_REG_RPIFR1);
+
+ /* Check whether interrupt valid */
+ if (!(val & XILINX_PCIE_RPIFR1_INTR_VALID)) {
+ dev_warn(dev, "RP Intr FIFO1 read error\n");
+ goto error;
+ }
+
+ /* Decode the IRQ number */
+ if (val & XILINX_PCIE_RPIFR1_MSI_INTR) {
+ val = pcie_read(port, XILINX_PCIE_REG_RPIFR2) &
+ XILINX_PCIE_RPIFR2_MSG_DATA;
+ } else {
+ val = (val & XILINX_PCIE_RPIFR1_INTR_MASK) >>
+ XILINX_PCIE_RPIFR1_INTR_SHIFT;
+ val = irq_find_mapping(port->leg_domain, val);
+ }
+
+ /* Clear interrupt FIFO register 1 */
+ pcie_write(port, XILINX_PCIE_RPIFR1_ALL_MASK,
+ XILINX_PCIE_REG_RPIFR1);
+
+ /* Handle the interrupt */
+ if (IS_ENABLED(CONFIG_PCI_MSI) ||
+ !(val & XILINX_PCIE_RPIFR1_MSI_INTR))
+ generic_handle_irq(val);
+ }
+
+ if (status & XILINX_PCIE_INTR_SLV_UNSUPP)
+ dev_warn(dev, "Slave unsupported request\n");
+
+ if (status & XILINX_PCIE_INTR_SLV_UNEXP)
+ dev_warn(dev, "Slave unexpected completion\n");
+
+ if (status & XILINX_PCIE_INTR_SLV_COMPL)
+ dev_warn(dev, "Slave completion timeout\n");
+
+ if (status & XILINX_PCIE_INTR_SLV_ERRP)
+ dev_warn(dev, "Slave Error Poison\n");
+
+ if (status & XILINX_PCIE_INTR_SLV_CMPABT)
+ dev_warn(dev, "Slave Completer Abort\n");
+
+ if (status & XILINX_PCIE_INTR_SLV_ILLBUR)
+ dev_warn(dev, "Slave Illegal Burst\n");
+
+ if (status & XILINX_PCIE_INTR_MST_DECERR)
+ dev_warn(dev, "Master decode error\n");
+
+ if (status & XILINX_PCIE_INTR_MST_SLVERR)
+ dev_warn(dev, "Master slave error\n");
+
+ if (status & XILINX_PCIE_INTR_MST_ERRP)
+ dev_warn(dev, "Master error poison\n");
+
+error:
+ /* Clear the Interrupt Decode register */
+ pcie_write(port, status, XILINX_PCIE_REG_IDR);
+
+ return IRQ_HANDLED;
+}
+
+/**
+ * xilinx_pcie_init_irq_domain - Initialize IRQ domain
+ * @port: PCIe port information
+ *
+ * Return: '0' on success and error value on failure
+ */
+static int xilinx_pcie_init_irq_domain(struct xilinx_pcie_port *port)
+{
+ struct device *dev = port->dev;
+ struct device_node *node = dev->of_node;
+ struct device_node *pcie_intc_node;
+ int ret;
+
+ /* Setup INTx */
+ pcie_intc_node = of_get_next_child(node, NULL);
+ if (!pcie_intc_node) {
+ dev_err(dev, "No PCIe Intc node found\n");
+ return -ENODEV;
+ }
+
+ port->leg_domain = irq_domain_add_linear(pcie_intc_node, PCI_NUM_INTX,
+ &intx_domain_ops,
+ port);
+ of_node_put(pcie_intc_node);
+ if (!port->leg_domain) {
+ dev_err(dev, "Failed to get a INTx IRQ domain\n");
+ return -ENODEV;
+ }
+
+ /* Setup MSI */
+ if (IS_ENABLED(CONFIG_PCI_MSI)) {
+ port->msi_domain = irq_domain_add_linear(node,
+ XILINX_NUM_MSI_IRQS,
+ &msi_domain_ops,
+ &xilinx_pcie_msi_chip);
+ if (!port->msi_domain) {
+ dev_err(dev, "Failed to get a MSI IRQ domain\n");
+ return -ENODEV;
+ }
+
+ ret = xilinx_pcie_enable_msi(port);
+ if (ret)
+ return ret;
+ }
+
+ return 0;
+}
+
+/**
+ * xilinx_pcie_init_port - Initialize hardware
+ * @port: PCIe port information
+ */
+static void xilinx_pcie_init_port(struct xilinx_pcie_port *port)
+{
+ struct device *dev = port->dev;
+
+ if (xilinx_pcie_link_is_up(port))
+ dev_info(dev, "PCIe Link is UP\n");
+ else
+ dev_info(dev, "PCIe Link is DOWN\n");
+
+ /* Disable all interrupts */
+ pcie_write(port, ~XILINX_PCIE_IDR_ALL_MASK,
+ XILINX_PCIE_REG_IMR);
+
+ /* Clear pending interrupts */
+ pcie_write(port, pcie_read(port, XILINX_PCIE_REG_IDR) &
+ XILINX_PCIE_IMR_ALL_MASK,
+ XILINX_PCIE_REG_IDR);
+
+ /* Enable all interrupts we handle */
+ pcie_write(port, XILINX_PCIE_IMR_ENABLE_MASK, XILINX_PCIE_REG_IMR);
+
+ /* Enable the Bridge enable bit */
+ pcie_write(port, pcie_read(port, XILINX_PCIE_REG_RPSC) |
+ XILINX_PCIE_REG_RPSC_BEN,
+ XILINX_PCIE_REG_RPSC);
+}
+
+/**
+ * xilinx_pcie_parse_dt - Parse Device tree
+ * @port: PCIe port information
+ *
+ * Return: '0' on success and error value on failure
+ */
+static int xilinx_pcie_parse_dt(struct xilinx_pcie_port *port)
+{
+ struct device *dev = port->dev;
+ struct device_node *node = dev->of_node;
+ struct resource regs;
+ const char *type;
+ int err;
+
+ type = of_get_property(node, "device_type", NULL);
+ if (!type || strcmp(type, "pci")) {
+ dev_err(dev, "invalid \"device_type\" %s\n", type);
+ return -EINVAL;
+ }
+
+ err = of_address_to_resource(node, 0, ®s);
+ if (err) {
+ dev_err(dev, "missing \"reg\" property\n");
+ return err;
+ }
+
+ port->reg_base = devm_pci_remap_cfg_resource(dev, ®s);
+ if (IS_ERR(port->reg_base))
+ return PTR_ERR(port->reg_base);
+
+ port->irq = irq_of_parse_and_map(node, 0);
+ err = devm_request_irq(dev, port->irq, xilinx_pcie_intr_handler,
+ IRQF_SHARED | IRQF_NO_THREAD,
+ "xilinx-pcie", port);
+ if (err) {
+ dev_err(dev, "unable to request irq %d\n", port->irq);
+ return err;
+ }
+
+ return 0;
+}
+
+/**
+ * xilinx_pcie_probe - Probe function
+ * @pdev: Platform device pointer
+ *
+ * Return: '0' on success and error value on failure
+ */
+static int xilinx_pcie_probe(struct platform_device *pdev)
+{
+ struct device *dev = &pdev->dev;
+ struct xilinx_pcie_port *port;
+ struct pci_bus *bus, *child;
+ struct pci_host_bridge *bridge;
+ int err;
+ resource_size_t iobase = 0;
+ LIST_HEAD(res);
+
+ if (!dev->of_node)
+ return -ENODEV;
+
+ bridge = devm_pci_alloc_host_bridge(dev, sizeof(*port));
+ if (!bridge)
+ return -ENODEV;
+
+ port = pci_host_bridge_priv(bridge);
+
+ port->dev = dev;
+
+ err = xilinx_pcie_parse_dt(port);
+ if (err) {
+ dev_err(dev, "Parsing DT failed\n");
+ return err;
+ }
+
+ xilinx_pcie_init_port(port);
+
+ err = xilinx_pcie_init_irq_domain(port);
+ if (err) {
+ dev_err(dev, "Failed creating IRQ Domain\n");
+ return err;
+ }
+
+ err = of_pci_get_host_bridge_resources(dev->of_node, 0, 0xff, &res,
+ &iobase);
+ if (err) {
+ dev_err(dev, "Getting bridge resources failed\n");
+ return err;
+ }
+
+ err = devm_request_pci_bus_resources(dev, &res);
+ if (err)
+ goto error;
+
+
+ list_splice_init(&res, &bridge->windows);
+ bridge->dev.parent = dev;
+ bridge->sysdata = port;
+ bridge->busnr = 0;
+ bridge->ops = &xilinx_pcie_ops;
+ bridge->map_irq = of_irq_parse_and_map_pci;
+ bridge->swizzle_irq = pci_common_swizzle;
+
+#ifdef CONFIG_PCI_MSI
+ xilinx_pcie_msi_chip.dev = dev;
+ bridge->msi = &xilinx_pcie_msi_chip;
+#endif
+ err = pci_scan_root_bus_bridge(bridge);
+ if (err < 0)
+ goto error;
+
+ bus = bridge->bus;
+
+ pci_assign_unassigned_bus_resources(bus);
+ list_for_each_entry(child, &bus->children, node)
+ pcie_bus_configure_settings(child);
+ pci_bus_add_devices(bus);
+ return 0;
+
+error:
+ pci_free_resource_list(&res);
+ return err;
+}
+
+static const struct of_device_id xilinx_pcie_of_match[] = {
+ { .compatible = "xlnx,axi-pcie-host-1.00.a", },
+ {}
+};
+
+static struct platform_driver xilinx_pcie_driver = {
+ .driver = {
+ .name = "xilinx-pcie",
+ .of_match_table = xilinx_pcie_of_match,
+ .suppress_bind_attrs = true,
+ },
+ .probe = xilinx_pcie_probe,
+};
+builtin_platform_driver(xilinx_pcie_driver);
diff --git a/src/kernel/linux/v4.14/drivers/pci/host/vmd.c b/src/kernel/linux/v4.14/drivers/pci/host/vmd.c
new file mode 100644
index 0000000..79d5663
--- /dev/null
+++ b/src/kernel/linux/v4.14/drivers/pci/host/vmd.c
@@ -0,0 +1,822 @@
+/*
+ * Volume Management Device driver
+ * Copyright (c) 2015, Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ */
+
+#include <linux/device.h>
+#include <linux/interrupt.h>
+#include <linux/irq.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/msi.h>
+#include <linux/pci.h>
+#include <linux/srcu.h>
+#include <linux/rculist.h>
+#include <linux/rcupdate.h>
+
+#include <asm/irqdomain.h>
+#include <asm/device.h>
+#include <asm/msi.h>
+#include <asm/msidef.h>
+
+#define VMD_CFGBAR 0
+#define VMD_MEMBAR1 2
+#define VMD_MEMBAR2 4
+
+/*
+ * Lock for manipulating VMD IRQ lists.
+ */
+static DEFINE_RAW_SPINLOCK(list_lock);
+
+/**
+ * struct vmd_irq - private data to map driver IRQ to the VMD shared vector
+ * @node: list item for parent traversal.
+ * @irq: back pointer to parent.
+ * @enabled: true if driver enabled IRQ
+ * @virq: the virtual IRQ value provided to the requesting driver.
+ *
+ * Every MSI/MSI-X IRQ requested for a device in a VMD domain will be mapped to
+ * a VMD IRQ using this structure.
+ */
+struct vmd_irq {
+ struct list_head node;
+ struct vmd_irq_list *irq;
+ bool enabled;
+ unsigned int virq;
+};
+
+/**
+ * struct vmd_irq_list - list of driver requested IRQs mapping to a VMD vector
+ * @irq_list: the list of irq's the VMD one demuxes to.
+ * @srcu: SRCU struct for local synchronization.
+ * @count: number of child IRQs assigned to this vector; used to track
+ * sharing.
+ */
+struct vmd_irq_list {
+ struct list_head irq_list;
+ struct srcu_struct srcu;
+ unsigned int count;
+};
+
+struct vmd_dev {
+ struct pci_dev *dev;
+
+ spinlock_t cfg_lock;
+ char __iomem *cfgbar;
+
+ int msix_count;
+ struct vmd_irq_list *irqs;
+
+ struct pci_sysdata sysdata;
+ struct resource resources[3];
+ struct irq_domain *irq_domain;
+ struct pci_bus *bus;
+
+#ifdef CONFIG_X86_DEV_DMA_OPS
+ struct dma_map_ops dma_ops;
+ struct dma_domain dma_domain;
+#endif
+};
+
+static inline struct vmd_dev *vmd_from_bus(struct pci_bus *bus)
+{
+ return container_of(bus->sysdata, struct vmd_dev, sysdata);
+}
+
+static inline unsigned int index_from_irqs(struct vmd_dev *vmd,
+ struct vmd_irq_list *irqs)
+{
+ return irqs - vmd->irqs;
+}
+
+/*
+ * Drivers managing a device in a VMD domain allocate their own IRQs as before,
+ * but the MSI entry for the hardware it's driving will be programmed with a
+ * destination ID for the VMD MSI-X table. The VMD muxes interrupts in its
+ * domain into one of its own, and the VMD driver de-muxes these for the
+ * handlers sharing that VMD IRQ. The vmd irq_domain provides the operations
+ * and irq_chip to set this up.
+ */
+static void vmd_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
+{
+ struct vmd_irq *vmdirq = data->chip_data;
+ struct vmd_irq_list *irq = vmdirq->irq;
+ struct vmd_dev *vmd = irq_data_get_irq_handler_data(data);
+
+ msg->address_hi = MSI_ADDR_BASE_HI;
+ msg->address_lo = MSI_ADDR_BASE_LO |
+ MSI_ADDR_DEST_ID(index_from_irqs(vmd, irq));
+ msg->data = 0;
+}
+
+/*
+ * We rely on MSI_FLAG_USE_DEF_CHIP_OPS to set the IRQ mask/unmask ops.
+ */
+static void vmd_irq_enable(struct irq_data *data)
+{
+ struct vmd_irq *vmdirq = data->chip_data;
+ unsigned long flags;
+
+ raw_spin_lock_irqsave(&list_lock, flags);
+ WARN_ON(vmdirq->enabled);
+ list_add_tail_rcu(&vmdirq->node, &vmdirq->irq->irq_list);
+ vmdirq->enabled = true;
+ raw_spin_unlock_irqrestore(&list_lock, flags);
+
+ data->chip->irq_unmask(data);
+}
+
+static void vmd_irq_disable(struct irq_data *data)
+{
+ struct vmd_irq *vmdirq = data->chip_data;
+ unsigned long flags;
+
+ data->chip->irq_mask(data);
+
+ raw_spin_lock_irqsave(&list_lock, flags);
+ if (vmdirq->enabled) {
+ list_del_rcu(&vmdirq->node);
+ vmdirq->enabled = false;
+ }
+ raw_spin_unlock_irqrestore(&list_lock, flags);
+}
+
+/*
+ * XXX: Stubbed until we develop acceptable way to not create conflicts with
+ * other devices sharing the same vector.
+ */
+static int vmd_irq_set_affinity(struct irq_data *data,
+ const struct cpumask *dest, bool force)
+{
+ return -EINVAL;
+}
+
+static struct irq_chip vmd_msi_controller = {
+ .name = "VMD-MSI",
+ .irq_enable = vmd_irq_enable,
+ .irq_disable = vmd_irq_disable,
+ .irq_compose_msi_msg = vmd_compose_msi_msg,
+ .irq_set_affinity = vmd_irq_set_affinity,
+};
+
+static irq_hw_number_t vmd_get_hwirq(struct msi_domain_info *info,
+ msi_alloc_info_t *arg)
+{
+ return 0;
+}
+
+/*
+ * XXX: We can be even smarter selecting the best IRQ once we solve the
+ * affinity problem.
+ */
+static struct vmd_irq_list *vmd_next_irq(struct vmd_dev *vmd, struct msi_desc *desc)
+{
+ int i, best = 1;
+ unsigned long flags;
+
+ if (vmd->msix_count == 1)
+ return &vmd->irqs[0];
+
+ /*
+ * White list for fast-interrupt handlers. All others will share the
+ * "slow" interrupt vector.
+ */
+ switch (msi_desc_to_pci_dev(desc)->class) {
+ case PCI_CLASS_STORAGE_EXPRESS:
+ break;
+ default:
+ return &vmd->irqs[0];
+ }
+
+ raw_spin_lock_irqsave(&list_lock, flags);
+ for (i = 1; i < vmd->msix_count; i++)
+ if (vmd->irqs[i].count < vmd->irqs[best].count)
+ best = i;
+ vmd->irqs[best].count++;
+ raw_spin_unlock_irqrestore(&list_lock, flags);
+
+ return &vmd->irqs[best];
+}
+
+static int vmd_msi_init(struct irq_domain *domain, struct msi_domain_info *info,
+ unsigned int virq, irq_hw_number_t hwirq,
+ msi_alloc_info_t *arg)
+{
+ struct msi_desc *desc = arg->desc;
+ struct vmd_dev *vmd = vmd_from_bus(msi_desc_to_pci_dev(desc)->bus);
+ struct vmd_irq *vmdirq = kzalloc(sizeof(*vmdirq), GFP_KERNEL);
+ unsigned int index, vector;
+
+ if (!vmdirq)
+ return -ENOMEM;
+
+ INIT_LIST_HEAD(&vmdirq->node);
+ vmdirq->irq = vmd_next_irq(vmd, desc);
+ vmdirq->virq = virq;
+ index = index_from_irqs(vmd, vmdirq->irq);
+ vector = pci_irq_vector(vmd->dev, index);
+
+ irq_domain_set_info(domain, virq, vector, info->chip, vmdirq,
+ handle_untracked_irq, vmd, NULL);
+ return 0;
+}
+
+static void vmd_msi_free(struct irq_domain *domain,
+ struct msi_domain_info *info, unsigned int virq)
+{
+ struct vmd_irq *vmdirq = irq_get_chip_data(virq);
+ unsigned long flags;
+
+ synchronize_srcu(&vmdirq->irq->srcu);
+
+ /* XXX: Potential optimization to rebalance */
+ raw_spin_lock_irqsave(&list_lock, flags);
+ vmdirq->irq->count--;
+ raw_spin_unlock_irqrestore(&list_lock, flags);
+
+ kfree(vmdirq);
+}
+
+static int vmd_msi_prepare(struct irq_domain *domain, struct device *dev,
+ int nvec, msi_alloc_info_t *arg)
+{
+ struct pci_dev *pdev = to_pci_dev(dev);
+ struct vmd_dev *vmd = vmd_from_bus(pdev->bus);
+
+ if (nvec > vmd->msix_count)
+ return vmd->msix_count;
+
+ memset(arg, 0, sizeof(*arg));
+ return 0;
+}
+
+static void vmd_set_desc(msi_alloc_info_t *arg, struct msi_desc *desc)
+{
+ arg->desc = desc;
+}
+
+static struct msi_domain_ops vmd_msi_domain_ops = {
+ .get_hwirq = vmd_get_hwirq,
+ .msi_init = vmd_msi_init,
+ .msi_free = vmd_msi_free,
+ .msi_prepare = vmd_msi_prepare,
+ .set_desc = vmd_set_desc,
+};
+
+static struct msi_domain_info vmd_msi_domain_info = {
+ .flags = MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS |
+ MSI_FLAG_PCI_MSIX,
+ .ops = &vmd_msi_domain_ops,
+ .chip = &vmd_msi_controller,
+};
+
+#ifdef CONFIG_X86_DEV_DMA_OPS
+/*
+ * VMD replaces the requester ID with its own. DMA mappings for devices in a
+ * VMD domain need to be mapped for the VMD, not the device requiring
+ * the mapping.
+ */
+static struct device *to_vmd_dev(struct device *dev)
+{
+ struct pci_dev *pdev = to_pci_dev(dev);
+ struct vmd_dev *vmd = vmd_from_bus(pdev->bus);
+
+ return &vmd->dev->dev;
+}
+
+static const struct dma_map_ops *vmd_dma_ops(struct device *dev)
+{
+ return get_dma_ops(to_vmd_dev(dev));
+}
+
+static void *vmd_alloc(struct device *dev, size_t size, dma_addr_t *addr,
+ gfp_t flag, unsigned long attrs)
+{
+ return vmd_dma_ops(dev)->alloc(to_vmd_dev(dev), size, addr, flag,
+ attrs);
+}
+
+static void vmd_free(struct device *dev, size_t size, void *vaddr,
+ dma_addr_t addr, unsigned long attrs)
+{
+ return vmd_dma_ops(dev)->free(to_vmd_dev(dev), size, vaddr, addr,
+ attrs);
+}
+
+static int vmd_mmap(struct device *dev, struct vm_area_struct *vma,
+ void *cpu_addr, dma_addr_t addr, size_t size,
+ unsigned long attrs)
+{
+ return vmd_dma_ops(dev)->mmap(to_vmd_dev(dev), vma, cpu_addr, addr,
+ size, attrs);
+}
+
+static int vmd_get_sgtable(struct device *dev, struct sg_table *sgt,
+ void *cpu_addr, dma_addr_t addr, size_t size,
+ unsigned long attrs)
+{
+ return vmd_dma_ops(dev)->get_sgtable(to_vmd_dev(dev), sgt, cpu_addr,
+ addr, size, attrs);
+}
+
+static dma_addr_t vmd_map_page(struct device *dev, struct page *page,
+ unsigned long offset, size_t size,
+ enum dma_data_direction dir,
+ unsigned long attrs)
+{
+ return vmd_dma_ops(dev)->map_page(to_vmd_dev(dev), page, offset, size,
+ dir, attrs);
+}
+
+static void vmd_unmap_page(struct device *dev, dma_addr_t addr, size_t size,
+ enum dma_data_direction dir, unsigned long attrs)
+{
+ vmd_dma_ops(dev)->unmap_page(to_vmd_dev(dev), addr, size, dir, attrs);
+}
+
+static int vmd_map_sg(struct device *dev, struct scatterlist *sg, int nents,
+ enum dma_data_direction dir, unsigned long attrs)
+{
+ return vmd_dma_ops(dev)->map_sg(to_vmd_dev(dev), sg, nents, dir, attrs);
+}
+
+static void vmd_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
+ enum dma_data_direction dir, unsigned long attrs)
+{
+ vmd_dma_ops(dev)->unmap_sg(to_vmd_dev(dev), sg, nents, dir, attrs);
+}
+
+static void vmd_sync_single_for_cpu(struct device *dev, dma_addr_t addr,
+ size_t size, enum dma_data_direction dir)
+{
+ vmd_dma_ops(dev)->sync_single_for_cpu(to_vmd_dev(dev), addr, size, dir);
+}
+
+static void vmd_sync_single_for_device(struct device *dev, dma_addr_t addr,
+ size_t size, enum dma_data_direction dir)
+{
+ vmd_dma_ops(dev)->sync_single_for_device(to_vmd_dev(dev), addr, size,
+ dir);
+}
+
+static void vmd_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
+ int nents, enum dma_data_direction dir)
+{
+ vmd_dma_ops(dev)->sync_sg_for_cpu(to_vmd_dev(dev), sg, nents, dir);
+}
+
+static void vmd_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
+ int nents, enum dma_data_direction dir)
+{
+ vmd_dma_ops(dev)->sync_sg_for_device(to_vmd_dev(dev), sg, nents, dir);
+}
+
+static int vmd_mapping_error(struct device *dev, dma_addr_t addr)
+{
+ return vmd_dma_ops(dev)->mapping_error(to_vmd_dev(dev), addr);
+}
+
+static int vmd_dma_supported(struct device *dev, u64 mask)
+{
+ return vmd_dma_ops(dev)->dma_supported(to_vmd_dev(dev), mask);
+}
+
+#ifdef ARCH_HAS_DMA_GET_REQUIRED_MASK
+static u64 vmd_get_required_mask(struct device *dev)
+{
+ return vmd_dma_ops(dev)->get_required_mask(to_vmd_dev(dev));
+}
+#endif
+
+static void vmd_teardown_dma_ops(struct vmd_dev *vmd)
+{
+ struct dma_domain *domain = &vmd->dma_domain;
+
+ if (get_dma_ops(&vmd->dev->dev))
+ del_dma_domain(domain);
+}
+
+#define ASSIGN_VMD_DMA_OPS(source, dest, fn) \
+ do { \
+ if (source->fn) \
+ dest->fn = vmd_##fn; \
+ } while (0)
+
+static void vmd_setup_dma_ops(struct vmd_dev *vmd)
+{
+ const struct dma_map_ops *source = get_dma_ops(&vmd->dev->dev);
+ struct dma_map_ops *dest = &vmd->dma_ops;
+ struct dma_domain *domain = &vmd->dma_domain;
+
+ domain->domain_nr = vmd->sysdata.domain;
+ domain->dma_ops = dest;
+
+ if (!source)
+ return;
+ ASSIGN_VMD_DMA_OPS(source, dest, alloc);
+ ASSIGN_VMD_DMA_OPS(source, dest, free);
+ ASSIGN_VMD_DMA_OPS(source, dest, mmap);
+ ASSIGN_VMD_DMA_OPS(source, dest, get_sgtable);
+ ASSIGN_VMD_DMA_OPS(source, dest, map_page);
+ ASSIGN_VMD_DMA_OPS(source, dest, unmap_page);
+ ASSIGN_VMD_DMA_OPS(source, dest, map_sg);
+ ASSIGN_VMD_DMA_OPS(source, dest, unmap_sg);
+ ASSIGN_VMD_DMA_OPS(source, dest, sync_single_for_cpu);
+ ASSIGN_VMD_DMA_OPS(source, dest, sync_single_for_device);
+ ASSIGN_VMD_DMA_OPS(source, dest, sync_sg_for_cpu);
+ ASSIGN_VMD_DMA_OPS(source, dest, sync_sg_for_device);
+ ASSIGN_VMD_DMA_OPS(source, dest, mapping_error);
+ ASSIGN_VMD_DMA_OPS(source, dest, dma_supported);
+#ifdef ARCH_HAS_DMA_GET_REQUIRED_MASK
+ ASSIGN_VMD_DMA_OPS(source, dest, get_required_mask);
+#endif
+ add_dma_domain(domain);
+}
+#undef ASSIGN_VMD_DMA_OPS
+#else
+static void vmd_teardown_dma_ops(struct vmd_dev *vmd) {}
+static void vmd_setup_dma_ops(struct vmd_dev *vmd) {}
+#endif
+
+static char __iomem *vmd_cfg_addr(struct vmd_dev *vmd, struct pci_bus *bus,
+ unsigned int devfn, int reg, int len)
+{
+ char __iomem *addr = vmd->cfgbar +
+ (bus->number << 20) + (devfn << 12) + reg;
+
+ if ((addr - vmd->cfgbar) + len >=
+ resource_size(&vmd->dev->resource[VMD_CFGBAR]))
+ return NULL;
+
+ return addr;
+}
+
+/*
+ * CPU may deadlock if config space is not serialized on some versions of this
+ * hardware, so all config space access is done under a spinlock.
+ */
+static int vmd_pci_read(struct pci_bus *bus, unsigned int devfn, int reg,
+ int len, u32 *value)
+{
+ struct vmd_dev *vmd = vmd_from_bus(bus);
+ char __iomem *addr = vmd_cfg_addr(vmd, bus, devfn, reg, len);
+ unsigned long flags;
+ int ret = 0;
+
+ if (!addr)
+ return -EFAULT;
+
+ spin_lock_irqsave(&vmd->cfg_lock, flags);
+ switch (len) {
+ case 1:
+ *value = readb(addr);
+ break;
+ case 2:
+ *value = readw(addr);
+ break;
+ case 4:
+ *value = readl(addr);
+ break;
+ default:
+ ret = -EINVAL;
+ break;
+ }
+ spin_unlock_irqrestore(&vmd->cfg_lock, flags);
+ return ret;
+}
+
+/*
+ * VMD h/w converts non-posted config writes to posted memory writes. The
+ * read-back in this function forces the completion so it returns only after
+ * the config space was written, as expected.
+ */
+static int vmd_pci_write(struct pci_bus *bus, unsigned int devfn, int reg,
+ int len, u32 value)
+{
+ struct vmd_dev *vmd = vmd_from_bus(bus);
+ char __iomem *addr = vmd_cfg_addr(vmd, bus, devfn, reg, len);
+ unsigned long flags;
+ int ret = 0;
+
+ if (!addr)
+ return -EFAULT;
+
+ spin_lock_irqsave(&vmd->cfg_lock, flags);
+ switch (len) {
+ case 1:
+ writeb(value, addr);
+ readb(addr);
+ break;
+ case 2:
+ writew(value, addr);
+ readw(addr);
+ break;
+ case 4:
+ writel(value, addr);
+ readl(addr);
+ break;
+ default:
+ ret = -EINVAL;
+ break;
+ }
+ spin_unlock_irqrestore(&vmd->cfg_lock, flags);
+ return ret;
+}
+
+static struct pci_ops vmd_ops = {
+ .read = vmd_pci_read,
+ .write = vmd_pci_write,
+};
+
+static void vmd_attach_resources(struct vmd_dev *vmd)
+{
+ vmd->dev->resource[VMD_MEMBAR1].child = &vmd->resources[1];
+ vmd->dev->resource[VMD_MEMBAR2].child = &vmd->resources[2];
+}
+
+static void vmd_detach_resources(struct vmd_dev *vmd)
+{
+ vmd->dev->resource[VMD_MEMBAR1].child = NULL;
+ vmd->dev->resource[VMD_MEMBAR2].child = NULL;
+}
+
+/*
+ * VMD domains start at 0x10000 to not clash with ACPI _SEG domains.
+ * Per ACPI r6.0, sec 6.5.6, _SEG returns an integer, of which the lower
+ * 16 bits are the PCI Segment Group (domain) number. Other bits are
+ * currently reserved.
+ */
+static int vmd_find_free_domain(void)
+{
+ int domain = 0xffff;
+ struct pci_bus *bus = NULL;
+
+ while ((bus = pci_find_next_bus(bus)) != NULL)
+ domain = max_t(int, domain, pci_domain_nr(bus));
+ return domain + 1;
+}
+
+static int vmd_enable_domain(struct vmd_dev *vmd)
+{
+ struct pci_sysdata *sd = &vmd->sysdata;
+ struct fwnode_handle *fn;
+ struct resource *res;
+ u32 upper_bits;
+ unsigned long flags;
+ LIST_HEAD(resources);
+
+ res = &vmd->dev->resource[VMD_CFGBAR];
+ vmd->resources[0] = (struct resource) {
+ .name = "VMD CFGBAR",
+ .start = 0,
+ .end = (resource_size(res) >> 20) - 1,
+ .flags = IORESOURCE_BUS | IORESOURCE_PCI_FIXED,
+ };
+
+ /*
+ * If the window is below 4GB, clear IORESOURCE_MEM_64 so we can
+ * put 32-bit resources in the window.
+ *
+ * There's no hardware reason why a 64-bit window *couldn't*
+ * contain a 32-bit resource, but pbus_size_mem() computes the
+ * bridge window size assuming a 64-bit window will contain no
+ * 32-bit resources. __pci_assign_resource() enforces that
+ * artificial restriction to make sure everything will fit.
+ *
+ * The only way we could use a 64-bit non-prefechable MEMBAR is
+ * if its address is <4GB so that we can convert it to a 32-bit
+ * resource. To be visible to the host OS, all VMD endpoints must
+ * be initially configured by platform BIOS, which includes setting
+ * up these resources. We can assume the device is configured
+ * according to the platform needs.
+ */
+ res = &vmd->dev->resource[VMD_MEMBAR1];
+ upper_bits = upper_32_bits(res->end);
+ flags = res->flags & ~IORESOURCE_SIZEALIGN;
+ if (!upper_bits)
+ flags &= ~IORESOURCE_MEM_64;
+ vmd->resources[1] = (struct resource) {
+ .name = "VMD MEMBAR1",
+ .start = res->start,
+ .end = res->end,
+ .flags = flags,
+ .parent = res,
+ };
+
+ res = &vmd->dev->resource[VMD_MEMBAR2];
+ upper_bits = upper_32_bits(res->end);
+ flags = res->flags & ~IORESOURCE_SIZEALIGN;
+ if (!upper_bits)
+ flags &= ~IORESOURCE_MEM_64;
+ vmd->resources[2] = (struct resource) {
+ .name = "VMD MEMBAR2",
+ .start = res->start + 0x2000,
+ .end = res->end,
+ .flags = flags,
+ .parent = res,
+ };
+
+ sd->vmd_domain = true;
+ sd->domain = vmd_find_free_domain();
+ if (sd->domain < 0)
+ return sd->domain;
+
+ sd->node = pcibus_to_node(vmd->dev->bus);
+
+ fn = irq_domain_alloc_named_id_fwnode("VMD-MSI", vmd->sysdata.domain);
+ if (!fn)
+ return -ENODEV;
+
+ vmd->irq_domain = pci_msi_create_irq_domain(fn, &vmd_msi_domain_info,
+ x86_vector_domain);
+ if (!vmd->irq_domain) {
+ irq_domain_free_fwnode(fn);
+ return -ENODEV;
+ }
+
+ pci_add_resource(&resources, &vmd->resources[0]);
+ pci_add_resource(&resources, &vmd->resources[1]);
+ pci_add_resource(&resources, &vmd->resources[2]);
+ vmd->bus = pci_create_root_bus(&vmd->dev->dev, 0, &vmd_ops, sd,
+ &resources);
+ if (!vmd->bus) {
+ pci_free_resource_list(&resources);
+ irq_domain_remove(vmd->irq_domain);
+ irq_domain_free_fwnode(fn);
+ return -ENODEV;
+ }
+
+ vmd_attach_resources(vmd);
+ vmd_setup_dma_ops(vmd);
+ dev_set_msi_domain(&vmd->bus->dev, vmd->irq_domain);
+ pci_rescan_bus(vmd->bus);
+
+ WARN(sysfs_create_link(&vmd->dev->dev.kobj, &vmd->bus->dev.kobj,
+ "domain"), "Can't create symlink to domain\n");
+ return 0;
+}
+
+static irqreturn_t vmd_irq(int irq, void *data)
+{
+ struct vmd_irq_list *irqs = data;
+ struct vmd_irq *vmdirq;
+ int idx;
+
+ idx = srcu_read_lock(&irqs->srcu);
+ list_for_each_entry_rcu(vmdirq, &irqs->irq_list, node)
+ generic_handle_irq(vmdirq->virq);
+ srcu_read_unlock(&irqs->srcu, idx);
+
+ return IRQ_HANDLED;
+}
+
+static int vmd_probe(struct pci_dev *dev, const struct pci_device_id *id)
+{
+ struct vmd_dev *vmd;
+ int i, err;
+
+ if (resource_size(&dev->resource[VMD_CFGBAR]) < (1 << 20))
+ return -ENOMEM;
+
+ vmd = devm_kzalloc(&dev->dev, sizeof(*vmd), GFP_KERNEL);
+ if (!vmd)
+ return -ENOMEM;
+
+ vmd->dev = dev;
+ err = pcim_enable_device(dev);
+ if (err < 0)
+ return err;
+
+ vmd->cfgbar = pcim_iomap(dev, VMD_CFGBAR, 0);
+ if (!vmd->cfgbar)
+ return -ENOMEM;
+
+ pci_set_master(dev);
+ if (dma_set_mask_and_coherent(&dev->dev, DMA_BIT_MASK(64)) &&
+ dma_set_mask_and_coherent(&dev->dev, DMA_BIT_MASK(32)))
+ return -ENODEV;
+
+ vmd->msix_count = pci_msix_vec_count(dev);
+ if (vmd->msix_count < 0)
+ return -ENODEV;
+
+ vmd->msix_count = pci_alloc_irq_vectors(dev, 1, vmd->msix_count,
+ PCI_IRQ_MSIX);
+ if (vmd->msix_count < 0)
+ return vmd->msix_count;
+
+ vmd->irqs = devm_kcalloc(&dev->dev, vmd->msix_count, sizeof(*vmd->irqs),
+ GFP_KERNEL);
+ if (!vmd->irqs)
+ return -ENOMEM;
+
+ for (i = 0; i < vmd->msix_count; i++) {
+ err = init_srcu_struct(&vmd->irqs[i].srcu);
+ if (err)
+ return err;
+
+ INIT_LIST_HEAD(&vmd->irqs[i].irq_list);
+ err = devm_request_irq(&dev->dev, pci_irq_vector(dev, i),
+ vmd_irq, IRQF_NO_THREAD,
+ "vmd", &vmd->irqs[i]);
+ if (err)
+ return err;
+ }
+
+ spin_lock_init(&vmd->cfg_lock);
+ pci_set_drvdata(dev, vmd);
+ err = vmd_enable_domain(vmd);
+ if (err)
+ return err;
+
+ dev_info(&vmd->dev->dev, "Bound to PCI domain %04x\n",
+ vmd->sysdata.domain);
+ return 0;
+}
+
+static void vmd_cleanup_srcu(struct vmd_dev *vmd)
+{
+ int i;
+
+ for (i = 0; i < vmd->msix_count; i++)
+ cleanup_srcu_struct(&vmd->irqs[i].srcu);
+}
+
+static void vmd_remove(struct pci_dev *dev)
+{
+ struct vmd_dev *vmd = pci_get_drvdata(dev);
+ struct fwnode_handle *fn = vmd->irq_domain->fwnode;
+
+ sysfs_remove_link(&vmd->dev->dev.kobj, "domain");
+ pci_stop_root_bus(vmd->bus);
+ pci_remove_root_bus(vmd->bus);
+ vmd_cleanup_srcu(vmd);
+ vmd_teardown_dma_ops(vmd);
+ vmd_detach_resources(vmd);
+ irq_domain_remove(vmd->irq_domain);
+ irq_domain_free_fwnode(fn);
+}
+
+#ifdef CONFIG_PM_SLEEP
+static int vmd_suspend(struct device *dev)
+{
+ struct pci_dev *pdev = to_pci_dev(dev);
+ struct vmd_dev *vmd = pci_get_drvdata(pdev);
+ int i;
+
+ for (i = 0; i < vmd->msix_count; i++)
+ devm_free_irq(dev, pci_irq_vector(pdev, i), &vmd->irqs[i]);
+
+ pci_save_state(pdev);
+ return 0;
+}
+
+static int vmd_resume(struct device *dev)
+{
+ struct pci_dev *pdev = to_pci_dev(dev);
+ struct vmd_dev *vmd = pci_get_drvdata(pdev);
+ int err, i;
+
+ for (i = 0; i < vmd->msix_count; i++) {
+ err = devm_request_irq(dev, pci_irq_vector(pdev, i),
+ vmd_irq, IRQF_NO_THREAD,
+ "vmd", &vmd->irqs[i]);
+ if (err)
+ return err;
+ }
+
+ pci_restore_state(pdev);
+ return 0;
+}
+#endif
+static SIMPLE_DEV_PM_OPS(vmd_dev_pm_ops, vmd_suspend, vmd_resume);
+
+static const struct pci_device_id vmd_ids[] = {
+ {PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x201d),},
+ {0,}
+};
+MODULE_DEVICE_TABLE(pci, vmd_ids);
+
+static struct pci_driver vmd_drv = {
+ .name = "vmd",
+ .id_table = vmd_ids,
+ .probe = vmd_probe,
+ .remove = vmd_remove,
+ .driver = {
+ .pm = &vmd_dev_pm_ops,
+ },
+};
+module_pci_driver(vmd_drv);
+
+MODULE_AUTHOR("Intel Corporation");
+MODULE_LICENSE("GPL v2");
+MODULE_VERSION("0.6");