marvell/linux/drivers/pci/controller/dwc/pcie-kestrel.c

// SPDX-License-Identifier: GPL-2.0
/*
 * PCIe host controller driver for ASR ASR1901 SoCs
 *
 * SR1901 PCIe Glue Layer Source Code
 *
 * Copyright (C) 2022 ASR Technology Group Ltd.
 *
 */

#include <linux/clk.h>
#include <linux/delay.h>
#include <linux/interrupt.h>
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/of.h>
#include <linux/iopoll.h>
#include <linux/pci.h>
#include <linux/phy/phy.h>
#include <linux/platform_device.h>
#include <linux/resource.h>
#include <linux/of_pci.h>
#include <linux/of_irq.h>
#include <linux/of_gpio.h>
#include <linux/pm_runtime.h>
#include <linux/reset.h>
#include <linux/pm_qos.h>
#include <linux/cputype.h>
#include <soc/asr/regs-addr.h>
#include "pcie-designware.h"

#define DEVICE_NAME "ASR1901 PCIe Host"

#define APMU_PCIE_CLK_RES_CTRL		0x3CC
#define APMU_PCIE_CTRL_LOGIC		0x3D0
#define APMU_PCIE2_CLK_RES_CTRL		0x3E4
#define APMU_PCIE2_CTRL_LOGIC		0x3E8
#define APMU_USB3PHY0_CTRL0		0x3B8

#define HSIO_RC_R_CAL_STATUS		(0x15c)
#define PCIE_CAL_DONE			(0x3<<14)

#define LANE1_OFFSET			0x400
#define	LTSSM_EN			(0x1 << 6)
#define	APP_HOLD_PHY_RST		(0x1 << 30)
#define	DEVICE_TYPE_RC			(0x1 << 31)		/* BIT31 0: EP, 1: RC*/

/* PCIe Config registers */
/* PCIe controller wrapper ASR configuration registers */
#define	PCIE_AHB_IRQ			0x0000
#define	IRQ_EN				0x1

#define	PCIE_AHB_LINK_STS		0x0004
#define DLL_LINK_UP            		(0x1 << 12)
#define PHY_LINK_UP            		(0x1 << 1)
#define LTSSM_L0			(0x11 << 6)
#define LTSSM_STS			(0x3f << 6)

#define PCIE_AHB_LEGACY_INT		0x0008
#define PLL_READY			(0x1)

#define	PCIE_AHB_IRQENABLE_SET_INTX	0x000c
#define	INTA				(0x1 << 6)
#define	INTB				(0x1 << 7)
#define	INTC				(0x1 << 8)
#define	INTD				(0x1 << 9)
#define	LEG_EP_INTERRUPTS (INTA | INTB | INTC | INTD)
#define INTX_MASK			GENMASK(9, 6)
#define INTX_SHIFT			6

#define	PCIE_AHB_IRQSTATUS		0x0010
#define	PCIE_AHB_IRQENABLE_SET		0x0014
#define	MSI_INT				(0x1 << 11)
#define	DLL_LINK_INT			(0x1 << 20)

/* PCIe PHY registers */
#define PUPHY_CLK_CFG			0x8
#define PUPHY_MODE_CFG			0x0c
#define PUPHY_ERROR_STATUS		0x10
#define PUPHY_OVERRIDE			0x18
#define PUPHY_RC_REG			0x44
#define PUPHY_PCIE3X2_REG		0x50
#define PUPHY_PLL_REG1			0x58
#define PUPHY_PLL_REG2			0x5c
#define PUPHY_RX_REG1               	0x60
#define PUPHY_TX_REG                	0x74
#define PUPHY_TEST_REG			0x78
#define PUPHY_TEST_INFO			0x84

#define OVRD_MPU_U3			(0x1 << 17)
#define CFG_MPU_U3			(0x1 << 16)
#define OVRD_PU_RX_LFPS			(0x1 << 15)

#define LINK_WAIT_MIN			900
#define LINK_WAIT_MAX			1000
/* Time for delay */
#define REF_PERST_MIN			20000
#define REF_PERST_MAX			25000
#define PERST_ACCESS_MIN		10000
#define PERST_ACCESS_MAX		12000

#define to_kst_pcie(x)	dev_get_drvdata((x)->dev)

struct kst_pcie {
	struct device *dev;
	struct dw_pcie *pci;
	void __iomem        *phy_base;
	void __iomem        *app_base;
	void __iomem        *usb3_base;
	struct clk			*clk;
	struct phy			*phy[2];
	unsigned int	phy_count;
	unsigned int	slot;
	unsigned int	lanes;
	void __iomem 	*pcie_pmua_reg;
	s32				lpm_qos;
	int				irq;
	int				gpio_reset;
	struct pm_qos_request   qos_idle;
};

extern u32 usb31_rterm_cal_value;

static inline void kst_phy_writel(struct kst_pcie *pcie, u32 val, u32 reg)
{
	writel(val, pcie->phy_base + reg);
}
static inline u32 kst_phy_readl(struct kst_pcie *pcie, u32 reg)
{
	return readl(pcie->phy_base + reg);
}
static inline void kst_app_writel(struct kst_pcie *pcie, u32 val, u32 reg)
{
	writel(val, pcie->app_base + reg);
}
static inline u32 kst_app_readl(struct kst_pcie *pcie, u32 reg)
{
	return readl(pcie->app_base + reg);
}

#ifndef CONFIG_USB_DWC3
static inline void kst_usb30phy_writel(struct kst_pcie *pcie, u32 val, u32 reg)
{
	writel(val, pcie->usb3_base + reg);
}
static inline u32 kst_usb30phy_readl(struct kst_pcie *pcie, u32 reg)
{
	return readl(pcie->usb3_base + reg);
}
int rterm_val, cali_done = 0;
#define USB_CALI_TIMEOUT 50000
static void kst_usb_cali_phy(struct kst_pcie *kst_pcie)
{
	u32 val, timeout = 0;
	if (cali_done == 1)
		return;

	cali_done = 1;
	writel(0x1e00b, regs_addr_get_va(REGS_ADDR_APMU) + APMU_USB3PHY0_CTRL0);

	val = kst_usb30phy_readl(kst_pcie, PUPHY_PLL_REG2);
	kst_usb30phy_writel(kst_pcie, val&(~(0x1<<21)), PUPHY_PLL_REG2);

	val = kst_usb30phy_readl(kst_pcie, PUPHY_PLL_REG1);
	kst_usb30phy_writel(kst_pcie, val&(0xFFFFC0FF), PUPHY_PLL_REG1);

	do {
		val = kst_usb30phy_readl(kst_pcie, PUPHY_TEST_INFO);
		udelay(10);
		timeout++;
		if (timeout > USB_CALI_TIMEOUT)
			break;
	} while (((val>>24)&0x1) == 0);

	val = kst_usb30phy_readl(kst_pcie, PUPHY_TEST_INFO);
	val = kst_usb30phy_readl(kst_pcie, PUPHY_TEST_INFO);
	pr_debug("usb rterm = %08x\r\n", (val>>8) & 0x000000FF);
	writel(0xb, regs_addr_get_va(REGS_ADDR_APMU) + APMU_USB3PHY0_CTRL0);

	rterm_val = val;
}

static void kst2_usb_cali_phy(void)
{
	u32 value;
	int timeout = 0;
	void __iomem *apbs_base = regs_addr_get_va(REGS_ADDR_APBS);

	value = readl(apbs_base + HSIO_RC_R_CAL_STATUS);
	value &= ~0x1;
	writel(value, apbs_base + HSIO_RC_R_CAL_STATUS);

	do {
		value = readl(apbs_base + HSIO_RC_R_CAL_STATUS);
		udelay(10);
		timeout++;
		if (timeout > USB_CALI_TIMEOUT) {
			pr_err("PHY Calibration fail");
			break;
		}
	} while ((value&PCIE_CAL_DONE) != PCIE_CAL_DONE);
}

#endif

static int kst_pcie_init_phy(struct kst_pcie *kst_pcie)
{
	u32 val, data;
	int count = 0;

	/* release pcie reset and enable pcie axi clk */
	__raw_writel(0x8000043f, kst_pcie->pcie_pmua_reg);
	val = __raw_readl(kst_pcie->pcie_pmua_reg);
	val |= DEVICE_TYPE_RC;
	val &= ~APP_HOLD_PHY_RST;
	__raw_writel(val, kst_pcie->pcie_pmua_reg);
	/* enable port0 dbi aclock for port1 only case */
	if (kst_pcie->slot == 1) {
		val = __raw_readl(regs_addr_get_va(REGS_ADDR_APMU)
                       + APMU_PCIE_CLK_RES_CTRL);
		if (!(val&0x9)) {
			val |= (0x80000009);
			__raw_writel(val, regs_addr_get_va(REGS_ADDR_APMU)
				+ APMU_PCIE_CLK_RES_CTRL);
		}
	}

	val = kst_phy_readl(kst_pcie, PUPHY_OVERRIDE);
	val |= (OVRD_MPU_U3 | OVRD_PU_RX_LFPS);
	kst_phy_writel(kst_pcie, val, PUPHY_OVERRIDE);
	if (kst_pcie->lanes == 2) {
		val = kst_phy_readl(kst_pcie, PUPHY_OVERRIDE + LANE1_OFFSET);
		val |= (OVRD_MPU_U3 | OVRD_PU_RX_LFPS);
		kst_phy_writel(kst_pcie, val, PUPHY_OVERRIDE + LANE1_OFFSET);
	}

	val = kst_phy_readl(kst_pcie, PUPHY_RC_REG);
	val = (val & 0xFFFF0000) | (0x1387);
	kst_phy_writel(kst_pcie, val, PUPHY_RC_REG);

	if (cpu_is_asr1901_a0_plus()) {
#ifndef CONFIG_USB_DWC3
        kst_usb_cali_phy(kst_pcie);
        val = rterm_val;
#else
        val = usb31_rterm_cal_value;
#endif
        data = kst_phy_readl(kst_pcie, PUPHY_RC_REG);
        data = (data & 0xffffff00) | ((val>>8) & 0xFF);
        kst_phy_writel(kst_pcie, data, PUPHY_RC_REG);
        pr_debug("pcie rterm = %08x\r\n", kst_phy_readl(kst_pcie, PUPHY_RC_REG));
    }
#ifndef CONFIG_USB_DWC3
    if (cpu_is_asr1906()) {
        kst2_usb_cali_phy();
    }
#endif
	val = kst_phy_readl(kst_pcie, PUPHY_TEST_REG);
	val |= (0x1<<1);
	kst_phy_writel(kst_pcie, val, PUPHY_TEST_REG);
	if (kst_pcie->lanes == 2) {
		val = kst_phy_readl(kst_pcie, PUPHY_TEST_REG + LANE1_OFFSET);
		val |= (0x1<<1);
		kst_phy_writel(kst_pcie, val, PUPHY_TEST_REG + LANE1_OFFSET);
	}

	val = kst_phy_readl(kst_pcie, PUPHY_OVERRIDE);
	val &= ~(OVRD_MPU_U3 | OVRD_PU_RX_LFPS);
	kst_phy_writel(kst_pcie, val, PUPHY_OVERRIDE);
	if (kst_pcie->lanes == 2) {
		val = kst_phy_readl(kst_pcie, PUPHY_OVERRIDE + LANE1_OFFSET);
		val &= ~(OVRD_MPU_U3 | OVRD_PU_RX_LFPS);
		kst_phy_writel(kst_pcie, val, PUPHY_OVERRIDE + LANE1_OFFSET);
	}

	val = kst_phy_readl(kst_pcie, PUPHY_PLL_REG1);
	kst_phy_writel(kst_pcie, val&0xffff0fff, PUPHY_PLL_REG1);
	val = kst_phy_readl(kst_pcie, PUPHY_PLL_REG1);
	kst_phy_writel(kst_pcie, val|0xC000, PUPHY_PLL_REG1);
	val = kst_phy_readl(kst_pcie, PUPHY_PLL_REG2);
	kst_phy_writel(kst_pcie, val|(0x1<<20), PUPHY_PLL_REG2);
	val = kst_phy_readl(kst_pcie, PUPHY_PLL_REG2);
	kst_phy_writel(kst_pcie, val&(~(0x1<<21)), PUPHY_PLL_REG2);
	kst_phy_writel(kst_pcie, 0x6505, PUPHY_PCIE3X2_REG);

	val = kst_phy_readl(kst_pcie, PUPHY_PLL_REG1);
	kst_phy_writel(kst_pcie, val&0xf0ffffff, PUPHY_PLL_REG1);
	val = kst_phy_readl(kst_pcie, PUPHY_CLK_CFG);
	kst_phy_writel(kst_pcie, 0xB7c, PUPHY_CLK_CFG);

	if (kst_pcie->lanes == 1) {
		pr_debug("PCIE only one Lane, disable Lane1...\r\n");
		val = kst_phy_readl(kst_pcie, PUPHY_MODE_CFG + LANE1_OFFSET);
		kst_phy_writel(kst_pcie, val|(0x1<<30), PUPHY_MODE_CFG + LANE1_OFFSET);
	} else {
		kst_phy_writel(kst_pcie, 0xB7c, PUPHY_CLK_CFG + LANE1_OFFSET);
	}

	val = kst_phy_readl(kst_pcie, PUPHY_MODE_CFG);
	val |= (0x1<<2);
	kst_phy_writel(kst_pcie, val, PUPHY_MODE_CFG);
	if (kst_pcie->lanes == 2) {
		val = kst_phy_readl(kst_pcie, PUPHY_MODE_CFG + LANE1_OFFSET);
		val |= (0x1<<2);
		kst_phy_writel(kst_pcie, val, PUPHY_MODE_CFG + LANE1_OFFSET);
	}

	do {
		val = kst_phy_readl(kst_pcie, PUPHY_CLK_CFG);
		count++;
		usleep_range(LINK_WAIT_MIN, LINK_WAIT_MAX);
		if (count == 100) {
			pr_info(DEVICE_NAME "PCIe wait pll ready timeout.\n");
			return -EINVAL;
		}
	}while(( val & PLL_READY ) != PLL_READY);

	return 0;
}

static int kst_pcie_disable_phy(struct kst_pcie *kst_pcie)
{
	u32 val;

	val = kst_phy_readl(kst_pcie, PUPHY_OVERRIDE);
	val |= OVRD_MPU_U3;
	val &= ~CFG_MPU_U3;
	kst_phy_writel(kst_pcie, val, PUPHY_OVERRIDE);

	val = kst_phy_readl(kst_pcie, PUPHY_OVERRIDE + LANE1_OFFSET);
	val |= OVRD_MPU_U3;
	val &= ~CFG_MPU_U3;

	kst_phy_writel(kst_pcie, val, PUPHY_OVERRIDE + LANE1_OFFSET);

	return 0;
}

static int kst_pcie_reenable_phy(struct kst_pcie *kst_pcie)
{
	u32 val;

	val = kst_phy_readl(kst_pcie, PUPHY_OVERRIDE);
	val &= ~(OVRD_MPU_U3 | OVRD_PU_RX_LFPS);
	kst_phy_writel(kst_pcie, val, PUPHY_OVERRIDE);
	if (kst_pcie->lanes == 2) {
		val = kst_phy_readl(kst_pcie, PUPHY_OVERRIDE + LANE1_OFFSET);
		val &= ~(OVRD_MPU_U3 | OVRD_PU_RX_LFPS);
		kst_phy_writel(kst_pcie, val, PUPHY_OVERRIDE + LANE1_OFFSET);
	}

	return 0;
}

static int kst_pcie_link_up(struct dw_pcie *pci)
{
	struct kst_pcie *kst_pcie = to_kst_pcie(pci);
	u32 status = kst_app_readl(kst_pcie, PCIE_AHB_LINK_STS);

	if ((status & DLL_LINK_UP) && (status & PHY_LINK_UP))
		return 1;

	return 0;
}

static int kst_pcie_establish_link(struct pcie_port *pp)
{
	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
	struct kst_pcie *kst_pcie = to_kst_pcie(pci);
	struct device *dev = kst_pcie->pci->dev;
	unsigned int val, count = 0;

	if (kst_pcie_link_up(pci))
		return 0;

	val = __raw_readl(kst_pcie->pcie_pmua_reg);
	__raw_writel(val | DEVICE_TYPE_RC, kst_pcie->pcie_pmua_reg);

	dw_pcie_setup_rc(pp);
#if 0
	val = readl(pci->dbi_base + 0x8a8);
	val &= ~(0xffff<<8);
	val |= (0x10<<8);				//set TX preset P4
	writel(val, pci->dbi_base + 0x8a8);
#endif
	/* Release app_hold_phy_reset and enable ltssm */
	val = __raw_readl(kst_pcie->pcie_pmua_reg);
	val |= LTSSM_EN;
	val &= ~APP_HOLD_PHY_RST;
	__raw_writel(val, kst_pcie->pcie_pmua_reg);

	udelay(10);
	val = readl(pci->dbi_base + 0x80c);
	val |= 1<<17;
	writel(val, pci->dbi_base + 0x80c);
	udelay(10);

	do {
		usleep_range(LINK_WAIT_MIN, LINK_WAIT_MAX);
		val = kst_app_readl(kst_pcie, PCIE_AHB_LINK_STS);
		pr_debug("%s, ltssm: 0x%x.\n", __func__, val);
		count++;
		if (count == 1000) {
			pr_info(DEVICE_NAME "PCIe enter L0 failed, ltssm: 0x%x\n", val);
			return -EINVAL;
		}
	}while(( val & LTSSM_STS ) != LTSSM_L0 );

	count = 0;
	/* check if the link is up or not */
	while (!kst_pcie_link_up(pci)) {
		usleep_range(LINK_WAIT_MIN, LINK_WAIT_MAX);
		count++;
		if (count == 100) {
			dev_err(dev, "Link Fail\n");
			return -EINVAL;
		}
	}

	val = readl(pci->dbi_base + 0x80);
	pr_info(DEVICE_NAME " %dx link negotiated (gen %d)\n",
				(val>>20)&0xf, (val>>16)&0xf);

	return 0;
}

static int kst_pcie_host_init(struct pcie_port *pp)
{
	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
	struct kst_pcie *kst_pcie = to_kst_pcie(pci);
	int ret;

	ret = kst_pcie_establish_link(pp);
	if(ret) {
		__raw_writel(0x0, kst_pcie->pcie_pmua_reg);
		return -1;
	}
	return 0;
}
static const struct dw_pcie_host_ops kst_pcie_host_ops = {
	.host_init = kst_pcie_host_init,
};

#ifndef CONFIG_PCI_MSI
static irqreturn_t kst_pcie_irq_handler(int irq, void *arg)
{
	struct kst_pcie *kst_pcie = arg;
	u32 status;

	pm_wakeup_event(kst_pcie->dev, 2000);
	status = kst_app_readl(kst_pcie, PCIE_AHB_LEGACY_INT);
	kst_app_writel(kst_pcie, status, PCIE_AHB_LEGACY_INT);

	return IRQ_HANDLED;
}
#endif
#ifdef CONFIG_PCI_MSI
static int kst_pcie_add_msi(struct dw_pcie *pci,
				struct platform_device *pdev)
{
	int irq;

	if (IS_ENABLED(CONFIG_PCI_MSI)) {
		irq = platform_get_irq(pdev, 0);
		if (irq < 0) {
			dev_err(&pdev->dev,
				"failed to get MSI IRQ (%d)\n", irq);
			return irq;
		}
		pci->pp.msi_irq = irq;
	}

	return 0;
}
#endif

static void kst_pcie_enable_interrupts(struct pcie_port *pp)
{
	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
	struct kst_pcie *kst_pcie = to_kst_pcie(pci);
	u32 val;

#ifdef CONFIG_PCI_MSI
	dw_pcie_msi_init(pp);

	val = kst_app_readl(kst_pcie, PCIE_AHB_IRQENABLE_SET);
	val |= MSI_INT;
	kst_app_writel(kst_pcie, val, PCIE_AHB_IRQENABLE_SET);
#else	/* legacy interrupt */
	val = kst_app_readl(kst_pcie, PCIE_AHB_IRQENABLE_SET_INTX);
	val |= LEG_EP_INTERRUPTS;
	kst_app_writel(kst_pcie, val, PCIE_AHB_IRQENABLE_SET_INTX);
#endif
	val = kst_app_readl(kst_pcie, PCIE_AHB_IRQ);
	val |= IRQ_EN;
	kst_app_writel(kst_pcie, val, PCIE_AHB_IRQ);

	return;
}

static int kst_add_pcie_port(struct kst_pcie *pcie,
				  struct platform_device *pdev)
{
	struct dw_pcie *pci = pcie->pci;
	struct pcie_port *pp = &pci->pp;
	struct device *dev = &pdev->dev;
	int ret;

	pp->ops = &kst_pcie_host_ops;
#ifdef CONFIG_PCI_MSI
	ret = kst_pcie_add_msi(pci, pdev);
	if (ret)
		return ret;
#else
	pp->irq = platform_get_irq(pdev, 0);
	if (pp->irq < 0) {
		dev_err(dev, "failed to get irq for port\n");
		return pp->irq;
	}
	ret = devm_request_irq(dev, pp->irq, kst_pcie_irq_handler,
			       IRQF_SHARED, "kst-pcie", pcie);
	if (ret) {
		dev_err(dev, "failed to request irq %d\n", pp->irq);
		return ret;
	}
#endif

	ret = dw_pcie_host_init(pp);
	if (ret) {
		dev_err(dev, "failed to initialize host: %d\n", ret);
		return ret;
	}
	kst_pcie_enable_interrupts(pp);

	return 0;
}

static const struct dw_pcie_ops dw_pcie_ops = {
	.link_up = kst_pcie_link_up,
};

static long kst_pcie_get_resource(struct kst_pcie *kst_pcie,
				    struct platform_device *pdev)
{
	struct device *dev = &pdev->dev;
	struct resource *phy;
	struct resource *dbi;
	struct device_node *np = pdev->dev.of_node;
	const __be32 *prop;
	unsigned int proplen;

	phy = platform_get_resource_byname(pdev, IORESOURCE_MEM, "pcie-phy");
	kst_pcie->phy_base = devm_ioremap_resource(dev, phy);
	if (IS_ERR(kst_pcie->phy_base))
		return PTR_ERR(kst_pcie->phy_base);
	kst_pcie->app_base = kst_pcie->phy_base + 0x10000;

	dbi = platform_get_resource_byname(pdev, IORESOURCE_MEM, "pcie-dbi");
	kst_pcie->pci->dbi_base = devm_ioremap_resource(dev, dbi);
	if (IS_ERR(kst_pcie->pci->dbi_base))
		return PTR_ERR(kst_pcie->pci->dbi_base);
#ifndef CONFIG_USB_DWC3
	if (cali_done == 0) {	//for 2 pcie only remap 1 usb3-phy address
		dbi = platform_get_resource_byname(pdev, IORESOURCE_MEM, "usb3-phy");
		kst_pcie->usb3_base = devm_ioremap_resource(dev, dbi);
		if (IS_ERR(kst_pcie->usb3_base))
			return PTR_ERR(kst_pcie->usb3_base);
	}
#endif
	if (of_property_read_u32(np, "num-lanes", &(kst_pcie->lanes))) {
		pr_err("Failed to parse the PCIE0 or PCIE1 lane number\n");
		return -EINVAL;
	}

	prop = of_get_property(np, "lpm-qos", &proplen);
	if (!prop) {
		pr_err("lpm-qos config in DT for PCIe is not defined\n");
		return -EINVAL;
	} else
		kst_pcie->lpm_qos = be32_to_cpup(prop);

	if (of_property_read_u32(np, "num-slot", &(kst_pcie->slot))) {
		pr_err("Failed to parse the PCIE0 or PCIE1\n");
		return -EINVAL;
	}
	kst_pcie->gpio_reset = of_get_named_gpio(np, "reset-gpios", 0);
	if (kst_pcie->gpio_reset < 0)
		return -ENODEV;

	if (kst_pcie->slot == 0)
		kst_pcie->pcie_pmua_reg = regs_addr_get_va(REGS_ADDR_APMU)
					+ APMU_PCIE_CLK_RES_CTRL;
	else if (kst_pcie->slot == 1)
		kst_pcie->pcie_pmua_reg = regs_addr_get_va(REGS_ADDR_APMU)
					+ APMU_PCIE2_CLK_RES_CTRL;

	return 0;
}

static int kst_pcie_probe(struct platform_device *pdev)
{
	struct dw_pcie *pci;
	struct kst_pcie *pcie;
	struct device *dev = &pdev->dev;
	int ret;

	pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL);
	if (!pcie)
		return -ENOMEM;

	pci = devm_kzalloc(dev, sizeof(*pci), GFP_KERNEL);
	if (!pci)
		return -ENOMEM;

	pci->dev = dev;
	pci->ops = &dw_pcie_ops;
	pcie->pci = pci;

	kst_pcie_get_resource(pcie, pdev);

	pcie->clk = devm_clk_get(dev, NULL);
	if (IS_ERR(pcie->clk))
		return PTR_ERR(pcie->clk);

	ret = clk_prepare_enable(pcie->clk);
	if (ret)
		return ret;

	ret = kst_pcie_init_phy(pcie);
	if (ret)
		goto fail_clk;

	/* perst assert Endpoint */
	if (!gpio_request(pcie->gpio_reset, "pcie_perst")) {
		usleep_range(REF_PERST_MIN, REF_PERST_MAX);
		ret = gpio_direction_output(pcie->gpio_reset, 1);
		if (ret) {
			pr_info(DEVICE_NAME "PCIE reset device failed.\r\n");
			goto disable_phy;
		}
		gpio_free(pcie->gpio_reset);
		usleep_range(PERST_ACCESS_MIN, PERST_ACCESS_MAX);
	}
	platform_set_drvdata(pdev, pcie);

	ret = kst_add_pcie_port(pcie, pdev);
	if (ret)
		goto disable_phy;

	device_init_wakeup(&pdev->dev, 1);
	pm_qos_add_request(&pcie->qos_idle, PM_QOS_CPUIDLE_BLOCK,
			PM_QOS_CPUIDLE_BLOCK_DEFAULT_VALUE);
	pcie->qos_idle.name = pdev->name;
	pm_qos_update_request(&pcie->qos_idle, pcie->lpm_qos);

	return 0;

disable_phy:
	kst_pcie_disable_phy(pcie);
fail_clk:
	clk_disable_unprepare(pcie->clk);

	return ret;
}
#ifdef CONFIG_PM_SLEEP
static int __maybe_unused kst_pcie_suspend_noirq(struct device *dev)
{
	struct kst_pcie *pcie = dev_get_drvdata(dev);

	kst_pcie_disable_phy(pcie);
	pm_qos_update_request(&pcie->qos_idle,
			PM_QOS_CPUIDLE_BLOCK_DEFAULT_VALUE);

	return 0;
}

static int __maybe_unused kst_pcie_resume_noirq(struct device *dev)
{
	struct kst_pcie *pcie = dev_get_drvdata(dev);

	kst_pcie_reenable_phy(pcie);
	pm_qos_update_request(&pcie->qos_idle, pcie->lpm_qos);

	return 0;
}

static const struct dev_pm_ops kst_pcie_pm_ops = {
	SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(kst_pcie_suspend_noirq,
				      kst_pcie_resume_noirq)
};
#endif

static const struct of_device_id kst_pcie_of_match[] = {
	{ .compatible = "asr,kst-pcie", },
	{},
};

static struct platform_driver kst_pcie_driver = {
	.probe		= kst_pcie_probe,
	.driver = {
		.name	= "kst-pcie",
		.of_match_table = of_match_ptr(kst_pcie_of_match),
		.suppress_bind_attrs = true,
#ifdef CONFIG_PM_SLEEP
		.pm = &kst_pcie_pm_ops,
#endif
	},
};

static int __init kst_pcie_init(void)
{
	return platform_driver_probe(&kst_pcie_driver, kst_pcie_probe);
}
device_initcall_sync(kst_pcie_init);