diff mbox

[v7,01/10] PCI: host: rcar: Add Renesas R-Car PCIe driver

Message ID 1396261856-22465-2-git-send-email-phil.edworthy@renesas.com
State Changes Requested
Headers show

Commit Message

Phil Edworthy March 31, 2014, 10:30 a.m. UTC 
This PCIe Host driver currently does not support MSI, so cards
fall back to INTx interrupts.

Signed-off-by: Phil Edworthy <phil.edworthy@renesas.com>

v5:
 - Use module_platform_driver instead of subsys_initcall
 - Use the of_device_id data field for HW init function
 - Init hw_pci struct in declaration
 - Renesas SoC compatible string has peripheral before device name
 - Add PCIe bus clock reference
 - Use dma-ranges property to specify inbound memory regions
 - Support multiple IO windows and correct resources

v4:
 - Use runtime PM properly

v3:
 - Add DT support
 - Use 'of_irq_parse_and_map_pci' for '.map_irq'
 - Use pm ops to enable clocks
 - Fix checkpatch errors
 - Use subsys_initcall to overcome issues with port bus driver
 - Adjust Kconfig to match other R-Car drivers

v2:
 - Use msleep instead of udelay when waiting for the link
 - Use pm_runtime
 - Removed unused definition
 - Also replaced call to devm_request_and_ioremap with devm_ioremap_resource
   and fixed a bug with this when reporting errors.
---
 drivers/pci/host/Kconfig     |   6 +
 drivers/pci/host/Makefile    |   1 +
 drivers/pci/host/pcie-rcar.c | 693 +++++++++++++++++++++++++++++++++++++++++++
 drivers/pci/host/pcie-rcar.h |  82 +++++
 4 files changed, 782 insertions(+)
 create mode 100644 drivers/pci/host/pcie-rcar.c
 create mode 100644 drivers/pci/host/pcie-rcar.h

Comments

Bjorn Helgaas April 24, 2014, 7:19 p.m. UTC | #1 
On Mon, Mar 31, 2014 at 11:30:46AM +0100, Phil Edworthy wrote:
> This PCIe Host driver currently does not support MSI, so cards
> fall back to INTx interrupts.
> 
> Signed-off-by: Phil Edworthy <phil.edworthy@renesas.com>
> ...

> --- /dev/null
> +++ b/drivers/pci/host/pcie-rcar.c
> @@ -0,0 +1,693 @@
> ...
> +#include <linux/slab.h>
> +#include "pcie-rcar.h"

It looks like the things in pcie-rcar.h are used only in this file
(pcie-rcar.c), so you might as well just put the contents here and not have
a pcie-rcar.h at all.  Of course, if there are things needed by more than
one file, they should stay in pcie-rcar.h.

> +#define DRV_NAME "rcar-pcie"
> +
> +#define RCONF(x)	(PCICONF(0)+(x))
> +#define RPMCAP(x)	(PMCAP(0)+(x))
> +#define REXPCAP(x)	(EXPCAP(0)+(x))
> +#define RVCCAP(x)	(VCCAP(0)+(x))
> +
> +#define  PCIE_CONF_BUS(b)	(((b) & 0xff) << 24)
> +#define  PCIE_CONF_DEV(d)	(((d) & 0x1f) << 19)
> +#define  PCIE_CONF_FUNC(f)	(((f) & 0x7) << 16)
> +
> +#define PCI_MAX_RESOURCES 4
> +#define MAX_NR_INBOUND_MAPS 6
> +
> +/* Structure representing the PCIe interface */
> +struct rcar_pcie {
> +	struct device		*dev;
> +	void __iomem		*base;
> +	struct resource		res[PCI_MAX_RESOURCES];
> +	u8			root_bus_nr;

I don't understand how root_bus_nr works.  From its name, it sounds like
it's the bus number of the PCI bus on the downstream side of the host
bridge.  That bus number should be a property of the host bridge, i.e.,
it's either hard-wired into the bridge, or it's programmable somewhere.
But root_bus_nr comes from sys->busnr, which is apparently from some
generic code that knows nothing about this particular host bridge.

> +	struct clk		*clk;
> +	struct clk		*bus_clk;
> +};
> +
> +static inline struct rcar_pcie *sys_to_pcie(struct pci_sys_data *sys)
> +{
> +	return sys->private_data;
> +}
> +
> +static void
> +pci_write_reg(struct rcar_pcie *pcie, unsigned long val, unsigned long reg)

Use this indentation style:

    static void pci_write_reg(struct rcar_pcie *pcie, unsigned long val,

and wrap the args as necessary (as you already did with
rcar_pcie_read_conf() below).

> +{
> +	writel(val, pcie->base + reg);
> +}
> +
> +static unsigned long
> +pci_read_reg(struct rcar_pcie *pcie, unsigned long reg)
> +{
> +	return readl(pcie->base + reg);
> +}
> +
> +enum {
> +	PCI_ACCESS_READ,
> +	PCI_ACCESS_WRITE,
> +};
> +
> +static void rcar_rmw32(struct rcar_pcie *pcie,
> +			    int where, u32 mask, u32 data)

No wrapping necessary here.

> +{
> +	int shift = 8 * (where & 3);
> +	u32 val = pci_read_reg(pcie, where & ~3);

A blank line is typical here (after the local variables).

> +	val &= ~(mask << shift);
> +	val |= data << shift;
> +	pci_write_reg(pcie, val, where & ~3);
> +}
> +
> +static u32 rcar_read_conf(struct rcar_pcie *pcie, int where)
> +{
> +	int shift = 8 * (where & 3);
> +	u32 val = pci_read_reg(pcie, where & ~3);

And here.

> +	return val >> shift;
> +}
> +
> +static int rcar_pcie_config_access(struct rcar_pcie *pcie,
> +		unsigned char access_type, struct pci_bus *bus,
> +		unsigned int devfn, int where, u32 *data)
> +{
> +	int dev, func, reg, index;
> +
> +	dev = PCI_SLOT(devfn);
> +	func = PCI_FUNC(devfn);
> +	reg = where & ~3;
> +	index = reg / 4;
> +
> +	if (bus->number > 255 || dev > 31 || func > 7)
> +		return PCIBIOS_FUNC_NOT_SUPPORTED;

I do see one other place in the tree (sh7786_pcie_config_access()) where we
test all these, but I don't think it's really necessary.  It's actually
impossible to satisfy this condition, given the definitions of bus->number,
PCI_SLOT(), and PCI_FUNC().

> ...
> +	/* Clear errors */
> +	pci_write_reg(pcie, pci_read_reg(pcie, PCIEERRFR), PCIEERRFR);
> +
> +	/* Set the PIO address */
> +	pci_write_reg(pcie, PCIE_CONF_BUS(bus->number) | PCIE_CONF_DEV(dev) |
> +				PCIE_CONF_FUNC(func) | reg, PCIECAR);
> +
> +	/* Enable the configuration access */
> +	if (bus->parent->number == pcie->root_bus_nr)
> +		pci_write_reg(pcie, CONFIG_SEND_ENABLE | TYPE0, PCIECCTLR);
> +	else
> +		pci_write_reg(pcie, CONFIG_SEND_ENABLE | TYPE1, PCIECCTLR);
> +
> +	/* Check for errors */
> +	if (pci_read_reg(pcie, PCIEERRFR) & UNSUPPORTED_REQUEST)
> +		return PCIBIOS_DEVICE_NOT_FOUND;
> +
> +	/* Check for master and target aborts */
> +	if (rcar_read_conf(pcie, RCONF(PCI_STATUS)) &
> +		(PCI_STATUS_REC_MASTER_ABORT | PCI_STATUS_REC_TARGET_ABORT))
> +		return PCIBIOS_DEVICE_NOT_FOUND;
> +
> +	if (access_type == PCI_ACCESS_READ)
> +		*data = pci_read_reg(pcie, PCIECDR);
> +	else
> +		pci_write_reg(pcie, *data, PCIECDR);
> +
> +	/* Disable the configuration access */
> +	pci_write_reg(pcie, 0, PCIECCTLR);

It looks like the above all needs to be atomic, so I suppose you're relying
on higher-level locking to ensure that.  It might be worth a mention of the
lock you rely on, just in case that ever changes (I doubt it would, but
it's conceivable).

> ...
> +static int rcar_pcie_setup_window(int win, struct resource *res,
> +					struct rcar_pcie *pcie)
> +{
> +	/* Setup PCIe address space mappings for each resource */
> +	resource_size_t size;
> +	u32 mask;
> +
> +	pci_write_reg(pcie, 0x00000000, PCIEPTCTLR(win));
> +
> +	/*
> +	 * The PAMR mask is calculated in units of 128Bytes, which
> +	 * keeps things pretty simple.
> +	 */
> +	size = resource_size(res);
> +	mask = (roundup_pow_of_two(size) / SZ_128) - 1;
> +	pci_write_reg(pcie, mask << 7, PCIEPAMR(win));
> +
> +	pci_write_reg(pcie, upper_32_bits(res->start), PCIEPARH(win));
> +	pci_write_reg(pcie, lower_32_bits(res->start), PCIEPARL(win));
> +
> +	/* First resource is for IO */
> +	mask = PAR_ENABLE;
> +	if (res->flags & IORESOURCE_IO)
> +		mask |= IO_SPACE;
> +
> +	pci_write_reg(pcie, mask, PCIEPTCTLR(win));
> +
> +	return 0;

Maybe could be a void function, since no error is possible?

> ...
> +static int __init rcar_pcie_get_resources(struct platform_device *pdev,
> +	struct rcar_pcie *pcie)
> +{
> +	struct resource res;
> +	int err;
> +
> +	err = of_address_to_resource(pdev->dev.of_node, 0, &res);
> +	if (err)
> +		return err;

I don't see anywhere that figures out the bus number range behind this host
bridge.  The PCI core needs to know this.  I know the core assumes 00-ff if
it's not supplied, but that's just to deal with the legacy situation where
we assumed one host bridge was all anybody would ever need.

For new code, we should be explicit about the range.

Bjorn
--
To unsubscribe from this list: send the line "unsubscribe linux-pci" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Phil Edworthy April 28, 2014, 10:03 a.m. UTC | #2 
Hi Bjorn,

Thanks for the review.

On 24 April 2014 20:19, Bjorn wrote:
> On Mon, Mar 31, 2014 at 11:30:46AM +0100, Phil Edworthy wrote:
> > This PCIe Host driver currently does not support MSI, so cards
> > fall back to INTx interrupts.
> >
> > Signed-off-by: Phil Edworthy <phil.edworthy@renesas.com>
> > ...
> 
> > --- /dev/null
> > +++ b/drivers/pci/host/pcie-rcar.c
> > @@ -0,0 +1,693 @@
> > ...
> > +#include <linux/slab.h>
> > +#include "pcie-rcar.h"
> 
> It looks like the things in pcie-rcar.h are used only in this file
> (pcie-rcar.c), so you might as well just put the contents here and not have
> a pcie-rcar.h at all.  Of course, if there are things needed by more than
> one file, they should stay in pcie-rcar.h.
Nothing else uses them so I'll put in the c file.

> > +#define DRV_NAME "rcar-pcie"
> > +
> > +#define RCONF(x)	(PCICONF(0)+(x))
> > +#define RPMCAP(x)	(PMCAP(0)+(x))
> > +#define REXPCAP(x)	(EXPCAP(0)+(x))
> > +#define RVCCAP(x)	(VCCAP(0)+(x))
> > +
> > +#define  PCIE_CONF_BUS(b)	(((b) & 0xff) << 24)
> > +#define  PCIE_CONF_DEV(d)	(((d) & 0x1f) << 19)
> > +#define  PCIE_CONF_FUNC(f)	(((f) & 0x7) << 16)
> > +
> > +#define PCI_MAX_RESOURCES 4
> > +#define MAX_NR_INBOUND_MAPS 6
> > +
> > +/* Structure representing the PCIe interface */
> > +struct rcar_pcie {
> > +	struct device		*dev;
> > +	void __iomem		*base;
> > +	struct resource		res[PCI_MAX_RESOURCES];
> > +	u8			root_bus_nr;
> 
> I don't understand how root_bus_nr works.  From its name, it sounds like
> it's the bus number of the PCI bus on the downstream side of the host
> bridge.
That's correct.

> That bus number should be a property of the host bridge, i.e.,
> it's either hard-wired into the bridge, or it's programmable somewhere.
> But root_bus_nr comes from sys->busnr, which is apparently from some
> generic code that knows nothing about this particular host bridge.
The manual for this hardware says that the HW doesn't care what the bus number
is set to. The only thing the HW needs to know is that when sending config
accesses, we need to indicate whether it's a TYPE0 or TYPE1 header; so we use
root_bus_nr to determine this. The generic code that sets up sys->busnr (ARM
bios32 in this case), just increments bus number for each controller.

This is very similar to the pcie-designware driver, in dw_pcie_scan_bus().

> > +	struct clk		*clk;
> > +	struct clk		*bus_clk;
> > +};
> > +
> > +static inline struct rcar_pcie *sys_to_pcie(struct pci_sys_data *sys)
> > +{
> > +	return sys->private_data;
> > +}
> > +
> > +static void
> > +pci_write_reg(struct rcar_pcie *pcie, unsigned long val, unsigned long
> reg)
> 
> Use this indentation style:
> 
>     static void pci_write_reg(struct rcar_pcie *pcie, unsigned long val,
> 
> and wrap the args as necessary (as you already did with
> rcar_pcie_read_conf() below).
Ok
 
> > +{
> > +	writel(val, pcie->base + reg);
> > +}
> > +
> > +static unsigned long
> > +pci_read_reg(struct rcar_pcie *pcie, unsigned long reg)
> > +{
> > +	return readl(pcie->base + reg);
> > +}
> > +
> > +enum {
> > +	PCI_ACCESS_READ,
> > +	PCI_ACCESS_WRITE,
> > +};
> > +
> > +static void rcar_rmw32(struct rcar_pcie *pcie,
> > +			    int where, u32 mask, u32 data)
> 
> No wrapping necessary here.
Sure
 
> > +{
> > +	int shift = 8 * (where & 3);
> > +	u32 val = pci_read_reg(pcie, where & ~3);
> 
> A blank line is typical here (after the local variables).
Ok

> > +	val &= ~(mask << shift);
> > +	val |= data << shift;
> > +	pci_write_reg(pcie, val, where & ~3);
> > +}
> > +
> > +static u32 rcar_read_conf(struct rcar_pcie *pcie, int where)
> > +{
> > +	int shift = 8 * (where & 3);
> > +	u32 val = pci_read_reg(pcie, where & ~3);
> 
> And here.
Ok
 
> > +	return val >> shift;
> > +}
> > +
> > +static int rcar_pcie_config_access(struct rcar_pcie *pcie,
> > +		unsigned char access_type, struct pci_bus *bus,
> > +		unsigned int devfn, int where, u32 *data)
> > +{
> > +	int dev, func, reg, index;
> > +
> > +	dev = PCI_SLOT(devfn);
> > +	func = PCI_FUNC(devfn);
> > +	reg = where & ~3;
> > +	index = reg / 4;
> > +
> > +	if (bus->number > 255 || dev > 31 || func > 7)
> > +		return PCIBIOS_FUNC_NOT_SUPPORTED;
> 
> I do see one other place in the tree (sh7786_pcie_config_access()) where we
> test all these, but I don't think it's really necessary.  It's actually
> impossible to satisfy this condition, given the definitions of bus->number,
> PCI_SLOT(), and PCI_FUNC().
Ok, I'll remove them. They were simply copied over from the sh7786 driver.

> > ...
> > +	/* Clear errors */
> > +	pci_write_reg(pcie, pci_read_reg(pcie, PCIEERRFR), PCIEERRFR);
> > +
> > +	/* Set the PIO address */
> > +	pci_write_reg(pcie, PCIE_CONF_BUS(bus->number) |
> PCIE_CONF_DEV(dev) |
> > +				PCIE_CONF_FUNC(func) | reg, PCIECAR);
> > +
> > +	/* Enable the configuration access */
> > +	if (bus->parent->number == pcie->root_bus_nr)
> > +		pci_write_reg(pcie, CONFIG_SEND_ENABLE | TYPE0,
> PCIECCTLR);
> > +	else
> > +		pci_write_reg(pcie, CONFIG_SEND_ENABLE | TYPE1,
> PCIECCTLR);
> > +
> > +	/* Check for errors */
> > +	if (pci_read_reg(pcie, PCIEERRFR) & UNSUPPORTED_REQUEST)
> > +		return PCIBIOS_DEVICE_NOT_FOUND;
> > +
> > +	/* Check for master and target aborts */
> > +	if (rcar_read_conf(pcie, RCONF(PCI_STATUS)) &
> > +		(PCI_STATUS_REC_MASTER_ABORT |
> PCI_STATUS_REC_TARGET_ABORT))
> > +		return PCIBIOS_DEVICE_NOT_FOUND;
> > +
> > +	if (access_type == PCI_ACCESS_READ)
> > +		*data = pci_read_reg(pcie, PCIECDR);
> > +	else
> > +		pci_write_reg(pcie, *data, PCIECDR);
> > +
> > +	/* Disable the configuration access */
> > +	pci_write_reg(pcie, 0, PCIECCTLR);
> 
> It looks like the above all needs to be atomic, so I suppose you're relying
> on higher-level locking to ensure that.  It might be worth a mention of the
> lock you rely on, just in case that ever changes (I doubt it would, but
> it's conceivable).
Yes, the locking is done at a higher level; I'll add comment for this.

> > ...
> > +static int rcar_pcie_setup_window(int win, struct resource *res,
> > +					struct rcar_pcie *pcie)
> > +{
> > +	/* Setup PCIe address space mappings for each resource */
> > +	resource_size_t size;
> > +	u32 mask;
> > +
> > +	pci_write_reg(pcie, 0x00000000, PCIEPTCTLR(win));
> > +
> > +	/*
> > +	 * The PAMR mask is calculated in units of 128Bytes, which
> > +	 * keeps things pretty simple.
> > +	 */
> > +	size = resource_size(res);
> > +	mask = (roundup_pow_of_two(size) / SZ_128) - 1;
> > +	pci_write_reg(pcie, mask << 7, PCIEPAMR(win));
> > +
> > +	pci_write_reg(pcie, upper_32_bits(res->start), PCIEPARH(win));
> > +	pci_write_reg(pcie, lower_32_bits(res->start), PCIEPARL(win));
> > +
> > +	/* First resource is for IO */
> > +	mask = PAR_ENABLE;
> > +	if (res->flags & IORESOURCE_IO)
> > +		mask |= IO_SPACE;
> > +
> > +	pci_write_reg(pcie, mask, PCIEPTCTLR(win));
> > +
> > +	return 0;
> 
> Maybe could be a void function, since no error is possible?
Sure

> > ...
> > +static int __init rcar_pcie_get_resources(struct platform_device *pdev,
> > +	struct rcar_pcie *pcie)
> > +{
> > +	struct resource res;
> > +	int err;
> > +
> > +	err = of_address_to_resource(pdev->dev.of_node, 0, &res);
> > +	if (err)
> > +		return err;
> 
> I don't see anywhere that figures out the bus number range behind this host
> bridge.  The PCI core needs to know this.  I know the core assumes 00-ff if
> it's not supplied, but that's just to deal with the legacy situation where
> we assumed one host bridge was all anybody would ever need.
> 
> For new code, we should be explicit about the range.
This means add a DT entry for bus-range, and simply calling pci_add_resource for
the range, right? If so, ok.

Thanks
Phil
--
To unsubscribe from this list: send the line "unsubscribe linux-pci" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Bjorn Helgaas April 28, 2014, 7:11 p.m. UTC | #3 
On Mon, Apr 28, 2014 at 4:03 AM, Phil Edworthy
<phil.edworthy@renesas.com> wrote:
> Hi Bjorn,
>
> Thanks for the review.
>
> On 24 April 2014 20:19, Bjorn wrote:
>> On Mon, Mar 31, 2014 at 11:30:46AM +0100, Phil Edworthy wrote:
>> > This PCIe Host driver currently does not support MSI, so cards
>> > fall back to INTx interrupts.
>> >
>> > Signed-off-by: Phil Edworthy <phil.edworthy@renesas.com>
>> > ...
>>
>> > --- /dev/null
>> > +++ b/drivers/pci/host/pcie-rcar.c
>> > @@ -0,0 +1,693 @@
>> > ...
>> > +#include <linux/slab.h>
>> > +#include "pcie-rcar.h"
>>
>> It looks like the things in pcie-rcar.h are used only in this file
>> (pcie-rcar.c), so you might as well just put the contents here and not have
>> a pcie-rcar.h at all.  Of course, if there are things needed by more than
>> one file, they should stay in pcie-rcar.h.
> Nothing else uses them so I'll put in the c file.
>
>> > +#define DRV_NAME "rcar-pcie"
>> > +
>> > +#define RCONF(x)   (PCICONF(0)+(x))
>> > +#define RPMCAP(x)  (PMCAP(0)+(x))
>> > +#define REXPCAP(x) (EXPCAP(0)+(x))
>> > +#define RVCCAP(x)  (VCCAP(0)+(x))
>> > +
>> > +#define  PCIE_CONF_BUS(b)  (((b) & 0xff) << 24)
>> > +#define  PCIE_CONF_DEV(d)  (((d) & 0x1f) << 19)
>> > +#define  PCIE_CONF_FUNC(f) (((f) & 0x7) << 16)
>> > +
>> > +#define PCI_MAX_RESOURCES 4
>> > +#define MAX_NR_INBOUND_MAPS 6
>> > +
>> > +/* Structure representing the PCIe interface */
>> > +struct rcar_pcie {
>> > +   struct device           *dev;
>> > +   void __iomem            *base;
>> > +   struct resource         res[PCI_MAX_RESOURCES];
>> > +   u8                      root_bus_nr;
>>
>> I don't understand how root_bus_nr works.  From its name, it sounds like
>> it's the bus number of the PCI bus on the downstream side of the host
>> bridge.
> That's correct.
>
>> That bus number should be a property of the host bridge, i.e.,
>> it's either hard-wired into the bridge, or it's programmable somewhere.
>> But root_bus_nr comes from sys->busnr, which is apparently from some
>> generic code that knows nothing about this particular host bridge.
> The manual for this hardware says that the HW doesn't care what the bus number
> is set to. The only thing the HW needs to know is that when sending config
> accesses, we need to indicate whether it's a TYPE0 or TYPE1 header; so we use
> root_bus_nr to determine this. The generic code that sets up sys->busnr (ARM
> bios32 in this case), just increments bus number for each controller.
>
> This is very similar to the pcie-designware driver, in dw_pcie_scan_bus().

OK.  From what you said below, it sounds like you would add a DT entry
for the bus range you want to have below this bridge, and use TYPE0
accesses for the base of that range.

I'm not sure how you reconcile this with the idea that the bios32 code
just increments a bus number for each controller.

>> > +static int __init rcar_pcie_get_resources(struct platform_device *pdev,
>> > +   struct rcar_pcie *pcie)
>> > +{
>> > +   struct resource res;
>> > +   int err;
>> > +
>> > +   err = of_address_to_resource(pdev->dev.of_node, 0, &res);
>> > +   if (err)
>> > +           return err;
>>
>> I don't see anywhere that figures out the bus number range behind this host
>> bridge.  The PCI core needs to know this.  I know the core assumes 00-ff if
>> it's not supplied, but that's just to deal with the legacy situation where
>> we assumed one host bridge was all anybody would ever need.
>>
>> For new code, we should be explicit about the range.
> This means add a DT entry for bus-range, and simply calling pci_add_resource for
> the range, right? If so, ok.

Yeah, I guess.  If the HW really doesn't look at the bus number at
all, it sounds like the range is effectively 00-ff, and each host
bridge should be in its own domain.

Bjorn
--
To unsubscribe from this list: send the line "unsubscribe linux-pci" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Jason Gunthorpe April 28, 2014, 8:35 p.m. UTC | #4 
On Mon, Apr 28, 2014 at 01:11:08PM -0600, Bjorn Helgaas wrote:

> >> That bus number should be a property of the host bridge, i.e.,
> >> it's either hard-wired into the bridge, or it's programmable somewhere.
> >> But root_bus_nr comes from sys->busnr, which is apparently from some
> >> generic code that knows nothing about this particular host bridge.

> > The manual for this hardware says that the HW doesn't care what the bus number
> > is set to. The only thing the HW needs to know is that when sending config
> > accesses, we need to indicate whether it's a TYPE0 or TYPE1 header; so we use
> > root_bus_nr to determine this. The generic code that sets up sys->busnr (ARM
> > bios32 in this case), just increments bus number for each controller.

I went back and looked at the lspci posted for this hardware and this
use of busnr is kinda sketch..

You should use use bus 0 as the root complex integrated bus number and
ensure every instance of your driver creates a new PCI domain, and
generally use 0-> FF as the subordinate bus number range.

FWIW, your HW does care, and does have a programmable field for
this.

You set the value of the integrated bus based on the bus number in the
TYPE0 accesses the driver performs and it shows up in the PCI config
space of the root port bridge:

root@koelsch:~# lspci -vv
00:00.0 PCI bridge: Renesas Technology Corp. Device 001f (prog-if 00
[Normal decode])
        Bus: primary=00, secondary=01, subordinate=01, sec-latency=0

The value here will be also be exported on-the-wire in certain PCI-E
TLPs.

So it is important, and must be set properly.

> Yeah, I guess.  If the HW really doesn't look at the bus number at
> all, it sounds like the range is effectively 00-ff, and each host
> bridge should be in its own domain.

This is where some of the other implementations have used some SW to
route config space in a single domain to the proper controller
per-port register set.

But, IMHO, that is mainly worth doing if the HW can implement the PCI
bridge windows according to the spec, and it looks like rcar requires
a 128MB alignment on the bridge windows, which makes it fairly
pointless.

Jason
--
To unsubscribe from this list: send the line "unsubscribe linux-pci" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Phil Edworthy April 30, 2014, 10:20 a.m. UTC | #5 
SGkgQmpvcm4sDQoNCk9uIDI4IEFwcmlsIDIwMTQgMjA6MTEsIEJqb3JuIHdyb3RlOg0KPiBPbiBN
b24sIEFwciAyOCwgMjAxNCBhdCA0OjAzIEFNLCBQaGlsIEVkd29ydGh5DQo+IDxwaGlsLmVkd29y
dGh5QHJlbmVzYXMuY29tPiB3cm90ZToNCj4gPiBIaSBCam9ybiwNCj4gPg0KPiA+IFRoYW5rcyBm
b3IgdGhlIHJldmlldy4NCj4gPg0KPiA+IE9uIDI0IEFwcmlsIDIwMTQgMjA6MTksIEJqb3JuIHdy
b3RlOg0KPiA+PiBPbiBNb24sIE1hciAzMSwgMjAxNCBhdCAxMTozMDo0NkFNICswMTAwLCBQaGls
IEVkd29ydGh5IHdyb3RlOg0KPiA+PiA+IFRoaXMgUENJZSBIb3N0IGRyaXZlciBjdXJyZW50bHkg
ZG9lcyBub3Qgc3VwcG9ydCBNU0ksIHNvIGNhcmRzDQo+ID4+ID4gZmFsbCBiYWNrIHRvIElOVHgg
aW50ZXJydXB0cy4NCj4gPj4gPg0KPiA+PiA+IFNpZ25lZC1vZmYtYnk6IFBoaWwgRWR3b3J0aHkg
PHBoaWwuZWR3b3J0aHlAcmVuZXNhcy5jb20+DQo+ID4+ID4gLi4uDQo+ID4+DQo+ID4+ID4gLS0t
IC9kZXYvbnVsbA0KPiA+PiA+ICsrKyBiL2RyaXZlcnMvcGNpL2hvc3QvcGNpZS1yY2FyLmMNCj4g
Pj4gPiBAQCAtMCwwICsxLDY5MyBAQA0KPiA+PiA+IC4uLg0KPiA+PiA+ICsjaW5jbHVkZSA8bGlu
dXgvc2xhYi5oPg0KPiA+PiA+ICsjaW5jbHVkZSAicGNpZS1yY2FyLmgiDQo+ID4+DQo+ID4+IEl0
IGxvb2tzIGxpa2UgdGhlIHRoaW5ncyBpbiBwY2llLXJjYXIuaCBhcmUgdXNlZCBvbmx5IGluIHRo
aXMgZmlsZQ0KPiA+PiAocGNpZS1yY2FyLmMpLCBzbyB5b3UgbWlnaHQgYXMgd2VsbCBqdXN0IHB1
dCB0aGUgY29udGVudHMgaGVyZSBhbmQgbm90IGhhdmUNCj4gPj4gYSBwY2llLXJjYXIuaCBhdCBh
bGwuICBPZiBjb3Vyc2UsIGlmIHRoZXJlIGFyZSB0aGluZ3MgbmVlZGVkIGJ5IG1vcmUgdGhhbg0K
PiA+PiBvbmUgZmlsZSwgdGhleSBzaG91bGQgc3RheSBpbiBwY2llLXJjYXIuaC4NCj4gPiBOb3Ro
aW5nIGVsc2UgdXNlcyB0aGVtIHNvIEknbGwgcHV0IGluIHRoZSBjIGZpbGUuDQo+ID4NCj4gPj4g
PiArI2RlZmluZSBEUlZfTkFNRSAicmNhci1wY2llIg0KPiA+PiA+ICsNCj4gPj4gPiArI2RlZmlu
ZSBSQ09ORih4KSAgIChQQ0lDT05GKDApKyh4KSkNCj4gPj4gPiArI2RlZmluZSBSUE1DQVAoeCkg
IChQTUNBUCgwKSsoeCkpDQo+ID4+ID4gKyNkZWZpbmUgUkVYUENBUCh4KSAoRVhQQ0FQKDApKyh4
KSkNCj4gPj4gPiArI2RlZmluZSBSVkNDQVAoeCkgIChWQ0NBUCgwKSsoeCkpDQo+ID4+ID4gKw0K
PiA+PiA+ICsjZGVmaW5lICBQQ0lFX0NPTkZfQlVTKGIpICAoKChiKSAmIDB4ZmYpIDw8IDI0KQ0K
PiA+PiA+ICsjZGVmaW5lICBQQ0lFX0NPTkZfREVWKGQpICAoKChkKSAmIDB4MWYpIDw8IDE5KQ0K
PiA+PiA+ICsjZGVmaW5lICBQQ0lFX0NPTkZfRlVOQyhmKSAoKChmKSAmIDB4NykgPDwgMTYpDQo+
ID4+ID4gKw0KPiA+PiA+ICsjZGVmaW5lIFBDSV9NQVhfUkVTT1VSQ0VTIDQNCj4gPj4gPiArI2Rl
ZmluZSBNQVhfTlJfSU5CT1VORF9NQVBTIDYNCj4gPj4gPiArDQo+ID4+ID4gKy8qIFN0cnVjdHVy
ZSByZXByZXNlbnRpbmcgdGhlIFBDSWUgaW50ZXJmYWNlICovDQo+ID4+ID4gK3N0cnVjdCByY2Fy
X3BjaWUgew0KPiA+PiA+ICsgICBzdHJ1Y3QgZGV2aWNlICAgICAgICAgICAqZGV2Ow0KPiA+PiA+
ICsgICB2b2lkIF9faW9tZW0gICAgICAgICAgICAqYmFzZTsNCj4gPj4gPiArICAgc3RydWN0IHJl
c291cmNlICAgICAgICAgcmVzW1BDSV9NQVhfUkVTT1VSQ0VTXTsNCj4gPj4gPiArICAgdTggICAg
ICAgICAgICAgICAgICAgICAgcm9vdF9idXNfbnI7DQo+ID4+DQo+ID4+IEkgZG9uJ3QgdW5kZXJz
dGFuZCBob3cgcm9vdF9idXNfbnIgd29ya3MuICBGcm9tIGl0cyBuYW1lLCBpdCBzb3VuZHMgbGlr
ZQ0KPiA+PiBpdCdzIHRoZSBidXMgbnVtYmVyIG9mIHRoZSBQQ0kgYnVzIG9uIHRoZSBkb3duc3Ry
ZWFtIHNpZGUgb2YgdGhlIGhvc3QNCj4gPj4gYnJpZGdlLg0KPiA+IFRoYXQncyBjb3JyZWN0Lg0K
PiA+DQo+ID4+IFRoYXQgYnVzIG51bWJlciBzaG91bGQgYmUgYSBwcm9wZXJ0eSBvZiB0aGUgaG9z
dCBicmlkZ2UsIGkuZS4sDQo+ID4+IGl0J3MgZWl0aGVyIGhhcmQtd2lyZWQgaW50byB0aGUgYnJp
ZGdlLCBvciBpdCdzIHByb2dyYW1tYWJsZSBzb21ld2hlcmUuDQo+ID4+IEJ1dCByb290X2J1c19u
ciBjb21lcyBmcm9tIHN5cy0+YnVzbnIsIHdoaWNoIGlzIGFwcGFyZW50bHkgZnJvbSBzb21lDQo+
ID4+IGdlbmVyaWMgY29kZSB0aGF0IGtub3dzIG5vdGhpbmcgYWJvdXQgdGhpcyBwYXJ0aWN1bGFy
IGhvc3QgYnJpZGdlLg0KPiA+IFRoZSBtYW51YWwgZm9yIHRoaXMgaGFyZHdhcmUgc2F5cyB0aGF0
IHRoZSBIVyBkb2Vzbid0IGNhcmUgd2hhdCB0aGUgYnVzDQo+IG51bWJlcg0KPiA+IGlzIHNldCB0
by4gVGhlIG9ubHkgdGhpbmcgdGhlIEhXIG5lZWRzIHRvIGtub3cgaXMgdGhhdCB3aGVuIHNlbmRp
bmcgY29uZmlnDQo+ID4gYWNjZXNzZXMsIHdlIG5lZWQgdG8gaW5kaWNhdGUgd2hldGhlciBpdCdz
IGEgVFlQRTAgb3IgVFlQRTEgaGVhZGVyOyBzbyB3ZQ0KPiB1c2UNCj4gPiByb290X2J1c19uciB0
byBkZXRlcm1pbmUgdGhpcy4gVGhlIGdlbmVyaWMgY29kZSB0aGF0IHNldHMgdXAgc3lzLT5idXNu
cg0KPiAoQVJNDQo+ID4gYmlvczMyIGluIHRoaXMgY2FzZSksIGp1c3QgaW5jcmVtZW50cyBidXMg
bnVtYmVyIGZvciBlYWNoIGNvbnRyb2xsZXIuDQo+ID4NCj4gPiBUaGlzIGlzIHZlcnkgc2ltaWxh
ciB0byB0aGUgcGNpZS1kZXNpZ253YXJlIGRyaXZlciwgaW4gZHdfcGNpZV9zY2FuX2J1cygpLg0K
PiANCj4gT0suICBGcm9tIHdoYXQgeW91IHNhaWQgYmVsb3csIGl0IHNvdW5kcyBsaWtlIHlvdSB3
b3VsZCBhZGQgYSBEVCBlbnRyeQ0KPiBmb3IgdGhlIGJ1cyByYW5nZSB5b3Ugd2FudCB0byBoYXZl
IGJlbG93IHRoaXMgYnJpZGdlLCBhbmQgdXNlIFRZUEUwDQo+IGFjY2Vzc2VzIGZvciB0aGUgYmFz
ZSBvZiB0aGF0IHJhbmdlLg0KT2suIFRoZSBkcml2ZXIgYWxyZWFkeSB1c2VzIFRZUEUwIGZvciB0
aGUgYmFzZSwgaS5lLiB0aGF0J3Mgd2hhdCByb290X2J1c19uciBpcyB1c2VkIGZvci4gSSdsbCBh
ZGQgdGhlIHJhbmdlcyBhYmlsaXR5Lg0KDQo+IEknbSBub3Qgc3VyZSBob3cgeW91IHJlY29uY2ls
ZSB0aGlzIHdpdGggdGhlIGlkZWEgdGhhdCB0aGUgYmlvczMyIGNvZGUNCj4ganVzdCBpbmNyZW1l
bnRzIGEgYnVzIG51bWJlciBmb3IgZWFjaCBjb250cm9sbGVyLg0KSSBsb29rZWQgYXQgdGhlIGJp
b3MzMiBjb2RlIGFnYWluLiBJIHdhcyB3cm9uZyB0byBzYXkgdGhhdCBpdCBqdXN0IGluY3JlbWVu
dHMgYSBidXMgbnVtYmVyIGZvciBlYWNoIGNvbnRyb2xsZXIuIFRoZSBmaXJzdCBjb250cm9sbGVy
IGFsd2F5cyBzdGFydHMgd2l0aCBidXNucj0wLCBhbmQgc3Vic2VxdWVudCBjb250cm9sbGVycyBn
ZXQgdGhlIG5leHQgYXZhaWxhYmxlIGJ1cyBudW1iZXIgYWZ0ZXIgc2Nhbm5pbmcgdGhlIGJ1cy4N
Cg0KPiA+PiA+ICtzdGF0aWMgaW50IF9faW5pdCByY2FyX3BjaWVfZ2V0X3Jlc291cmNlcyhzdHJ1
Y3QgcGxhdGZvcm1fZGV2aWNlDQo+ICpwZGV2LA0KPiA+PiA+ICsgICBzdHJ1Y3QgcmNhcl9wY2ll
ICpwY2llKQ0KPiA+PiA+ICt7DQo+ID4+ID4gKyAgIHN0cnVjdCByZXNvdXJjZSByZXM7DQo+ID4+
ID4gKyAgIGludCBlcnI7DQo+ID4+ID4gKw0KPiA+PiA+ICsgICBlcnIgPSBvZl9hZGRyZXNzX3Rv
X3Jlc291cmNlKHBkZXYtPmRldi5vZl9ub2RlLCAwLCAmcmVzKTsNCj4gPj4gPiArICAgaWYgKGVy
cikNCj4gPj4gPiArICAgICAgICAgICByZXR1cm4gZXJyOw0KPiA+Pg0KPiA+PiBJIGRvbid0IHNl
ZSBhbnl3aGVyZSB0aGF0IGZpZ3VyZXMgb3V0IHRoZSBidXMgbnVtYmVyIHJhbmdlIGJlaGluZCB0
aGlzDQo+IGhvc3QNCj4gPj4gYnJpZGdlLiAgVGhlIFBDSSBjb3JlIG5lZWRzIHRvIGtub3cgdGhp
cy4gIEkga25vdyB0aGUgY29yZSBhc3N1bWVzIDAwLWZmIGlmDQo+ID4+IGl0J3Mgbm90IHN1cHBs
aWVkLCBidXQgdGhhdCdzIGp1c3QgdG8gZGVhbCB3aXRoIHRoZSBsZWdhY3kgc2l0dWF0aW9uIHdo
ZXJlDQo+ID4+IHdlIGFzc3VtZWQgb25lIGhvc3QgYnJpZGdlIHdhcyBhbGwgYW55Ym9keSB3b3Vs
ZCBldmVyIG5lZWQuDQo+ID4+DQo+ID4+IEZvciBuZXcgY29kZSwgd2Ugc2hvdWxkIGJlIGV4cGxp
Y2l0IGFib3V0IHRoZSByYW5nZS4NCj4gPiBUaGlzIG1lYW5zIGFkZCBhIERUIGVudHJ5IGZvciBi
dXMtcmFuZ2UsIGFuZCBzaW1wbHkgY2FsbGluZw0KPiBwY2lfYWRkX3Jlc291cmNlIGZvcg0KPiA+
IHRoZSByYW5nZSwgcmlnaHQ/IElmIHNvLCBvay4NCj4gDQo+IFllYWgsIEkgZ3Vlc3MuICBJZiB0
aGUgSFcgcmVhbGx5IGRvZXNuJ3QgbG9vayBhdCB0aGUgYnVzIG51bWJlciBhdA0KPiBhbGwsIGl0
IHNvdW5kcyBsaWtlIHRoZSByYW5nZSBpcyBlZmZlY3RpdmVseSAwMC1mZiwgYW5kIGVhY2ggaG9z
dA0KPiBicmlkZ2Ugc2hvdWxkIGJlIGluIGl0cyBvd24gZG9tYWluLg0KT2suDQoNClRoYW5rcw0K
UGhpbA0K
--
To unsubscribe from this list: send the line "unsubscribe linux-pci" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Phil Edworthy April 30, 2014, 10:33 a.m. UTC | #6 
Hi Jason,

On 28 April 2014 21:35, Jason wrote:
> On Mon, Apr 28, 2014 at 01:11:08PM -0600, Bjorn Helgaas wrote:
> 
> > >> That bus number should be a property of the host bridge, i.e.,
> > >> it's either hard-wired into the bridge, or it's programmable somewhere.
> > >> But root_bus_nr comes from sys->busnr, which is apparently from
> some
> > >> generic code that knows nothing about this particular host bridge.
> 
> > > The manual for this hardware says that the HW doesn't care what the bus
> number
> > > is set to. The only thing the HW needs to know is that when sending
> config
> > > accesses, we need to indicate whether it's a TYPE0 or TYPE1 header; so
> we use
> > > root_bus_nr to determine this. The generic code that sets up sys->busnr
> (ARM
> > > bios32 in this case), just increments bus number for each controller.
> 
> I went back and looked at the lspci posted for this hardware and this
> use of busnr is kinda sketch..
> 
> You should use use bus 0 as the root complex integrated bus number and
> ensure every instance of your driver creates a new PCI domain, and
> generally use 0-> FF as the subordinate bus number range.
> 
> FWIW, your HW does care, and does have a programmable field for
> this.
> 
> You set the value of the integrated bus based on the bus number in the
> TYPE0 accesses the driver performs and it shows up in the PCI config
> space of the root port bridge:
> 
> root@koelsch:~# lspci -vv
> 00:00.0 PCI bridge: Renesas Technology Corp. Device 001f (prog-if 00
> [Normal decode])
>         Bus: primary=00, secondary=01, subordinate=01, sec-latency=0
> 
> The value here will be also be exported on-the-wire in certain PCI-E
> TLPs.
> 
> So it is important, and must be set properly.
I am not so sure about this... 

There are three things we need to consider:
1) talking to the host bridge itself
2) talking to immediate device (connected directly to host bridge)
3) talking to other devices

The PCI standard tells us we use type 1's for 3) and type 0's for 2). The driver uses root_bus_nr to make this decision.
For 1), there is a note in the driver that the hardware cannot target itself. The driver has code in rcar_pcie_config_access() to check if it's an access to the host bridge, and if so, use I/O (readl/writel) to perform the config access.

> > Yeah, I guess.  If the HW really doesn't look at the bus number at
> > all, it sounds like the range is effectively 00-ff, and each host
> > bridge should be in its own domain.
> 
> This is where some of the other implementations have used some SW to
> route config space in a single domain to the proper controller
> per-port register set.
> 
> But, IMHO, that is mainly worth doing if the HW can implement the PCI
> bridge windows according to the spec, and it looks like rcar requires
> a 128MB alignment on the bridge windows, which makes it fairly
> pointless.
> 
> Jason

Thanks
Phil
--
To unsubscribe from this list: send the line "unsubscribe linux-pci" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Jason Gunthorpe April 30, 2014, 3:43 p.m. UTC | #7 
On Wed, Apr 30, 2014 at 10:33:08AM +0000, Phil Edworthy wrote:

> > You set the value of the integrated bus based on the bus number in the
> > TYPE0 accesses the driver performs and it shows up in the PCI config
> > space of the root port bridge:
> > 
> > root@koelsch:~# lspci -vv
> > 00:00.0 PCI bridge: Renesas Technology Corp. Device 001f (prog-if 00
> > [Normal decode])
> >         Bus: primary=00, secondary=01, subordinate=01, sec-latency=0
> > 
> > The value here will be also be exported on-the-wire in certain PCI-E
> > TLPs.
> > 
> > So it is important, and must be set properly.
> I am not so sure about this... 
> 
> There are three things we need to consider:
> 1) talking to the host bridge itself
> 2) talking to immediate device (connected directly to host bridge)
> 3) talking to other devices
> 
> The PCI standard tells us we use type 1's for 3) and type 0's for
> 2). The driver uses root_bus_nr to make this decision.  For 1),
> there is a note in the driver that the hardware cannot target
> itself. The driver has code in rcar_pcie_config_access() to check if
> it's an access to the host bridge, and if so, use I/O (readl/writel)
> to perform the config access.

The logic should be:
 if (bus == primary)
    do io access to host bridge
 else if (bus == secondary)
    issue type 0 TLP on the wire
 else if (bus > secondary && bus <= subordinate)
    issue type 1 TLP on the wire
 else
    fail, invalid bus number

Where the three values come from the register in the PCI host bridge's
configuration space, and are kept in sync with the programming from
the Linux PCI core.

It is just a happy hapenstance that root_bus_nr equals the value the
PCI core programmed into secondary - that is not guarenteed, you must
use the primary value directly.

I recommend capturing it on write to the host bridge config
space.

Jason
--
To unsubscribe from this list: send the line "unsubscribe linux-pci" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Phil Edworthy May 1, 2014, 9:50 a.m. UTC | #8 
Hi Jason,

On 30 April 2014 16:44, Jason wrote:
> On Wed, Apr 30, 2014 at 10:33:08AM +0000, Phil Edworthy wrote:
> 
> > > You set the value of the integrated bus based on the bus number in the
> > > TYPE0 accesses the driver performs and it shows up in the PCI config
> > > space of the root port bridge:
> > >
> > > root@koelsch:~# lspci -vv
> > > 00:00.0 PCI bridge: Renesas Technology Corp. Device 001f (prog-if 00
> > > [Normal decode])
> > >         Bus: primary=00, secondary=01, subordinate=01, sec-latency=0
> > >
> > > The value here will be also be exported on-the-wire in certain PCI-E
> > > TLPs.
> > >
> > > So it is important, and must be set properly.
> > I am not so sure about this...
> >
> > There are three things we need to consider:
> > 1) talking to the host bridge itself
> > 2) talking to immediate device (connected directly to host bridge)
> > 3) talking to other devices
> >
> > The PCI standard tells us we use type 1's for 3) and type 0's for
> > 2). The driver uses root_bus_nr to make this decision.  For 1),
> > there is a note in the driver that the hardware cannot target
> > itself. The driver has code in rcar_pcie_config_access() to check if
> > it's an access to the host bridge, and if so, use I/O (readl/writel)
> > to perform the config access.
> 
> The logic should be:
>  if (bus == primary)
>     do io access to host bridge
>  else if (bus == secondary)
>     issue type 0 TLP on the wire
>  else if (bus > secondary && bus <= subordinate)
>     issue type 1 TLP on the wire
>  else
>     fail, invalid bus number
> Where the three values come from the register in the PCI host bridge's
> configuration space, and are kept in sync with the programming from
> the Linux PCI core.
> 
> It is just a happy hapenstance that root_bus_nr equals the value the
> PCI core programmed into secondary - that is not guarenteed, you must
> use the primary value directly.
For type0 TLPs, we are not checking that root_bus_nr equals the value the PCI core programmed into secondary, we are checking that the (root_bus_nr == bus->parent->number). The only way this wouldn't work is if root_bus_nr was not the root bus number.

Since the Synopsys DW driver also saves off sys->busnr and later uses this to determine if accesses are for the host bridge, I guess that means it won't always work either, right? Or is it ok because the DW driver saves sys->busnr in its .scan function?

When would the PCI core change the root bus number to something other than set in sys->busnr?

> I recommend capturing it on write to the host bridge config
> space.

Thanks
Phil

--
To unsubscribe from this list: send the line "unsubscribe linux-pci" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Jason Gunthorpe May 1, 2014, 4:50 p.m. UTC | #9 
On Thu, May 01, 2014 at 09:50:56AM +0000, Phil Edworthy wrote:

> > The logic should be:
> >  if (bus == primary)
> >     do io access to host bridge
> >  else if (bus == secondary)
> >     issue type 0 TLP on the wire
> >  else if (bus > secondary && bus <= subordinate)
> >     issue type 1 TLP on the wire
> >  else
> >     fail, invalid bus number
> > Where the three values come from the register in the PCI host bridge's
> > configuration space, and are kept in sync with the programming from
> > the Linux PCI core.
> > 
> > It is just a happy hapenstance that root_bus_nr equals the value the
> > PCI core programmed into secondary - that is not guarenteed, you must
> > use the primary value directly.

> For type0 TLPs, we are not checking that root_bus_nr equals the
> value the PCI core programmed into secondary, we are checking that
> the (root_bus_nr == bus->parent->number). The only way this wouldn't
> work is if root_bus_nr was not the root bus number.

Okay, that isn't as sketchy, but that process still ignores the
subordinate bus number and the failure case as required by PCI.

The goal here is to have the stuff below the drivers implement the PCI
spec so that the core code can assume everything below is
conformant. Drivers should not introduce gratuitous differences 'just
because'

There is no reason drivers should be using PCI core structures to make
decisions when the spec says those decisions are driven by config
space fields.

This way the PCI core code doesn't have to be aware of any weird
non-standard edge cases.. Such as not failing bus numbers beyond the
subordinate bus number.

> Since the Synopsys DW driver also saves off sys->busnr and later
> uses this to determine if accesses are for the host bridge, I guess
> that means it won't always work either, right? Or is it ok because
> the DW driver saves sys->busnr in its .scan function?

Sounds like it is making the same mistake, and nobody noticed. This is
another reason why it is important to implement correctly so people
copying copy the right stuff :)

> When would the PCI core change the root bus number to something
> other than set in sys->busnr?

I think the more likely scenario is that 'sys' in general is
architecture specific and its use is being discouraged so that host
drivers are not arch specific.

A domain driver like rcar should always place the root complex
integrated bus as bus 0 in the domain.

Jason
--
To unsubscribe from this list: send the line "unsubscribe linux-pci" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Bjorn Helgaas May 1, 2014, 5:31 p.m. UTC | #10 
[+cc Srikanth, Michal, Grant because we're having the same discussion
about root bus numbers in the Xilinx host bridge driver]

On Thu, May 1, 2014 at 3:50 AM, Phil Edworthy <phil.edworthy@renesas.com> wrote:

> When would the PCI core change the root bus number to something other than set in sys->busnr?

The PCI core never changes the root bus number of a host bridge, but
it's important for the core to know exactly what bus numbers are
behind each host bridge so we can correctly assign bus numbers, MMIO,
and I/O port resources.

The reason is that when we enumerate a PCI-to-PCI bridge, we have to
know whether bus numbers are available for the secondary bus.  For
example, assume two host bridges configured (or wired) as follows:

  A leads to [bus 00-1f], with aperture [mem 0x80000000-0x8fffffff]
  B leads to [bus 20-ff], with aperture [mem 0x90000000-0x9fffffff]

Further, assume a PCI-to-PCI bridge at 1f:00.0.  If the host bridge
driver doesn't tell us the [bus 00-1f] range for host bridge A, the
PCI core will assume [bus 00-ff].  When the core enumerates the
1f:00.0 bridge, it will assign bus number 20 to its secondary bus.
Now it enumerates devices on bus 20.  The core expects this to be done
via host bridge A, but enumeration uses ID-routed config transactions,
so they are claimed by host bridge B instead.  If we find a device
20:00.0, the core thinks it is below A, but it is really below B.

Now we assign resources to 20:00.0, and since we think it's below A,
we assign them from the [mem 0x80000000-0x8fffffff] aperture.  This
obviously doesn't work, because when the driver performs MMIO
accesses, which are address-routed, they are claimed by host bridge A,
not B, so the 20:00.0 device never sees them.

If your host bridge really claims ALL bus numbers (00-ff), that's
fine, and you can tell the PCI core that.  But if you have several
such bridges, they should be placed in separate domains.

This is probably more detail than you wanted, and maybe you don't ever
expect to have multiple host bridges or complicated hierarchies of
devices.  But the PCI core has to deal with these issues on other
systems, so it's important to get these details right.

Bjorn
--
To unsubscribe from this list: send the line "unsubscribe linux-pci" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Phil Edworthy May 6, 2014, 10:46 a.m. UTC | #11 
Hi Jason,

On 01 May 2014 17:50, Jason wrote:
> On Thu, May 01, 2014 at 09:50:56AM +0000, Phil Edworthy wrote:
> 
> > > The logic should be:
> > >  if (bus == primary)
> > >     do io access to host bridge
> > >  else if (bus == secondary)
> > >     issue type 0 TLP on the wire
> > >  else if (bus > secondary && bus <= subordinate)
> > >     issue type 1 TLP on the wire
> > >  else
> > >     fail, invalid bus number
> > > Where the three values come from the register in the PCI host bridge's
> > > configuration space, and are kept in sync with the programming from
> > > the Linux PCI core.
> > >
> > > It is just a happy hapenstance that root_bus_nr equals the value the
> > > PCI core programmed into secondary - that is not guarenteed, you must
> > > use the primary value directly.
> 
> > For type0 TLPs, we are not checking that root_bus_nr equals the
> > value the PCI core programmed into secondary, we are checking that
> > the (root_bus_nr == bus->parent->number). The only way this wouldn't
> > work is if root_bus_nr was not the root bus number.
> 
> Okay, that isn't as sketchy, but that process still ignores the
> subordinate bus number and the failure case as required by PCI.
> 
> The goal here is to have the stuff below the drivers implement the PCI
> spec so that the core code can assume everything below is
> conformant. Drivers should not introduce gratuitous differences 'just
> because'
> 
> There is no reason drivers should be using PCI core structures to make
> decisions when the spec says those decisions are driven by config
> space fields.
> 
> This way the PCI core code doesn't have to be aware of any weird
> non-standard edge cases.. Such as not failing bus numbers beyond the
> subordinate bus number.
> 
> > Since the Synopsys DW driver also saves off sys->busnr and later
> > uses this to determine if accesses are for the host bridge, I guess
> > that means it won't always work either, right? Or is it ok because
> > the DW driver saves sys->busnr in its .scan function?
> 
> Sounds like it is making the same mistake, and nobody noticed. This is
> another reason why it is important to implement correctly so people
> copying copy the right stuff :)
> 
> > When would the PCI core change the root bus number to something
> > other than set in sys->busnr?
> 
> I think the more likely scenario is that 'sys' in general is
> architecture specific and its use is being discouraged so that host
> drivers are not arch specific.
> 
> A domain driver like rcar should always place the root complex
> integrated bus as bus 0 in the domain.

Ok, I now understand the reason for removing the dependency on sys->busnr. That makes sense & I'll update the driver for this.

Thanks for your patience :)
Phil
--
To unsubscribe from this list: send the line "unsubscribe linux-pci" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Phil Edworthy May 6, 2014, 12:49 p.m. UTC | #12 
SGkgQmpvcm4sDQoNCk9uIDAxIE1heSAyMDE0IDE4OjMxLCBCam9ybiB3cm90ZToNCj4gWytjYyBT
cmlrYW50aCwgTWljaGFsLCBHcmFudCBiZWNhdXNlIHdlJ3JlIGhhdmluZyB0aGUgc2FtZSBkaXNj
dXNzaW9uDQo+IGFib3V0IHJvb3QgYnVzIG51bWJlcnMgaW4gdGhlIFhpbGlueCBob3N0IGJyaWRn
ZSBkcml2ZXJdDQo+IA0KPiBPbiBUaHUsIE1heSAxLCAyMDE0IGF0IDM6NTAgQU0sIFBoaWwgRWR3
b3J0aHkNCj4gPHBoaWwuZWR3b3J0aHlAcmVuZXNhcy5jb20+IHdyb3RlOg0KPiANCj4gPiBXaGVu
IHdvdWxkIHRoZSBQQ0kgY29yZSBjaGFuZ2UgdGhlIHJvb3QgYnVzIG51bWJlciB0byBzb21ldGhp
bmcgb3RoZXINCj4gdGhhbiBzZXQgaW4gc3lzLT5idXNucj8NCj4gDQo+IFRoZSBQQ0kgY29yZSBu
ZXZlciBjaGFuZ2VzIHRoZSByb290IGJ1cyBudW1iZXIgb2YgYSBob3N0IGJyaWRnZSwgYnV0DQo+
IGl0J3MgaW1wb3J0YW50IGZvciB0aGUgY29yZSB0byBrbm93IGV4YWN0bHkgd2hhdCBidXMgbnVt
YmVycyBhcmUNCj4gYmVoaW5kIGVhY2ggaG9zdCBicmlkZ2Ugc28gd2UgY2FuIGNvcnJlY3RseSBh
c3NpZ24gYnVzIG51bWJlcnMsIE1NSU8sDQo+IGFuZCBJL08gcG9ydCByZXNvdXJjZXMuDQo+IA0K
PiBUaGUgcmVhc29uIGlzIHRoYXQgd2hlbiB3ZSBlbnVtZXJhdGUgYSBQQ0ktdG8tUENJIGJyaWRn
ZSwgd2UgaGF2ZSB0bw0KPiBrbm93IHdoZXRoZXIgYnVzIG51bWJlcnMgYXJlIGF2YWlsYWJsZSBm
b3IgdGhlIHNlY29uZGFyeSBidXMuICBGb3INCj4gZXhhbXBsZSwgYXNzdW1lIHR3byBob3N0IGJy
aWRnZXMgY29uZmlndXJlZCAob3Igd2lyZWQpIGFzIGZvbGxvd3M6DQo+IA0KPiAgIEEgbGVhZHMg
dG8gW2J1cyAwMC0xZl0sIHdpdGggYXBlcnR1cmUgW21lbSAweDgwMDAwMDAwLTB4OGZmZmZmZmZd
DQo+ICAgQiBsZWFkcyB0byBbYnVzIDIwLWZmXSwgd2l0aCBhcGVydHVyZSBbbWVtIDB4OTAwMDAw
MDAtMHg5ZmZmZmZmZl0NCj4gDQo+IEZ1cnRoZXIsIGFzc3VtZSBhIFBDSS10by1QQ0kgYnJpZGdl
IGF0IDFmOjAwLjAuICBJZiB0aGUgaG9zdCBicmlkZ2UNCj4gZHJpdmVyIGRvZXNuJ3QgdGVsbCB1
cyB0aGUgW2J1cyAwMC0xZl0gcmFuZ2UgZm9yIGhvc3QgYnJpZGdlIEEsIHRoZQ0KPiBQQ0kgY29y
ZSB3aWxsIGFzc3VtZSBbYnVzIDAwLWZmXS4gIFdoZW4gdGhlIGNvcmUgZW51bWVyYXRlcyB0aGUN
Cj4gMWY6MDAuMCBicmlkZ2UsIGl0IHdpbGwgYXNzaWduIGJ1cyBudW1iZXIgMjAgdG8gaXRzIHNl
Y29uZGFyeSBidXMuDQo+IE5vdyBpdCBlbnVtZXJhdGVzIGRldmljZXMgb24gYnVzIDIwLiAgVGhl
IGNvcmUgZXhwZWN0cyB0aGlzIHRvIGJlIGRvbmUNCj4gdmlhIGhvc3QgYnJpZGdlIEEsIGJ1dCBl
bnVtZXJhdGlvbiB1c2VzIElELXJvdXRlZCBjb25maWcgdHJhbnNhY3Rpb25zLA0KPiBzbyB0aGV5
IGFyZSBjbGFpbWVkIGJ5IGhvc3QgYnJpZGdlIEIgaW5zdGVhZC4gIElmIHdlIGZpbmQgYSBkZXZp
Y2UNCj4gMjA6MDAuMCwgdGhlIGNvcmUgdGhpbmtzIGl0IGlzIGJlbG93IEEsIGJ1dCBpdCBpcyBy
ZWFsbHkgYmVsb3cgQi4NCj4gDQo+IE5vdyB3ZSBhc3NpZ24gcmVzb3VyY2VzIHRvIDIwOjAwLjAs
IGFuZCBzaW5jZSB3ZSB0aGluayBpdCdzIGJlbG93IEEsDQo+IHdlIGFzc2lnbiB0aGVtIGZyb20g
dGhlIFttZW0gMHg4MDAwMDAwMC0weDhmZmZmZmZmXSBhcGVydHVyZS4gIFRoaXMNCj4gb2J2aW91
c2x5IGRvZXNuJ3Qgd29yaywgYmVjYXVzZSB3aGVuIHRoZSBkcml2ZXIgcGVyZm9ybXMgTU1JTw0K
PiBhY2Nlc3Nlcywgd2hpY2ggYXJlIGFkZHJlc3Mtcm91dGVkLCB0aGV5IGFyZSBjbGFpbWVkIGJ5
IGhvc3QgYnJpZGdlIEEsDQo+IG5vdCBCLCBzbyB0aGUgMjA6MDAuMCBkZXZpY2UgbmV2ZXIgc2Vl
cyB0aGVtLg0KVGhhbmtzIGZvciB0aGUgY2xlYXIgZXhwbGFuYXRpb24uDQoNCj4gSWYgeW91ciBo
b3N0IGJyaWRnZSByZWFsbHkgY2xhaW1zIEFMTCBidXMgbnVtYmVycyAoMDAtZmYpLCB0aGF0J3MN
Cj4gZmluZSwgYW5kIHlvdSBjYW4gdGVsbCB0aGUgUENJIGNvcmUgdGhhdC4gIEJ1dCBpZiB5b3Ug
aGF2ZSBzZXZlcmFsDQo+IHN1Y2ggYnJpZGdlcywgdGhleSBzaG91bGQgYmUgcGxhY2VkIGluIHNl
cGFyYXRlIGRvbWFpbnMuDQpUaGF0IG1ha2VzIHNlbnNlLg0KDQpJbiB0aGUgY2FzZSBvZiB0aGUg
Ui1DYXIgUENJZSBob3N0IGJyaWRnZSwgYXMgZmFyIGFzIEkgY2FuIHRlbGwsIHRoZSBoYXJkd2Fy
ZSBkb2VzbuKAmXQgY2FyZSBhYm91dCBidXMgbnVtYmVycyBhdCBhbGwsIHdpdGggdGhlIGV4Y2Vw
dGlvbiB0aGF0IHRoZSBkcml2ZXIgdXNlcyBvbmUgYnVzIG51bWJlciB0byB0YXJnZXQgdGhlIGJy
aWRnZSBpdHNlbGYuIFdlIGNhbiB1c2UgYXMgbWFueSBvciBhcyBmZXcgYnVzIG51bWJlcnMgZm9y
IHRoZSBzZWNvbmRhcnkgYnVzIGFzIHlvdSBsaWtlLg0KDQpTbywgdGhlIFItQ2FyIFBDSWUgZHJp
dmVyIHNob3VsZCBpbXBsZW1lbnQgYnVzIHJhbmdlcywgYnV0IHNob3VsZCBpdCBhbHNvIHVzZSBz
ZXBhcmF0ZSBQQ0kgZG9tYWlucyBmb3IgbXVsdGlwbGUgY29udHJvbGxlcnM/DQoNCj4gVGhpcyBp
cyBwcm9iYWJseSBtb3JlIGRldGFpbCB0aGFuIHlvdSB3YW50ZWQsIGFuZCBtYXliZSB5b3UgZG9u
J3QgZXZlcg0KPiBleHBlY3QgdG8gaGF2ZSBtdWx0aXBsZSBob3N0IGJyaWRnZXMgb3IgY29tcGxp
Y2F0ZWQgaGllcmFyY2hpZXMgb2YNCj4gZGV2aWNlcy4gIEJ1dCB0aGUgUENJIGNvcmUgaGFzIHRv
IGRlYWwgd2l0aCB0aGVzZSBpc3N1ZXMgb24gb3RoZXINCj4gc3lzdGVtcywgc28gaXQncyBpbXBv
cnRhbnQgdG8gZ2V0IHRoZXNlIGRldGFpbHMgcmlnaHQuDQoNClRoYW5rcw0KUGhpbA0K
--
To unsubscribe from this list: send the line "unsubscribe linux-pci" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html
diff mbox

Patch

diff --git a/drivers/pci/host/Kconfig b/drivers/pci/host/Kconfig
index 47d46c6..dc627e5 100644
--- a/drivers/pci/host/Kconfig
+++ b/drivers/pci/host/Kconfig
@@ -33,4 +33,10 @@  config PCI_RCAR_GEN2
 	  There are 3 internal PCI controllers available with a single
 	  built-in EHCI/OHCI host controller present on each one.
 
+config PCI_RCAR_GEN2_PCIE
+	bool "Renesas R-Car PCIe controller"
+	depends on ARCH_SHMOBILE || (ARM && COMPILE_TEST)
+	help
+	  Say Y here if you want PCIe controller support on R-Car Gen2 SoCs.
+
 endmenu
diff --git a/drivers/pci/host/Makefile b/drivers/pci/host/Makefile
index 13fb333..19946f9 100644
--- a/drivers/pci/host/Makefile
+++ b/drivers/pci/host/Makefile
@@ -4,3 +4,4 @@  obj-$(CONFIG_PCI_IMX6) += pci-imx6.o
 obj-$(CONFIG_PCI_MVEBU) += pci-mvebu.o
 obj-$(CONFIG_PCI_TEGRA) += pci-tegra.o
 obj-$(CONFIG_PCI_RCAR_GEN2) += pci-rcar-gen2.o
+obj-$(CONFIG_PCI_RCAR_GEN2_PCIE) += pcie-rcar.o
diff --git a/drivers/pci/host/pcie-rcar.c b/drivers/pci/host/pcie-rcar.c
new file mode 100644
index 0000000..c22c896
--- /dev/null
+++ b/drivers/pci/host/pcie-rcar.c
@@ -0,0 +1,693 @@ 
+/*
+ * PCIe driver for Renesas R-Car SoCs
+ *  Copyright (C) 2014 Renesas Electronics Europe Ltd
+ *
+ * Based on:
+ *  arch/sh/drivers/pci/pcie-sh7786.c
+ *  arch/sh/drivers/pci/ops-sh7786.c
+ *  Copyright (C) 2009 - 2011  Paul Mundt
+ *
+ * This file is licensed under the terms of the GNU General Public
+ * License version 2.  This program is licensed "as is" without any
+ * warranty of any kind, whether express or implied.
+ */
+
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/interrupt.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of_address.h>
+#include <linux/of_irq.h>
+#include <linux/of_pci.h>
+#include <linux/of_platform.h>
+#include <linux/pci.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+#include "pcie-rcar.h"
+
+#define DRV_NAME "rcar-pcie"
+
+#define RCONF(x)	(PCICONF(0)+(x))
+#define RPMCAP(x)	(PMCAP(0)+(x))
+#define REXPCAP(x)	(EXPCAP(0)+(x))
+#define RVCCAP(x)	(VCCAP(0)+(x))
+
+#define  PCIE_CONF_BUS(b)	(((b) & 0xff) << 24)
+#define  PCIE_CONF_DEV(d)	(((d) & 0x1f) << 19)
+#define  PCIE_CONF_FUNC(f)	(((f) & 0x7) << 16)
+
+#define PCI_MAX_RESOURCES 4
+#define MAX_NR_INBOUND_MAPS 6
+
+/* Structure representing the PCIe interface */
+struct rcar_pcie {
+	struct device		*dev;
+	void __iomem		*base;
+	struct resource		res[PCI_MAX_RESOURCES];
+	u8			root_bus_nr;
+	struct clk		*clk;
+	struct clk		*bus_clk;
+};
+
+static inline struct rcar_pcie *sys_to_pcie(struct pci_sys_data *sys)
+{
+	return sys->private_data;
+}
+
+static void
+pci_write_reg(struct rcar_pcie *pcie, unsigned long val, unsigned long reg)
+{
+	writel(val, pcie->base + reg);
+}
+
+static unsigned long
+pci_read_reg(struct rcar_pcie *pcie, unsigned long reg)
+{
+	return readl(pcie->base + reg);
+}
+
+enum {
+	PCI_ACCESS_READ,
+	PCI_ACCESS_WRITE,
+};
+
+static void rcar_rmw32(struct rcar_pcie *pcie,
+			    int where, u32 mask, u32 data)
+{
+	int shift = 8 * (where & 3);
+	u32 val = pci_read_reg(pcie, where & ~3);
+	val &= ~(mask << shift);
+	val |= data << shift;
+	pci_write_reg(pcie, val, where & ~3);
+}
+
+static u32 rcar_read_conf(struct rcar_pcie *pcie, int where)
+{
+	int shift = 8 * (where & 3);
+	u32 val = pci_read_reg(pcie, where & ~3);
+	return val >> shift;
+}
+
+static int rcar_pcie_config_access(struct rcar_pcie *pcie,
+		unsigned char access_type, struct pci_bus *bus,
+		unsigned int devfn, int where, u32 *data)
+{
+	int dev, func, reg, index;
+
+	dev = PCI_SLOT(devfn);
+	func = PCI_FUNC(devfn);
+	reg = where & ~3;
+	index = reg / 4;
+
+	if (bus->number > 255 || dev > 31 || func > 7)
+		return PCIBIOS_FUNC_NOT_SUPPORTED;
+
+	/*
+	 * While each channel has its own memory-mapped extended config
+	 * space, it's generally only accessible when in endpoint mode.
+	 * When in root complex mode, the controller is unable to target
+	 * itself with either type 0 or type 1 accesses, and indeed, any
+	 * controller initiated target transfer to its own config space
+	 * result in a completer abort.
+	 *
+	 * Each channel effectively only supports a single device, but as
+	 * the same channel <-> device access works for any PCI_SLOT()
+	 * value, we cheat a bit here and bind the controller's config
+	 * space to devfn 0 in order to enable self-enumeration. In this
+	 * case the regular ECAR/ECDR path is sidelined and the mangled
+	 * config access itself is initiated as an internal bus  transaction.
+	 */
+	if (pci_is_root_bus(bus)) {
+		if (dev != 0)
+			return PCIBIOS_DEVICE_NOT_FOUND;
+
+		if (access_type == PCI_ACCESS_READ)
+			*data = pci_read_reg(pcie, PCICONF(index));
+		else
+			pci_write_reg(pcie, *data, PCICONF(index));
+
+		return PCIBIOS_SUCCESSFUL;
+	}
+
+	/* Clear errors */
+	pci_write_reg(pcie, pci_read_reg(pcie, PCIEERRFR), PCIEERRFR);
+
+	/* Set the PIO address */
+	pci_write_reg(pcie, PCIE_CONF_BUS(bus->number) | PCIE_CONF_DEV(dev) |
+				PCIE_CONF_FUNC(func) | reg, PCIECAR);
+
+	/* Enable the configuration access */
+	if (bus->parent->number == pcie->root_bus_nr)
+		pci_write_reg(pcie, CONFIG_SEND_ENABLE | TYPE0, PCIECCTLR);
+	else
+		pci_write_reg(pcie, CONFIG_SEND_ENABLE | TYPE1, PCIECCTLR);
+
+	/* Check for errors */
+	if (pci_read_reg(pcie, PCIEERRFR) & UNSUPPORTED_REQUEST)
+		return PCIBIOS_DEVICE_NOT_FOUND;
+
+	/* Check for master and target aborts */
+	if (rcar_read_conf(pcie, RCONF(PCI_STATUS)) &
+		(PCI_STATUS_REC_MASTER_ABORT | PCI_STATUS_REC_TARGET_ABORT))
+		return PCIBIOS_DEVICE_NOT_FOUND;
+
+	if (access_type == PCI_ACCESS_READ)
+		*data = pci_read_reg(pcie, PCIECDR);
+	else
+		pci_write_reg(pcie, *data, PCIECDR);
+
+	/* Disable the configuration access */
+	pci_write_reg(pcie, 0, PCIECCTLR);
+
+	return PCIBIOS_SUCCESSFUL;
+}
+
+static int rcar_pcie_read_conf(struct pci_bus *bus, unsigned int devfn,
+				int where, int size, u32 *val)
+{
+	struct rcar_pcie *pcie = sys_to_pcie(bus->sysdata);
+	int ret;
+
+	if ((size == 2) && (where & 1))
+		return PCIBIOS_BAD_REGISTER_NUMBER;
+	else if ((size == 4) && (where & 3))
+		return PCIBIOS_BAD_REGISTER_NUMBER;
+
+	ret = rcar_pcie_config_access(pcie, PCI_ACCESS_READ,
+				      bus, devfn, where, val);
+	if (ret != PCIBIOS_SUCCESSFUL) {
+		*val = 0xffffffff;
+		return ret;
+	}
+
+	if (size == 1)
+		*val = (*val >> (8 * (where & 3))) & 0xff;
+	else if (size == 2)
+		*val = (*val >> (8 * (where & 2))) & 0xffff;
+
+	dev_dbg(&bus->dev, "pcie-config-read: bus=%3d devfn=0x%04x "
+		"where=0x%04x size=%d val=0x%08lx\n", bus->number,
+		devfn, where, size, (unsigned long)*val);
+
+	return ret;
+}
+
+static int rcar_pcie_write_conf(struct pci_bus *bus, unsigned int devfn,
+				 int where, int size, u32 val)
+{
+	struct rcar_pcie *pcie = sys_to_pcie(bus->sysdata);
+	int shift, ret;
+	u32 data;
+
+	if ((size == 2) && (where & 1))
+		return PCIBIOS_BAD_REGISTER_NUMBER;
+	else if ((size == 4) && (where & 3))
+		return PCIBIOS_BAD_REGISTER_NUMBER;
+
+	ret = rcar_pcie_config_access(pcie, PCI_ACCESS_READ,
+				      bus, devfn, where, &data);
+	if (ret != PCIBIOS_SUCCESSFUL)
+		return ret;
+
+	dev_dbg(&bus->dev, "pcie-config-write: bus=%3d devfn=0x%04x "
+		"where=0x%04x size=%d val=0x%08lx\n", bus->number,
+		devfn, where, size, (unsigned long)val);
+
+	if (size == 1) {
+		shift = 8 * (where & 3);
+		data &= ~(0xff << shift);
+		data |= ((val & 0xff) << shift);
+	} else if (size == 2) {
+		shift = 8 * (where & 2);
+		data &= ~(0xffff << shift);
+		data |= ((val & 0xffff) << shift);
+	} else
+		data = val;
+
+	ret = rcar_pcie_config_access(pcie, PCI_ACCESS_WRITE,
+				      bus, devfn, where, &data);
+
+	return ret;
+}
+
+static struct pci_ops rcar_pcie_ops = {
+	.read	= rcar_pcie_read_conf,
+	.write	= rcar_pcie_write_conf,
+};
+
+static int rcar_pcie_setup_window(int win, struct resource *res,
+					struct rcar_pcie *pcie)
+{
+	/* Setup PCIe address space mappings for each resource */
+	resource_size_t size;
+	u32 mask;
+
+	pci_write_reg(pcie, 0x00000000, PCIEPTCTLR(win));
+
+	/*
+	 * The PAMR mask is calculated in units of 128Bytes, which
+	 * keeps things pretty simple.
+	 */
+	size = resource_size(res);
+	mask = (roundup_pow_of_two(size) / SZ_128) - 1;
+	pci_write_reg(pcie, mask << 7, PCIEPAMR(win));
+
+	pci_write_reg(pcie, upper_32_bits(res->start), PCIEPARH(win));
+	pci_write_reg(pcie, lower_32_bits(res->start), PCIEPARL(win));
+
+	/* First resource is for IO */
+	mask = PAR_ENABLE;
+	if (res->flags & IORESOURCE_IO)
+		mask |= IO_SPACE;
+
+	pci_write_reg(pcie, mask, PCIEPTCTLR(win));
+
+	return 0;
+}
+
+static int rcar_pcie_setup(int nr, struct pci_sys_data *sys)
+{
+	struct rcar_pcie *pcie = sys_to_pcie(sys);
+	struct resource *res;
+	int i;
+
+	pcie->root_bus_nr = sys->busnr;
+
+	/* Setup PCI resources */
+	for (i = 0; i < PCI_MAX_RESOURCES; i++) {
+
+		res = &pcie->res[i];
+		if (!res->flags)
+			continue;
+
+		rcar_pcie_setup_window(i, res, pcie);
+
+		if (res->flags & IORESOURCE_IO)
+			pci_ioremap_io(nr * SZ_64K, res->start);
+		else
+			pci_add_resource(&sys->resources, res);
+	}
+
+	return 1;
+}
+
+static void __init rcar_pcie_enable(struct rcar_pcie *pcie)
+{
+	struct platform_device *pdev = to_platform_device(pcie->dev);
+	struct hw_pci hw = {
+		.nr_controllers = 1,
+		.private_data   = (void **)&pcie,
+		.setup          = rcar_pcie_setup,
+		.map_irq        = of_irq_parse_and_map_pci,
+		.ops            = &rcar_pcie_ops,
+	};
+
+	pci_common_init_dev(&pdev->dev, &hw);
+}
+
+static int __init phy_wait_for_ack(struct rcar_pcie *pcie)
+{
+	unsigned int timeout = 100;
+
+	while (timeout--) {
+		if (pci_read_reg(pcie, H1_PCIEPHYADRR) & PHY_ACK)
+			return 0;
+
+		udelay(100);
+	}
+
+	dev_err(pcie->dev, "Access to PCIe phy timed out\n");
+
+	return -ETIMEDOUT;
+}
+
+static void __init phy_write_reg(struct rcar_pcie *pcie,
+				 unsigned int rate, unsigned int addr,
+				 unsigned int lane, unsigned int data)
+{
+	unsigned long phyaddr;
+
+	phyaddr = WRITE_CMD |
+		((rate & 1) << RATE_POS) |
+		((lane & 0xf) << LANE_POS) |
+		((addr & 0xff) << ADR_POS);
+
+	/* Set write data */
+	pci_write_reg(pcie, data, H1_PCIEPHYDOUTR);
+	pci_write_reg(pcie, phyaddr, H1_PCIEPHYADRR);
+
+	/* Ignore errors as they will be dealt with if the data link is down */
+	phy_wait_for_ack(pcie);
+
+	/* Clear command */
+	pci_write_reg(pcie, 0, H1_PCIEPHYDOUTR);
+	pci_write_reg(pcie, 0, H1_PCIEPHYADRR);
+
+	/* Ignore errors as they will be dealt with if the data link is down */
+	phy_wait_for_ack(pcie);
+}
+
+static int __init rcar_pcie_wait_for_dl(struct rcar_pcie *pcie)
+{
+	unsigned int timeout = 10;
+
+	while (timeout--) {
+		if ((pci_read_reg(pcie, PCIETSTR) & DATA_LINK_ACTIVE))
+			return 0;
+
+		msleep(5);
+	}
+
+	return -ETIMEDOUT;
+}
+
+static int __init rcar_pcie_hw_init(struct rcar_pcie *pcie)
+{
+	int err;
+
+	/* Begin initialization */
+	pci_write_reg(pcie, 0, PCIETCTLR);
+
+	/* Set mode */
+	pci_write_reg(pcie, 1, PCIEMSR);
+
+	/*
+	 * Initial header for port config space is type 1, set the device
+	 * class to match. Hardware takes care of propagating the IDSETR
+	 * settings, so there is no need to bother with a quirk.
+	 */
+	pci_write_reg(pcie, PCI_CLASS_BRIDGE_PCI << 16, IDSETR1);
+
+	/*
+	 * Setup Secondary Bus Number & Subordinate Bus Number, even though
+	 * they aren't used, to avoid bridge being detected as broken.
+	 */
+	rcar_rmw32(pcie, RCONF(PCI_SECONDARY_BUS), 0xff, 1);
+	rcar_rmw32(pcie, RCONF(PCI_SUBORDINATE_BUS), 0xff, 1);
+
+	/* Initialize default capabilities. */
+	rcar_rmw32(pcie, REXPCAP(0), 0, PCI_CAP_ID_EXP);
+	rcar_rmw32(pcie, REXPCAP(PCI_EXP_FLAGS),
+		PCI_EXP_FLAGS_TYPE, PCI_EXP_TYPE_ROOT_PORT << 4);
+	rcar_rmw32(pcie, RCONF(PCI_HEADER_TYPE), 0x7f,
+		PCI_HEADER_TYPE_BRIDGE);
+
+	/* Enable data link layer active state reporting */
+	rcar_rmw32(pcie, REXPCAP(PCI_EXP_LNKCAP), 0, PCI_EXP_LNKCAP_DLLLARC);
+
+	/* Write out the physical slot number = 0 */
+	rcar_rmw32(pcie, REXPCAP(PCI_EXP_SLTCAP), PCI_EXP_SLTCAP_PSN, 0);
+
+	/* Set the completion timer timeout to the maximum 50ms. */
+	rcar_rmw32(pcie, TLCTLR+1, 0x3f, 50);
+
+	/* Terminate list of capabilities (Next Capability Offset=0) */
+	rcar_rmw32(pcie, RVCCAP(0), 0xfff0, 0);
+
+	/* Enable MAC data scrambling. */
+	rcar_rmw32(pcie, MACCTLR, SCRAMBLE_DISABLE, 0);
+
+	/* Finish initialization - establish a PCI Express link */
+	pci_write_reg(pcie, CFINIT, PCIETCTLR);
+
+	/* This will timeout if we don't have a link. */
+	err = rcar_pcie_wait_for_dl(pcie);
+	if (err)
+		return err;
+
+	/* Enable INTx interrupts */
+	rcar_rmw32(pcie, PCIEINTXR, 0, 0xF << 8);
+
+	/* Enable slave Bus Mastering */
+	rcar_rmw32(pcie, RCONF(PCI_STATUS), PCI_STATUS_DEVSEL_MASK,
+		PCI_COMMAND_IO | PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER |
+		PCI_STATUS_CAP_LIST | PCI_STATUS_DEVSEL_FAST);
+
+	wmb();
+
+	return 0;
+}
+
+static int __init rcar_pcie_hw_init_h1(struct rcar_pcie *pcie)
+{
+	unsigned int timeout = 10;
+
+	/* Initialize the phy */
+	phy_write_reg(pcie, 0, 0x42, 0x1, 0x0EC34191);
+	phy_write_reg(pcie, 1, 0x42, 0x1, 0x0EC34180);
+	phy_write_reg(pcie, 0, 0x43, 0x1, 0x00210188);
+	phy_write_reg(pcie, 1, 0x43, 0x1, 0x00210188);
+	phy_write_reg(pcie, 0, 0x44, 0x1, 0x015C0014);
+	phy_write_reg(pcie, 1, 0x44, 0x1, 0x015C0014);
+	phy_write_reg(pcie, 1, 0x4C, 0x1, 0x786174A0);
+	phy_write_reg(pcie, 1, 0x4D, 0x1, 0x048000BB);
+	phy_write_reg(pcie, 0, 0x51, 0x1, 0x079EC062);
+	phy_write_reg(pcie, 0, 0x52, 0x1, 0x20000000);
+	phy_write_reg(pcie, 1, 0x52, 0x1, 0x20000000);
+	phy_write_reg(pcie, 1, 0x56, 0x1, 0x00003806);
+
+	phy_write_reg(pcie, 0, 0x60, 0x1, 0x004B03A5);
+	phy_write_reg(pcie, 0, 0x64, 0x1, 0x3F0F1F0F);
+	phy_write_reg(pcie, 0, 0x66, 0x1, 0x00008000);
+
+	while (timeout--) {
+		if (pci_read_reg(pcie, H1_PCIEPHYSR))
+			return rcar_pcie_hw_init(pcie);
+
+		msleep(5);
+	}
+
+	return -ETIMEDOUT;
+}
+
+static int __init rcar_pcie_get_resources(struct platform_device *pdev,
+	struct rcar_pcie *pcie)
+{
+	struct resource res;
+	int err;
+
+	err = of_address_to_resource(pdev->dev.of_node, 0, &res);
+	if (err)
+		return err;
+
+	pcie->clk = devm_clk_get(&pdev->dev, "pcie");
+	if (IS_ERR(pcie->clk)) {
+		dev_err(pcie->dev, "cannot get platform clock\n");
+		return PTR_ERR(pcie->clk);
+	}
+	err = clk_prepare_enable(pcie->clk);
+	if (err)
+		goto fail_clk;
+
+	pcie->bus_clk = devm_clk_get(&pdev->dev, "pcie_bus");
+	if (IS_ERR(pcie->bus_clk)) {
+		dev_err(pcie->dev, "cannot get pcie bus clock\n");
+		err = PTR_ERR(pcie->bus_clk);
+		goto fail_clk;
+	}
+	err = clk_prepare_enable(pcie->bus_clk);
+	if (err)
+		goto err_map_reg;
+
+	pcie->base = devm_ioremap_resource(&pdev->dev, &res);
+	if (IS_ERR(pcie->base)) {
+		err = PTR_ERR(pcie->base);
+		goto err_map_reg;
+	}
+
+	return 0;
+
+err_map_reg:
+	clk_disable_unprepare(pcie->bus_clk);
+fail_clk:
+	clk_disable_unprepare(pcie->clk);
+
+	return err;
+}
+
+static int rcar_pcie_inbound_ranges(struct rcar_pcie *pcie,
+				    struct of_pci_range *range,
+				    int *index)
+{
+	u64 restype = range->flags;
+	u64 cpu_addr = range->cpu_addr;
+	u64 cpu_end = range->cpu_addr + range->size;
+	u64 pci_addr = range->pci_addr;
+	u32 flags = LAM_64BIT | LAR_ENABLE;
+	u64 mask;
+	u64 size;
+	int idx = *index;
+
+	if (restype & IORESOURCE_PREFETCH)
+		flags |= LAM_PREFETCH;
+
+	/*
+	 * If the size of the range is larger than the alignment of the start
+	 * address, we have to use multiple entries to perform the mapping.
+	 */
+	if (cpu_addr > 0) {
+		unsigned long nr_zeros = __ffs64(cpu_addr);
+		u64 alignment = 1ULL << nr_zeros;
+		size = min(range->size, alignment);
+	} else {
+		size = range->size;
+	}
+	/* Hardware supports max 4GiB inbound region */
+	size = min(size, 1ULL << 32);
+
+	mask = roundup_pow_of_two(size) - 1;
+	mask &= ~0xf;
+
+	while (cpu_addr < cpu_end) {
+		/*
+		 * Set up 64-bit inbound regions as the range parser doesn't
+		 * distinguish between 32 and 64-bit types.
+		 */
+		pci_write_reg(pcie, lower_32_bits(pci_addr), PCIEPRAR(idx));
+		pci_write_reg(pcie, lower_32_bits(cpu_addr), PCIELAR(idx));
+		pci_write_reg(pcie, lower_32_bits(mask) | flags, PCIELAMR(idx));
+
+		pci_write_reg(pcie, upper_32_bits(pci_addr), PCIEPRAR(idx+1));
+		pci_write_reg(pcie, upper_32_bits(cpu_addr), PCIELAR(idx+1));
+		pci_write_reg(pcie, 0, PCIELAMR(idx+1));
+
+		pci_addr += size;
+		cpu_addr += size;
+		idx += 2;
+
+		if (idx > MAX_NR_INBOUND_MAPS) {
+			dev_err(pcie->dev, "Failed to map inbound regions!\n");
+			return -EINVAL;
+		}
+	}
+	*index = idx;
+
+	return 0;
+}
+
+static int pci_dma_range_parser_init(struct of_pci_range_parser *parser,
+				     struct device_node *node)
+{
+	const int na = 3, ns = 2;
+	int rlen;
+
+	parser->node = node;
+	parser->pna = of_n_addr_cells(node);
+	parser->np = parser->pna + na + ns;
+
+	parser->range = of_get_property(node, "dma-ranges", &rlen);
+	if (!parser->range)
+		return -ENOENT;
+
+	parser->end = parser->range + rlen / sizeof(__be32);
+	return 0;
+}
+
+static int rcar_pcie_parse_map_dma_ranges(struct rcar_pcie *pcie,
+					  struct device_node *np)
+{
+	struct of_pci_range range;
+	struct of_pci_range_parser parser;
+	int index = 0;
+	int err;
+
+	if (pci_dma_range_parser_init(&parser, np))
+		return -EINVAL;
+
+	/* Get the dma-ranges from DT */
+	for_each_of_pci_range(&parser, &range) {
+		u64 end = range.cpu_addr + range.size - 1;
+		dev_dbg(pcie->dev, "0x%08x 0x%016llx..0x%016llx -> 0x%016llx\n",
+			range.flags, range.cpu_addr, end, range.pci_addr);
+
+		err = rcar_pcie_inbound_ranges(pcie, &range, &index);
+		if (err)
+			return err;
+	}
+
+	return 0;
+}
+
+static const struct of_device_id rcar_pcie_of_match[] = {
+	{ .compatible = "renesas,pcie-r8a7779", .data = rcar_pcie_hw_init_h1 },
+	{ .compatible = "renesas,pcie-r8a7790", .data = rcar_pcie_hw_init },
+	{ .compatible = "renesas,pcie-r8a7791", .data = rcar_pcie_hw_init },
+	{},
+};
+MODULE_DEVICE_TABLE(of, rcar_pcie_of_match);
+
+static int __init rcar_pcie_probe(struct platform_device *pdev)
+{
+	struct rcar_pcie *pcie;
+	unsigned int data;
+	struct of_pci_range range;
+	struct of_pci_range_parser parser;
+	const struct of_device_id *of_id;
+	int err, win = 0;
+	int (*hw_init_fn)(struct rcar_pcie *);
+
+	pcie = devm_kzalloc(&pdev->dev, sizeof(*pcie), GFP_KERNEL);
+	if (!pcie)
+		return -ENOMEM;
+
+	pcie->dev = &pdev->dev;
+	platform_set_drvdata(pdev, pcie);
+
+	if (of_pci_range_parser_init(&parser, pdev->dev.of_node)) {
+		dev_err(&pdev->dev, "missing ranges property\n");
+		return -EINVAL;
+	}
+
+	err = rcar_pcie_get_resources(pdev, pcie);
+	if (err < 0) {
+		dev_err(&pdev->dev, "failed to request resources: %d\n", err);
+		return err;
+	}
+
+	for_each_of_pci_range(&parser, &range) {
+		of_pci_range_to_resource(&range, pdev->dev.of_node,
+						&pcie->res[win++]);
+
+		if (win > PCI_MAX_RESOURCES)
+			break;
+	}
+
+	 err = rcar_pcie_parse_map_dma_ranges(pcie, pdev->dev.of_node);
+	 if (err)
+		return err;
+
+	of_id = of_match_device(rcar_pcie_of_match, pcie->dev);
+	if (!of_id || !of_id->data)
+		return -EINVAL;
+	hw_init_fn = of_id->data;
+
+	/* Failure to get a link might just be that no cards are inserted */
+	err = hw_init_fn(pcie);
+	if (err) {
+		dev_info(&pdev->dev, "PCIe link down\n");
+		return 0;
+	}
+
+	data = pci_read_reg(pcie, MACSR);
+	dev_info(&pdev->dev, "PCIe x%d: link up\n", (data >> 20) & 0x3f);
+
+	rcar_pcie_enable(pcie);
+
+	return 0;
+}
+
+static struct platform_driver rcar_pcie_driver = {
+	.driver = {
+		.name = DRV_NAME,
+		.owner = THIS_MODULE,
+		.of_match_table = rcar_pcie_of_match,
+		.suppress_bind_attrs = true,
+	},
+	.probe = rcar_pcie_probe,
+};
+module_platform_driver(rcar_pcie_driver);
+
+MODULE_AUTHOR("Phil Edworthy <phil.edworthy@renesas.com>");
+MODULE_DESCRIPTION("Renesas R-Car PCIe driver");
+MODULE_LICENSE("GPLv2");
diff --git a/drivers/pci/host/pcie-rcar.h b/drivers/pci/host/pcie-rcar.h
new file mode 100644
index 0000000..3dc026b
--- /dev/null
+++ b/drivers/pci/host/pcie-rcar.h
@@ -0,0 +1,82 @@ 
+/*
+ * PCI Express definitions for Renesas R-Car SoCs
+ */
+#ifndef __PCI_RCAR_H
+#define __PCI_RCAR_H
+
+#define PCIECAR			0x000010
+#define PCIECCTLR		0x000018
+#define  CONFIG_SEND_ENABLE	(1 << 31)
+#define  TYPE0			(0 << 8)
+#define  TYPE1			(1 << 8)
+#define PCIECDR			0x000020
+#define PCIEMSR			0x000028
+#define PCIEINTXR		0x000400
+#define PCIEPHYSR		0x0007f0
+
+/* Transfer control */
+#define PCIETCTLR		0x02000
+#define  CFINIT			1
+#define PCIETSTR		0x02004
+#define  DATA_LINK_ACTIVE	1
+#define PCIEINTR		0x02008
+#define PCIEINTER		0x0200c
+#define PCIEERRFR		0x02020
+#define  UNSUPPORTED_REQUEST	(1 << 4)
+#define PCIEERRFER		0x02024
+#define PCIEERRFR2		0x02028
+#define PCIEPMSR		0x02034
+#define PCIEPMSCIER		0x02038
+#define PCIEMSIFR		0x02044
+
+/* root port address */
+#define PCIEPRAR(x)		(0x02080 + ((x) * 0x4))
+
+/* local address reg & mask */
+#define PCIELAR(x)		(0x02200 + ((x) * 0x20))
+#define PCIELAMR(x)		(0x02208 + ((x) * 0x20))
+#define  LAM_PMIOLAMnB3		(1 << 3)
+#define  LAM_PMIOLAMnB2		(1 << 2)
+#define  LAM_PREFETCH		(1 << 3)
+#define  LAM_64BIT		(1 << 2)
+#define  LAR_ENABLE		(1 << 1)
+
+/* PCIe address reg & mask */
+#define PCIEPARL(x)		(0x03400 + ((x) * 0x20))
+#define PCIEPARH(x)		(0x03404 + ((x) * 0x20))
+#define PCIEPAMR(x)		(0x03408 + ((x) * 0x20))
+#define PCIEPTCTLR(x)		(0x0340c + ((x) * 0x20))
+#define  PAR_ENABLE		(1 << 31)
+#define  IO_SPACE		(1 << 8)
+
+/* Configuration */
+#define PCICONF(x)		(0x010000 + ((x) * 0x4))
+#define PMCAP(x)		(0x010040 + ((x) * 0x4))
+#define MSICAP(x)		(0x010050 + ((x) * 0x4))
+#define EXPCAP(x)		(0x010070 + ((x) * 0x4))
+#define VCCAP(x)		(0x010100 + ((x) * 0x4))
+#define SERNUMCAP(x)		(0x0101b0 + ((x) * 0x4))
+
+/* link layer */
+#define IDSETR0			0x011000
+#define IDSETR1			0x011004
+#define SUBIDSETR		0x011024
+#define DSERSETR0		0x01102c
+#define DSERSETR1		0x011030
+#define TLCTLR			0x011048
+#define MACSR			0x011054
+#define MACCTLR			0x011058
+#define  SCRAMBLE_DISABLE	(1 << 27)
+
+/* R-Car H1 PHY */
+#define H1_PCIEPHYCTLR		0x040008
+#define H1_PCIEPHYADRR		0x04000c
+#define  WRITE_CMD		(1 << 16)
+#define  PHY_ACK		(1 << 24)
+#define  RATE_POS		12
+#define  LANE_POS		8
+#define  ADR_POS		0
+#define H1_PCIEPHYDOUTR		0x040014
+#define H1_PCIEPHYSR		0x040018
+
+#endif /* __PCI_RCAR_H */