From: Jon Mason <jdmason@kudzu.us>
To: Muli Ben-Yehuda <muli@il.ibm.com>
Cc: Jeff Garzik <jeff@garzik.org>, Andrew Morton <akpm@osdl.org>,
Greg KH <greg@kroah.com>, Jim Paradis <jparadis@redhat.com>,
Andi Kleen <ak@suse.de>, LKML <linux-kernel@vger.kernel.org>
Subject: Re: [PATCH] x86[-64] PCI domain support
Date: Fri, 29 Sep 2006 08:43:31 -0500 [thread overview]
Message-ID: <20060929134330.GA1687@kudzu.us> (raw)
In-Reply-To: <20060928224550.GJ22787@rhun.haifa.ibm.com>
On Fri, Sep 29, 2006 at 01:45:50AM +0300, Muli Ben-Yehuda wrote:
> On Thu, Sep 28, 2006 at 05:45:41AM -0400, Jeff Garzik wrote:
> > Muli Ben-Yehuda wrote:
> > >On Tue, Sep 26, 2006 at 03:15:08PM -0400, Jeff Garzik wrote:
> > >>The x86[-64] PCI domain effort needs to be restarted, because we've got
> > >>machines out in the field that need this in order for some devices to
> > >>work.
> > >>
> > >This breaks the Calgary IOMMU, since it uses sysdata for other
> > >purposes (going back from a bus to its IO address space). I'm looking
> > >into it.
> >
> > You'll need to modify struct pci_sysdata in
> > include/asm-{i386,x86_64}/pci.h to include the data that you previously
> > stored directly into the sysdata pointer.
>
> Something like this should do the trick. Note - this should not be
> applied yet - after several gigabytes of network and disk activity it
> takes aic94xx down. More investigation required.
Nak! The calgary code should not be affected by these patches. The
PCI domain code uses the sysdata pointer from struct pci_bus, where as
calgary uses the sysdata pointer from struct pci_dev. This should not
be an issue. Can you confirm actual breakage?
The sysdata pointer from struct pci_bus is used by NUMA, which we found
out when we tried to use it for our calgary code initially. I'm sure
someone has been checking that NUMA works with these PCI domain
patches....
Thanks,
Jon
>
> diff -Naurp -X /home/muli/w/dontdiff pci-domains/arch/x86_64/kernel/pci-calgary.c linux/arch/x86_64/kernel/pci-calgary.c
> --- pci-domains/arch/x86_64/kernel/pci-calgary.c 2006-09-28 13:31:14.000000000 +0300
> +++ linux/arch/x86_64/kernel/pci-calgary.c 2006-09-28 13:14:38.000000000 +0300
> @@ -317,7 +317,7 @@ void calgary_unmap_sg(struct device *dev
> int nelems, int direction)
> {
> unsigned long flags;
> - struct iommu_table *tbl = to_pci_dev(dev)->bus->self->sysdata;
> + struct iommu_table *tbl = pci_iommu(to_pci_dev(dev)->bus);
>
> if (!translate_phb(to_pci_dev(dev)))
> return;
> @@ -346,7 +346,7 @@ static int calgary_nontranslate_map_sg(s
> int calgary_map_sg(struct device *dev, struct scatterlist *sg,
> int nelems, int direction)
> {
> - struct iommu_table *tbl = to_pci_dev(dev)->bus->self->sysdata;
> + struct iommu_table *tbl = pci_iommu(to_pci_dev(dev)->bus);
> unsigned long flags;
> unsigned long vaddr;
> unsigned int npages;
> @@ -400,7 +400,7 @@ dma_addr_t calgary_map_single(struct dev
> dma_addr_t dma_handle = bad_dma_address;
> unsigned long uaddr;
> unsigned int npages;
> - struct iommu_table *tbl = to_pci_dev(dev)->bus->self->sysdata;
> + struct iommu_table *tbl = pci_iommu(to_pci_dev(dev)->bus);
>
> uaddr = (unsigned long)vaddr;
> npages = num_dma_pages(uaddr, size);
> @@ -416,7 +416,7 @@ dma_addr_t calgary_map_single(struct dev
> void calgary_unmap_single(struct device *dev, dma_addr_t dma_handle,
> size_t size, int direction)
> {
> - struct iommu_table *tbl = to_pci_dev(dev)->bus->self->sysdata;
> + struct iommu_table *tbl = pci_iommu(to_pci_dev(dev)->bus);
> unsigned int npages;
>
> if (!translate_phb(to_pci_dev(dev)))
> @@ -434,7 +434,7 @@ void* calgary_alloc_coherent(struct devi
> unsigned int npages, order;
> struct iommu_table *tbl;
>
> - tbl = to_pci_dev(dev)->bus->self->sysdata;
> + tbl = pci_iommu(to_pci_dev(dev)->bus);
>
> size = PAGE_ALIGN(size); /* size rounded up to full pages */
> npages = size >> PAGE_SHIFT;
> @@ -551,7 +551,7 @@ static void __init calgary_reserve_mem_r
> limit++;
>
> numpages = ((limit - start) >> PAGE_SHIFT);
> - iommu_range_reserve(dev->sysdata, start, numpages);
> + iommu_range_reserve(pci_iommu(dev->bus), start, numpages);
> }
>
> static void __init calgary_reserve_peripheral_mem_1(struct pci_dev *dev)
> @@ -559,7 +559,7 @@ static void __init calgary_reserve_perip
> void __iomem *target;
> u64 low, high, sizelow;
> u64 start, limit;
> - struct iommu_table *tbl = dev->sysdata;
> + struct iommu_table *tbl = pci_iommu(dev->bus);
> unsigned char busnum = dev->bus->number;
> void __iomem *bbar = tbl->bbar;
>
> @@ -583,7 +583,7 @@ static void __init calgary_reserve_perip
> u32 val32;
> u64 low, high, sizelow, sizehigh;
> u64 start, limit;
> - struct iommu_table *tbl = dev->sysdata;
> + struct iommu_table *tbl = pci_iommu(dev->bus);
> unsigned char busnum = dev->bus->number;
> void __iomem *bbar = tbl->bbar;
>
> @@ -621,7 +621,7 @@ static void __init calgary_reserve_regio
> void __iomem *bbar;
> unsigned char busnum;
> u64 start;
> - struct iommu_table *tbl = dev->sysdata;
> + struct iommu_table *tbl = pci_iommu(dev->bus);
>
> bbar = tbl->bbar;
> busnum = dev->bus->number;
> @@ -652,7 +652,7 @@ static int __init calgary_setup_tar(stru
> if (ret)
> return ret;
>
> - tbl = dev->sysdata;
> + tbl = pci_iommu(dev->bus);
> tbl->it_base = (unsigned long)bus_info[dev->bus->number].tce_space;
> tce_free(tbl, 0, tbl->it_size);
>
> @@ -665,7 +665,7 @@ static int __init calgary_setup_tar(stru
> /* zero out all TAR bits under sw control */
> val64 &= ~TAR_SW_BITS;
>
> - tbl = dev->sysdata;
> + tbl = pci_iommu(dev->bus);
> table_phys = (u64)__pa(tbl->it_base);
> val64 |= table_phys;
>
> @@ -682,7 +682,7 @@ static int __init calgary_setup_tar(stru
> static void __init calgary_free_bus(struct pci_dev *dev)
> {
> u64 val64;
> - struct iommu_table *tbl = dev->sysdata;
> + struct iommu_table *tbl = pci_iommu(dev->bus);
> void __iomem *target;
> unsigned int bitmapsz;
>
> @@ -697,7 +697,8 @@ static void __init calgary_free_bus(stru
> tbl->it_map = NULL;
>
> kfree(tbl);
> - dev->sysdata = NULL;
> +
> + set_pci_iommu(dev->bus, NULL);
>
> /* Can't free bootmem allocated memory after system is up :-( */
> bus_info[dev->bus->number].tce_space = NULL;
> @@ -706,7 +707,7 @@ static void __init calgary_free_bus(stru
> static void calgary_watchdog(unsigned long data)
> {
> struct pci_dev *dev = (struct pci_dev *)data;
> - struct iommu_table *tbl = dev->sysdata;
> + struct iommu_table *tbl = pci_iommu(dev->bus);
> void __iomem *bbar = tbl->bbar;
> u32 val32;
> void __iomem *target;
> @@ -742,7 +743,7 @@ static void __init calgary_enable_transl
> struct iommu_table *tbl;
>
> busnum = dev->bus->number;
> - tbl = dev->sysdata;
> + tbl = pci_iommu(dev->bus);
> bbar = tbl->bbar;
>
> /* enable TCE in PHB Config Register */
> @@ -772,7 +773,7 @@ static void __init calgary_disable_trans
> struct iommu_table *tbl;
>
> busnum = dev->bus->number;
> - tbl = dev->sysdata;
> + tbl = pci_iommu(dev->bus);
> bbar = tbl->bbar;
>
> /* disable TCE in PHB Config Register */
> @@ -813,8 +814,7 @@ static inline unsigned int __init locate
> static void __init calgary_init_one_nontraslated(struct pci_dev *dev)
> {
> pci_dev_get(dev);
> - dev->sysdata = NULL;
> - dev->bus->self = dev;
> + set_pci_iommu(dev->bus, NULL);
> }
>
> static int __init calgary_init_one(struct pci_dev *dev)
> diff -Naurp -X /home/muli/w/dontdiff pci-domains/arch/x86_64/kernel/tce.c linux/arch/x86_64/kernel/tce.c
> --- pci-domains/arch/x86_64/kernel/tce.c 2006-09-28 13:31:14.000000000 +0300
> +++ linux/arch/x86_64/kernel/tce.c 2006-09-28 13:40:52.000000000 +0300
> @@ -137,9 +137,9 @@ int build_tce_table(struct pci_dev *dev,
> struct iommu_table *tbl;
> int ret;
>
> - if (dev->sysdata) {
> - printk(KERN_ERR "Calgary: dev %p has sysdata %p\n",
> - dev, dev->sysdata);
> + if (pci_iommu(dev->bus)) {
> + printk(KERN_ERR "Calgary: dev %p has sysdata->iommu %p\n",
> + dev, pci_iommu(dev->bus));
> BUG();
> }
>
> @@ -156,11 +156,7 @@ int build_tce_table(struct pci_dev *dev,
>
> tbl->bbar = bbar;
>
> - /*
> - * NUMA is already using the bus's sysdata pointer, so we use
> - * the bus's pci_dev's sysdata instead.
> - */
> - dev->sysdata = tbl;
> + set_pci_iommu(dev->bus, tbl);
>
> return 0;
>
> diff -Naurp -X /home/muli/w/dontdiff pci-domains/include/asm-x86_64/pci.h linux/include/asm-x86_64/pci.h
> --- pci-domains/include/asm-x86_64/pci.h 2006-09-28 13:34:05.000000000 +0300
> +++ linux/include/asm-x86_64/pci.h 2006-09-28 15:19:56.000000000 +0300
> @@ -8,6 +8,7 @@
> struct pci_sysdata {
> int domain; /* PCI domain */
> int node; /* NUMA node */
> + void* iommu; /* IOMMU private data */
> };
>
> #ifdef CONFIG_PCI_DOMAINS
> @@ -23,6 +24,20 @@ static inline int pci_proc_domain(struct
> }
> #endif /* CONFIG_PCI_DOMAINS */
>
> +#ifdef CONFIG_CALGARY_IOMMU
> +static inline void* pci_iommu(struct pci_bus *bus)
> +{
> + struct pci_sysdata *sd = bus->sysdata;
> + return sd->iommu;
> +}
> +
> +static inline void set_pci_iommu(struct pci_bus *bus, void *val)
> +{
> + struct pci_sysdata *sd = bus->sysdata;
> + sd->iommu = val;
> +}
> +#endif /* CONFIG_CALGARY_IOMMU */
> +
> #include <linux/mm.h> /* for struct page */
>
> /* Can be used to override the logic in pci_scan_bus for skipping
>
next prev parent reply other threads:[~2006-09-29 13:40 UTC|newest]
Thread overview: 21+ messages / expand[flat|nested] mbox.gz Atom feed top
2006-09-26 19:15 [PATCH] x86[-64] PCI domain support Jeff Garzik
2006-09-26 20:23 ` Greg KH
2006-09-26 20:27 ` Andi Kleen
2006-09-26 20:44 ` Jeff Garzik
2006-09-27 4:40 ` Greg KH
2006-09-27 7:28 ` Rolf Eike Beer
2006-09-28 9:33 ` Muli Ben-Yehuda
2006-09-28 9:45 ` Jeff Garzik
2006-09-28 22:45 ` Muli Ben-Yehuda
2006-09-28 23:03 ` Jeff Garzik
2006-09-28 23:31 ` Muli Ben-Yehuda
2006-09-30 9:34 ` Muli Ben-Yehuda
2006-09-30 10:03 ` Jeff Garzik
2006-09-30 10:42 ` Muli Ben-Yehuda
2006-09-30 11:12 ` Jeff Garzik
2006-09-30 11:41 ` Jeff Garzik
2006-09-30 17:51 ` [PATCH] x86-64: Calgary IOMMU: update to work with PCI domains Muli Ben-Yehuda
2006-10-01 3:41 ` Jeff Garzik
2006-09-29 13:43 ` Jon Mason [this message]
2006-09-29 17:11 ` [PATCH] x86[-64] PCI domain support Muli Ben-Yehuda
2006-09-29 18:24 ` Jon Mason
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20060929134330.GA1687@kudzu.us \
--to=jdmason@kudzu.us \
--cc=ak@suse.de \
--cc=akpm@osdl.org \
--cc=greg@kroah.com \
--cc=jeff@garzik.org \
--cc=jparadis@redhat.com \
--cc=linux-kernel@vger.kernel.org \
--cc=muli@il.ibm.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox