[Linux-ia64] [PATCH] 1/4 multi-ioport space support for 2.5

[Linux-ia64] [PATCH] 1/4 multi-ioport space support for 2.5

[Bjorn Helgaas]

This has been in my 2.4 BK tree for a while, but I should have
posted it in case there's feedback from other people working
on large machines.  So here it is, in four parts:

  1  enhance __ia64_mk_io_addr(port)
  2  enhance pcibios_scan_root to get multiple mem & io windows
      from ACPI _CRS, and fixup all the resources
  3  add support for /proc/iomem and /proc/ioports
  4  trivial (whitespace, copyright, and move pcibios_fixup_device_resources
       closer to related code)

The current scheme is that IO ports are 64 bits, with the low 24
bits being the port number within an IO port space, and the upper
bits identifying the space.  There is currently a limit of 16
spaces.

Bjorn

diff -u -ur linux-2.5.67-ia64-030416/arch/ia64/kernel/setup.c linux-2.5.67-ia64-030416-io1/arch/ia64/kernel/setup.c
--- linux-2.5.67-ia64-030416/arch/ia64/kernel/setup.c	2003-04-24 10:10:58.000000000 -0600
+++ linux-2.5.67-ia64-030416-io1/arch/ia64/kernel/setup.c	2003-04-24 11:01:40.000000000 -0600
@@ -63,6 +63,8 @@
 struct screen_info screen_info;
 
 unsigned long ia64_iobase;	/* virtual address for I/O accesses */
+struct io_space io_space[MAX_IO_SPACES];
+unsigned int num_io_spaces;
 
 unsigned char aux_device_present = 0xaa;        /* XXX remove this when legacy I/O is gone */
 
@@ -415,6 +417,11 @@
 	}
 	ia64_iobase = (unsigned long) ioremap(phys_iobase, 0);
 
+	/* setup legacy IO port space */
+	io_space[0].mmio_base = ia64_iobase;
+	io_space[0].sparse = 1;
+	num_io_spaces = 1;
+
 #ifdef CONFIG_SMP
 	cpu_physical_id(0) = hard_smp_processor_id();
 #endif
diff -u -ur linux-2.5.67-ia64-030416/include/asm-ia64/io.h linux-2.5.67-ia64-030416-io1/include/asm-ia64/io.h
--- linux-2.5.67-ia64-030416/include/asm-ia64/io.h	2003-04-24 10:10:58.000000000 -0600
+++ linux-2.5.67-ia64-030416-io1/include/asm-ia64/io.h	2003-04-24 11:29:59.000000000 -0600
@@ -32,6 +32,24 @@
  */
 #define IO_SPACE_LIMIT		0xffffffffffffffffUL
 
+#define MAX_IO_SPACES			16
+#define IO_SPACE_BITS			24
+#define IO_SPACE_SIZE			(1UL << IO_SPACE_BITS)
+
+#define IO_SPACE_NR(port)		((port) >> IO_SPACE_BITS)
+#define IO_SPACE_BASE(space)		((space) << IO_SPACE_BITS)
+#define IO_SPACE_PORT(port)		((port) & (IO_SPACE_SIZE - 1))
+
+#define IO_SPACE_SPARSE_ENCODING(p)	((((p) >> 2) << 12) | (p & 0xfff))
+
+struct io_space {
+	unsigned long mmio_base;	/* base in MMIO space */
+	int sparse;
+};
+
+extern struct io_space io_space[];
+extern unsigned int num_io_spaces;
+
 # ifdef __KERNEL__
 
 #include <asm/machvec.h>
@@ -80,11 +98,17 @@
 static inline void*
 __ia64_mk_io_addr (unsigned long port)
 {
-	const unsigned long io_base = __ia64_get_io_port_base();
-	unsigned long addr;
+	struct io_space *space;
+	unsigned long offset;
+
+	space = &io_space[IO_SPACE_NR(port)];
+	port = IO_SPACE_PORT(port);
+	if (space->sparse)
+		offset = IO_SPACE_SPARSE_ENCODING(port);
+	else
+		offset = port;
 
-	addr = io_base | ((port >> 2) << 12) | (port & 0xfff);
-	return (void *) addr;
+	return (void *) (space->mmio_base | offset);
 }
 
 /*

Re: [Linux-ia64] [PATCH] 1/4 multi-ioport space support for 2.5

[Jesse Barnes]

Bjorn, what about the patches posted in this thread?

http://marc.theaimsgroup.com/?t\x104359481700001&r=1&w=2

They seem to deal with the legacy resource issue quite nicely, and for
the non-legacy case, pci_fixup should be making the
pci_dev->resource[] arrays contain valid I/O addresss, shouldn't it?

Or do you have something else in mind?

Thanks,
Jesse

On Thu, Apr 24, 2003 at 04:25:26PM -0600, Bjorn Helgaas wrote:
> This has been in my 2.4 BK tree for a while, but I should have
> posted it in case there's feedback from other people working
> on large machines.  So here it is, in four parts:
> 
>   1  enhance __ia64_mk_io_addr(port)
>   2  enhance pcibios_scan_root to get multiple mem & io windows
>       from ACPI _CRS, and fixup all the resources
>   3  add support for /proc/iomem and /proc/ioports
>   4  trivial (whitespace, copyright, and move pcibios_fixup_device_resources
>        closer to related code)
> 
> The current scheme is that IO ports are 64 bits, with the low 24
> bits being the port number within an IO port space, and the upper
> bits identifying the space.  There is currently a limit of 16
> spaces.
> 
> Bjorn
> 
> diff -u -ur linux-2.5.67-ia64-030416/arch/ia64/kernel/setup.c linux-2.5.67-ia64-030416-io1/arch/ia64/kernel/setup.c
> --- linux-2.5.67-ia64-030416/arch/ia64/kernel/setup.c	2003-04-24 10:10:58.000000000 -0600
> +++ linux-2.5.67-ia64-030416-io1/arch/ia64/kernel/setup.c	2003-04-24 11:01:40.000000000 -0600
> @@ -63,6 +63,8 @@
>  struct screen_info screen_info;
>  
>  unsigned long ia64_iobase;	/* virtual address for I/O accesses */
> +struct io_space io_space[MAX_IO_SPACES];
> +unsigned int num_io_spaces;
>  
>  unsigned char aux_device_present = 0xaa;        /* XXX remove this when legacy I/O is gone */
>  
> @@ -415,6 +417,11 @@
>  	}
>  	ia64_iobase = (unsigned long) ioremap(phys_iobase, 0);
>  
> +	/* setup legacy IO port space */
> +	io_space[0].mmio_base = ia64_iobase;
> +	io_space[0].sparse = 1;
> +	num_io_spaces = 1;
> +
>  #ifdef CONFIG_SMP
>  	cpu_physical_id(0) = hard_smp_processor_id();
>  #endif
> diff -u -ur linux-2.5.67-ia64-030416/include/asm-ia64/io.h linux-2.5.67-ia64-030416-io1/include/asm-ia64/io.h
> --- linux-2.5.67-ia64-030416/include/asm-ia64/io.h	2003-04-24 10:10:58.000000000 -0600
> +++ linux-2.5.67-ia64-030416-io1/include/asm-ia64/io.h	2003-04-24 11:29:59.000000000 -0600
> @@ -32,6 +32,24 @@
>   */
>  #define IO_SPACE_LIMIT		0xffffffffffffffffUL
>  
> +#define MAX_IO_SPACES			16
> +#define IO_SPACE_BITS			24
> +#define IO_SPACE_SIZE			(1UL << IO_SPACE_BITS)
> +
> +#define IO_SPACE_NR(port)		((port) >> IO_SPACE_BITS)
> +#define IO_SPACE_BASE(space)		((space) << IO_SPACE_BITS)
> +#define IO_SPACE_PORT(port)		((port) & (IO_SPACE_SIZE - 1))
> +
> +#define IO_SPACE_SPARSE_ENCODING(p)	((((p) >> 2) << 12) | (p & 0xfff))
> +
> +struct io_space {
> +	unsigned long mmio_base;	/* base in MMIO space */
> +	int sparse;
> +};
> +
> +extern struct io_space io_space[];
> +extern unsigned int num_io_spaces;
> +
>  # ifdef __KERNEL__
>  
>  #include <asm/machvec.h>
> @@ -80,11 +98,17 @@
>  static inline void*
>  __ia64_mk_io_addr (unsigned long port)
>  {
> -	const unsigned long io_base = __ia64_get_io_port_base();
> -	unsigned long addr;
> +	struct io_space *space;
> +	unsigned long offset;
> +
> +	space = &io_space[IO_SPACE_NR(port)];
> +	port = IO_SPACE_PORT(port);
> +	if (space->sparse)
> +		offset = IO_SPACE_SPARSE_ENCODING(port);
> +	else
> +		offset = port;
>  
> -	addr = io_base | ((port >> 2) << 12) | (port & 0xfff);
> -	return (void *) addr;
> +	return (void *) (space->mmio_base | offset);
>  }
>  
>  /*
> 
> 
> _______________________________________________
> Linux-IA64 mailing list
> Linux-IA64@linuxia64.org
> http://lists.linuxia64.org/lists/listinfo/linux-ia64

Re: [Linux-ia64] [PATCH] 1/4 multi-ioport space support for 2.5

[Bjorn Helgaas]

On Thursday 24 April 2003 4:37 pm, Jesse Barnes wrote:
> Bjorn, what about the patches posted in this thread?
> 
> http://marc.theaimsgroup.com/?t\x104359481700001&r=1&w=2
> 
> They seem to deal with the legacy resource issue quite nicely, and for
> the non-legacy case, pci_fixup should be making the
> pci_dev->resource[] arrays contain valid I/O addresss, shouldn't it?
> 
> Or do you have something else in mind?

I think we're talking about two different things.

If I understand it correctly, the reason for the patch you mentioned
is to take a device (VGA) that appears at a fixed IO port address,
and tweak the driver so it can talk to the device at a different
IO port address.

It doesn't expand the size of the IO port space, it just gives
you a hook to say "this hard-coded region of IO port space
really corresponds to this other region on my platform".

On the other hand, my patch is completely platform-specific and
allows us to address new IO port space that was previously not
accessible at all.  For example, on HP machines, the ia64 64K
"legacy IO port space" all gets routed to a single IO controller.
Here's a sample /proc/ioports:

    00000000-00000fff : PCI Bus 00:00
      00000060-0000006f : i8042
      00000500-000005ff : sym53c8xx
      00000600-000006ff : sym53c8xx
    00001000-00001fff : PCI Bus 00:04
    00002000-00003fff : PCI Bus 00:08
    00004000-00005fff : PCI Bus 00:10
    00006000-00007fff : PCI Bus 00:18
    00008000-00009fff : PCI Bus 00:20
      00008000-000080ff : sym53c8xx
      00008100-000081ff : sym53c8xx
    0000a000-0000bfff : PCI Bus 00:28
    0000c000-0000dfff : PCI Bus 00:30
    0000e000-0000ffff : PCI Bus 00:38

The above is the legacy IO port space (0-64K).  But the machine
has another IO controller with another 9 slots and another 64K
of IO port space that is currently unusable.  My patch just
makes that accessible, like this:

    01000000-01000fff : PCI Bus 00:40
      01000500-010005ff : sym53c8xx
      01000600-010006ff : sym53c8xx
    01001000-01001fff : PCI Bus 00:44
      01001000-010010ff : sym53c8xx
      01001100-010011ff : sym53c8xx
    01002000-01003fff : PCI Bus 00:48
    01004000-01005fff : PCI Bus 00:50
    01006000-01007fff : PCI Bus 00:58
    01008000-01009fff : PCI Bus 00:60
    0100a000-0100bfff : PCI Bus 00:68
    0100c000-0100dfff : PCI Bus 00:70
    0100e000-0100ffff : PCI Bus 00:78

Of course, the ports outside of the legacy 64K range aren't
usable by IA32 apps that use IN/OUT instructions, but they work
fine for the kernel and drivers.  David's book has a good
section on this, but I don't have it handy to give a specific
reference.

Bjorn

Re: [Linux-ia64] [PATCH] 1/4 multi-ioport space support for 2.5

[Jesse Barnes]

On Thu, Apr 24, 2003 at 06:44:41PM -0600, Bjorn Helgaas wrote:
> I think we're talking about two different things.

Maybe, but either way there's something I don't understand.

> If I understand it correctly, the reason for the patch you mentioned
> is to take a device (VGA) that appears at a fixed IO port address,
> and tweak the driver so it can talk to the device at a different
> IO port address.

Right.

> It doesn't expand the size of the IO port space, it just gives
> you a hook to say "this hard-coded region of IO port space
> really corresponds to this other region on my platform".

Couldn't it be used to do that though?  If drivers are required to
call request_legacy_region and pass in a pci_bus, they'll get back a
new io_base that they need to make use of.  Isn't that similar to the
way you create multiple legacy I/O port spaces?

> On the other hand, my patch is completely platform-specific and
> allows us to address new IO port space that was previously not
> accessible at all.  For example, on HP machines, the ia64 64K
> "legacy IO port space" all gets routed to a single IO controller.
> Here's a sample /proc/ioports:

I must be missing something.  I understand that you've effectively
created legacy I/O port spaces for each pci controller, but I don't
see how that helps you with a driver that does e.g. inb(0x3e8).  What
do you anticipate that your stuff will be used for?  That would help
out a bit...

Thanks,
Jesse

Re: [Linux-ia64] [PATCH] 1/4 multi-ioport space support for 2.5

[Matthew Wilcox]

On Thu, Apr 24, 2003 at 06:05:37PM -0700, Jesse Barnes wrote:
> I must be missing something.  I understand that you've effectively
> created legacy I/O port spaces for each pci controller, but I don't
> see how that helps you with a driver that does e.g. inb(0x3e8).  What
> do you anticipate that your stuff will be used for?  That would help
> out a bit...

Most drivers don't do that any more though.  All PCI drivers just read
their BARs and do inb() from there.  There's no ISA devices on hp's
ia64 boxes, so ISA probes aren't an issue.  We've been doing something
similar on PA for a while (io addresses are 0xBBiiii; the EISA bus if
present gets bus 0) with no problems.

-- 
"It's not Hollywood.  War is real, war is primarily not about defeat or
victory, it is about death.  I've seen thousands and thousands of dead bodies.
Do you think I want to have an academic debate on this subject?" -- Robert Fisk

Re: [Linux-ia64] [PATCH] 1/4 multi-ioport space support for 2.5

[Bjorn Helgaas]

On Thursday 24 April 2003 7:05 pm, Jesse Barnes wrote:
> ...  If drivers are required to
> call request_legacy_region and pass in a pci_bus, they'll get back a
> new io_base that they need to make use of.  Isn't that similar to the
> way you create multiple legacy I/O port spaces?

No.  I think part of the confusion here is that "legacy" is being used
to describe several different things.  In the VGA IO thread, it refers
to devices that are not relocatable by the standard PCI BAR mechanism.
When I said "the ia64 64K 'legacy IO port space'", I meant the 64K
IO port space inherited from the i386 processor model.  Finally,
David refers to "legacy devices" on pp. 304-307, and I think what
he's referring to are simply "low-cost and ill-designed PCI devices
that place some control registers in I/O space only," but are still
relocatable via PCI BARs.

My patch has nothing to do with the non-relocatable variety of "legacy"
devices.  Rather, I want to enable the "ill-designed" (but relocatable)
devices that happen to live outside the i386 64K IO port space.  This
should be transparent to the driver.  No request_legacy_region() call
should be required.  It wouldn't even make sense, because we're not
talking about a non-relocatable device.

> I must be missing something.  I understand that you've effectively
> created legacy I/O port spaces for each pci controller

Nope.  Let's not use "legacy" in this context.  For ia64, there is
exactly ONE "legacy IO port space", and it means "IO ports 0-65535,
which can be directly accessed in i386 mode".  On HP boxes, these
are all routed to one controller.  What I'm adding is support for
additional (non-legacy) IO port spaces that are routed to the
other controllers.

> but I don't
> see how that helps you with a driver that does e.g. inb(0x3e8).

My patch doesn't affect that driver at all.  Such a driver can
only talk to devices in certain slots.  The patch you mentioned
is a way to relax the "certain slots" restriction and is orthogonal
to mine.

> What do you anticipate that your stuff will be used for?

It enables a PCI card that requires IO ports but happens to be
plugged into a slot that doesn't get part of the 0-64K IO port
space.

I'd be interested in learning a little about how SGI boxes route
IO port space.  I don't know whether anything in my patch would
be applicable to them.  I'm guessing the ACPI discovery part would
not be applicable, but it's conceivable that the rest might be.

Bjorn

Re: [Linux-ia64] [PATCH] 1/4 multi-ioport space support for 2.5

[Jesse Barnes]

On Fri, Apr 25, 2003 at 12:36:28PM +0100, Matthew Wilcox wrote:
> On Thu, Apr 24, 2003 at 06:05:37PM -0700, Jesse Barnes wrote:
> > I must be missing something.  I understand that you've effectively
> > created legacy I/O port spaces for each pci controller, but I don't
> > see how that helps you with a driver that does e.g. inb(0x3e8).  What
> > do you anticipate that your stuff will be used for?  That would help
> > out a bit...
> 
> Most drivers don't do that any more though.  All PCI drivers just read
> their BARs and do inb() from there.  There's no ISA devices on hp's
> ia64 boxes, so ISA probes aren't an issue.  We've been doing something
> similar on PA for a while (io addresses are 0xBBiiii; the EISA bus if
> present gets bus 0) with no problems.

Ok, if that's the case, then SN boxes should be ok (I think), since
we'll mess around with the BARs in pci_fixup so that they're all
accessible.

Jesse

Re: [Linux-ia64] [PATCH] 1/4 multi-ioport space support for 2.5

[David Mosberger]

>>>>> On Thu, 24 Apr 2003 16:25:26 -0600, Bjorn Helgaas <bjorn_helgaas@HP.COM> said:

  Bjorn> 3 add support for /proc/iomem and /proc/ioports

I'm suspecting this patch may be causing me some grief on a Big Sur.
What happens is that the qla1280 is configured to use I/O space by
default (no idea why that is the case, apart from a small buglet in
reading the base address) and with the patch applied, it fails to get
the range from 0x2000-0x20ff.

I can work around the problem by using memory-mapped I/O.  When I do
that, /proc/ioports looks like this:

$ cat /proc/ioports
00000000-000003af : PCI Bus 00:00
  00000060-0000006f : i8042
  00000170-00000177 : ide1
  000001f0-000001f7 : ide0
  00000376-00000376 : ide1
000003c0-000003df : vga+
000003e0-00000cf7 : PCI Bus 00:00
  000003f6-000003f6 : ide0
  000003f8-000003ff : serial
00000d00-00000fff : PCI Bus 00:00
00001000-00001fff : PCI Bus 00:00
  00001000-0000103f : eepro100
  00001040-0000105f : uhci-hcd
00002000-00002000 : PCI Bus 00:01
00003000-0000ffff : PCI Bus 00:03

Perhaps the problem is that the reported range for Bus 00:01 is empty?
Does this make any sense to you?  Perhaps it's a firmware problem.

	--david

Re: [Linux-ia64] [PATCH] 1/4 multi-ioport space support for 2.5

[David Mosberger]

Oops, some more info: I missed the following boot-time error messages:

PCI: Probing PCI hardware on bus (00:00)
alloc 0x3b0-0x3df from PCI IO for PCI Bus 00:00 failed
ACPI: PCI Interrupt Routing Table [\_SB_.PCI0._PRT]
ACPI: PCI Root Bridge [PCI1] (00:01)
PCI: Probing PCI hardware on bus (00:01)
alloc 0x0-0x0 from PCI IO for PCI Bus 00:01 failed
ACPI: PCI Interrupt Routing Table [\_SB_.PCI1._PRT]
ACPI: PCI Root Bridge [PCI2] (00:02)
PCI: Probing PCI hardware on bus (00:02)
alloc 0x2000-0x2fff from PCI IO for PCI Bus 00:02 failed
alloc 0x0-0x0 from PCI mem for PCI Bus 00:02 failed
alloc 0x0-0x0 from PCI IO for PCI Bus 00:02 failed
ACPI: PCI Interrupt Routing Table [\_SB_.PCI2._PRT]
ACPI: PCI Root Bridge [PCI3] (00:03)
PCI: Probing PCI hardware on bus (00:03)
alloc 0xa0000-0xfffff from PCI mem for PCI Bus 00:03 failed
alloc 0x3b0-0x3df from PCI IO for PCI Bus 00:03 failed

Maybe it isn't a firmware problem, after all.

	--david

Re: [Linux-ia64] [PATCH] 1/4 multi-ioport space support for 2.5

[Bjorn Helgaas]

On Friday 09 May 2003 5:20 pm, David Mosberger wrote:
> I'm suspecting this patch may be causing me some grief on a Big Sur.
> What happens is that the qla1280 is configured to use I/O space by
> default (no idea why that is the case, apart from a small buglet in
> reading the base address) and with the patch applied, it fails to get
> the range from 0x2000-0x20ff.
> ...
> $ cat /proc/ioports
> 00000000-000003af : PCI Bus 00:00
>   00000060-0000006f : i8042
>   00000170-00000177 : ide1
>   000001f0-000001f7 : ide0
>   00000376-00000376 : ide1
> 000003c0-000003df : vga+
> 000003e0-00000cf7 : PCI Bus 00:00
>   000003f6-000003f6 : ide0
>   000003f8-000003ff : serial
> 00000d00-00000fff : PCI Bus 00:00
> 00001000-00001fff : PCI Bus 00:00
>   00001000-0000103f : eepro100
>   00001040-0000105f : uhci-hcd
> 00002000-00002000 : PCI Bus 00:01
> 00003000-0000ffff : PCI Bus 00:03
> 
> Perhaps the problem is that the reported range for Bus 00:01 is empty?
> Does this make any sense to you?  Perhaps it's a firmware problem.

I think this is a firmware problem.  On my box, the range for
Bus 00:01 is more sensible (0x2000-0x2fff), but there are several
other oddities:

	- IO space 0x0-0x0 reported for 00:01 and 00:02
	- mem space 0x0-0x0 reported for 00:01 and 00:02
	- IO space 0x3000-0x3000 reported for 00:02
	- mem space 0xa0000-0xfffff reported for both 00:00 and 00:03
	- IO space 0x3b0-0x3df reported for both 00:00 and 00:03

We could ignore the zero-length regions, which would kill a few warnings,
but the latter two (regions reported for two buses) just seem completely
broken.  And I think I'd rather just leave the warnings as aids to
finding firmware defects.

We will probably always get the warning about ports 0x3b0-0x3df
already being allocated, because the VGA ports are reserved long
before the root bridges are enumerated.

Bjorn

Re: [Linux-ia64] [PATCH] 1/4 multi-ioport space support for 2.5

[David Mosberger]

>>>>> On Mon, 12 May 2003 14:58:41 -0600, Bjorn Helgaas <bjorn_helgaas@hp.com> said:

  Bjorn> I think this is a firmware problem.  On my box, the range for
  Bjorn> Bus 00:01 is more sensible (0x2000-0x2fff)

OK with me, then.  I can boot my box with MMIO enabled, which is
probably a good idea anyhow.  Probably nobody else is running firmware
that old anyhow (I belive the box in question is too old to upgrade to
the latest firmware and I don't use it much anymore anyhow).

	--david