patch / rfc

patch / rfc

[Pete Popov]

This is an experimental (though tested) patch for early ioremap support
on mips, before mem_init runs. Something like this is only needed on
certain SoCs that have all of their I/O on high addresses such that they
can't me ioremapped through kseg1.

I think the CONFIG_64 stuff needs to removed since we don't need it. The
patch was tested on a MIPS32 CPU only. Some of the significant changes:

- trap_init() became early_trap_init() since too much stuff happens
there that is needed to support early ioremap. The old trap_init() is
now empty.

- added plat_setup_late() call. All ports would need to add that call,
or at least a placeholder. Early ioremap is possible only at that point.
However, most of the code in each plat_setup() can be moved to
plat_setup_late()

diff -Naur --exclude=CVS linux-2.6-orig/arch/mips/kernel/setup.c linux-2.6-dev/arch/mips/kernel/setup.c
--- linux-2.6-orig/arch/mips/kernel/setup.c	2005-08-24 17:12:31.000000000 -0700
+++ linux-2.6-dev/arch/mips/kernel/setup.c	2005-08-25 10:56:40.000000000 -0700
@@ -42,7 +42,10 @@
 #include <asm/sections.h>
 #include <asm/setup.h>
 #include <asm/system.h>
+#include <asm/traps.h>
+#include <asm/io.h>
 
+int init_bootmem_done;
 struct cpuinfo_mips cpu_data[NR_CPUS] __read_mostly;
 
 EXPORT_SYMBOL(cpu_data);
@@ -417,6 +420,8 @@
 
 	/* Reserve the bootmap memory.  */
 	reserve_bootmem(PFN_PHYS(first_usable_pfn), bootmap_size);
+
+	init_bootmem_done = 1;
 #endif /* CONFIG_SGI_IP27 */
 
 #ifdef CONFIG_BLK_DEV_INITRD
@@ -511,7 +516,23 @@
 #undef MAXMEM
 #undef MAXMEM_PFN
 
+void __init early_ioremap_init(void)
+{
+#ifdef CONFIG_64BIT
+	ioremap_base = (VMALLOC_START + PTRS_PER_PGD * PTRS_PER_PMD * PTRS_PER_PTE * PAGE_SIZE);
+#else
+#ifdef CONFIG_HIGHMEM
+	ioremap_base = (PKMAP_BASE-2*PAGE_SIZE);
+#else
+	ioremap_base = (FIXADDR_START-2*PAGE_SIZE);
+#endif
+#endif /* CONFIG_64BIT */
+
+	ioremap_bot = ioremap_base;
+}
+
 extern void plat_setup(void);
+extern void plat_setup_late(void);
 
 void __init setup_arch(char **cmdline_p)
 {
@@ -540,6 +561,11 @@
 	sparse_init();
 	paging_init();
 	resource_init();
+
+	early_trap_init();
+	early_ioremap_init();
+
+	plat_setup_late();	/* safe to do early ioremap */
 }
 
 int __init fpu_disable(char *s)
diff -Naur --exclude=CVS linux-2.6-orig/arch/mips/kernel/traps.c linux-2.6-dev/arch/mips/kernel/traps.c
--- linux-2.6-orig/arch/mips/kernel/traps.c	2005-08-24 17:12:32.000000000 -0700
+++ linux-2.6-dev/arch/mips/kernel/traps.c	2005-08-25 00:39:39.000000000 -0700
@@ -1239,6 +1238,10 @@
 
 void __init trap_init(void)
 {
+}
+
+void __init early_trap_init(void)
+{
 	extern char except_vec3_generic, except_vec3_r4000;
 	extern char except_vec4;
 	unsigned long i;
diff -Naur --exclude=CVS linux-2.6-orig/arch/mips/mm/init.c linux-2.6-dev/arch/mips/mm/init.c
--- linux-2.6-orig/arch/mips/mm/init.c	2005-08-10 18:15:55.000000000 -0700
+++ linux-2.6-dev/arch/mips/mm/init.c	2005-08-25 00:39:40.000000000 -0700
@@ -47,6 +47,8 @@
  * don't have to care about aliases on other CPUs.
  */
 unsigned long empty_zero_page, zero_page_mask;
+int mem_init_done;
+extern int init_bootmem_done;
 
 /*
  * Not static inline because used by IP27 special magic initialization code
@@ -200,6 +202,17 @@
 	return 0;
 }
 
+void __init *early_get_page(void)
+{
+	void *p;
+	if (init_bootmem_done) {
+		p = alloc_bootmem_pages(PAGE_SIZE);
+	} else {
+		p = NULL;
+	}
+	return p;
+}
+
 void __init mem_init(void)
 {
 	unsigned long codesize, reservedpages, datasize, initsize;
@@ -258,6 +271,8 @@
 	       datasize >> 10,
 	       initsize >> 10,
 	       (unsigned long) (totalhigh_pages << (PAGE_SHIFT-10)));
+
+	mem_init_done = 1;
 }
 #endif /* !CONFIG_NEED_MULTIPLE_NODES */
 
diff -Naur --exclude=CVS linux-2.6-orig/arch/mips/mm/ioremap.c linux-2.6-dev/arch/mips/mm/ioremap.c
--- linux-2.6-orig/arch/mips/mm/ioremap.c	2005-08-10 18:15:55.000000000 -0700
+++ linux-2.6-dev/arch/mips/mm/ioremap.c	2005-08-25 13:35:25.045624752 -0700
@@ -15,6 +15,8 @@
 #include <asm/io.h>
 #include <asm/tlbflush.h>
 
+extern int mem_init_done;
+
 static inline void remap_area_pte(pte_t * pte, unsigned long address,
 	phys_t size, phys_t phys_addr, unsigned long flags)
 {
@@ -141,8 +143,9 @@
 
 	/*
 	 * Don't allow anybody to remap normal RAM that we're using..
+	 * mem_init() sets high_memory so only do the check after that.
 	 */
-	if (phys_addr < virt_to_phys(high_memory)) {
+	if (mem_init_done && (phys_addr < virt_to_phys(high_memory))) {
 		char *t_addr, *t_end;
 		struct page *page;
 
@@ -164,12 +167,18 @@
 	/*
 	 * Ok, go for it..
 	 */
-	area = get_vm_area(size, VM_IOREMAP);
-	if (!area)
-		return NULL;
-	addr = area->addr;
+	if (mem_init_done) {
+		area = get_vm_area(size, VM_IOREMAP);
+		if (!area)
+			return NULL;
+		addr = area->addr;
+	}
+	else {
+		addr = (void *)(ioremap_bot -= size);
+	}
 	if (remap_area_pages((unsigned long) addr, phys_addr, size, flags)) {
-		vunmap(addr);
+		if (mem_init_done)
+			vunmap(addr);
 		return NULL;
 	}
 
@@ -185,13 +194,15 @@
 	if (IS_KSEG1(addr))
 		return;
 
-	p = remove_vm_area((void *) (PAGE_MASK & (unsigned long __force) addr));
-	if (!p) {
-		printk(KERN_ERR "iounmap: bad address %p\n", addr);
-		return;
-	}
+	if ((unsigned long) addr < ioremap_bot) {
+		p = remove_vm_area((void *) (PAGE_MASK & (unsigned long __force) addr));
+		if (!p) {
+			printk(KERN_ERR "iounmap: bad address %p\n", addr);
+			return;
+		}
 
-        kfree(p);
+		kfree(p);
+	}
 }
 
 EXPORT_SYMBOL(__ioremap);
diff -Naur --exclude=CVS linux-2.6-orig/arch/mips/mm/pgtable-32.c linux-2.6-dev/arch/mips/mm/pgtable-32.c
--- linux-2.6-orig/arch/mips/mm/pgtable-32.c	2005-08-10 18:15:56.000000000 -0700
+++ linux-2.6-dev/arch/mips/mm/pgtable-32.c	2005-08-23 00:46:14.000000000 -0700
@@ -13,6 +13,9 @@
 #include <asm/fixmap.h>
 #include <asm/pgtable.h>
 
+unsigned long ioremap_base;
+unsigned long ioremap_bot;
+
 void pgd_init(unsigned long page)
 {
 	unsigned long *p = (unsigned long *) page;
diff -Naur --exclude=CVS linux-2.6-orig/include/asm-mips/io.h linux-2.6-dev/include/asm-mips/io.h
--- linux-2.6-orig/include/asm-mips/io.h	2005-08-10 18:22:04.000000000 -0700
+++ linux-2.6-dev/include/asm-mips/io.h	2005-08-25 13:35:31.000000000 -0700
@@ -637,4 +637,6 @@
  */
 #define xlate_dev_kmem_ptr(p)	p
 
+extern void early_ioremap_init(void);
+
 #endif /* _ASM_IO_H */
diff -Naur --exclude=CVS linux-2.6-orig/include/asm-mips/pgalloc.h linux-2.6-dev/include/asm-mips/pgalloc.h
--- linux-2.6-orig/include/asm-mips/pgalloc.h	2005-08-10 18:22:02.000000000 -0700
+++ linux-2.6-dev/include/asm-mips/pgalloc.h	2005-08-25 01:49:04.000000000 -0700
@@ -67,8 +67,17 @@
 	unsigned long address)
 {
 	pte_t *pte;
+	extern int mem_init_done;
+	extern void *early_get_page(void);
 
-	pte = (pte_t *) __get_free_pages(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO, PTE_ORDER);
+	if (mem_init_done) {
+		pte = (pte_t *) __get_free_pages(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO, PTE_ORDER);
+	} else {
+		pte = (pte_t *)early_get_page();
+		if (pte) {
+			clear_page(pte);
+		}
+	}
 
 	return pte;
 }
diff -Naur --exclude=CVS linux-2.6-orig/include/asm-mips/pgtable-32.h linux-2.6-dev/include/asm-mips/pgtable-32.h
--- linux-2.6-orig/include/asm-mips/pgtable-32.h	2005-08-25 13:02:30.000000000 -0700
+++ linux-2.6-dev/include/asm-mips/pgtable-32.h	2005-08-25 01:49:04.000000000 -0700
@@ -19,6 +19,8 @@
 
 #include <asm-generic/pgtable-nopmd.h>
 
+extern unsigned long ioremap_bot, ioremap_base;
+
 /*
  * - add_wired_entry() add a fixed TLB entry, and move wired register
  */
@@ -74,13 +76,9 @@
 #define USER_PTRS_PER_PGD	(0x80000000UL/PGDIR_SIZE)
 #define FIRST_USER_ADDRESS	0
 
-#define VMALLOC_START     MAP_BASE
+#define VMALLOC_START	MAP_BASE
 
-#ifdef CONFIG_HIGHMEM
-# define VMALLOC_END	(PKMAP_BASE-2*PAGE_SIZE)
-#else
-# define VMALLOC_END	(FIXADDR_START-2*PAGE_SIZE)
-#endif
+#define VMALLOC_END	ioremap_bot
 
 #ifdef CONFIG_64BIT_PHYS_ADDR
 #define pte_ERROR(e) \
diff -Naur --exclude=CVS linux-2.6-orig/include/asm-mips/traps.h linux-2.6-dev/include/asm-mips/traps.h
--- linux-2.6-orig/include/asm-mips/traps.h	2005-08-10 18:22:04.000000000 -0700
+++ linux-2.6-dev/include/asm-mips/traps.h	2005-08-25 10:56:01.000000000 -0700
@@ -24,4 +24,6 @@
 extern void (*board_nmi_handler_setup)(void);
 extern void (*board_ejtag_handler_setup)(void);
 
+extern void early_trap_init(void);
+
 #endif /* _ASM_TRAPS_H */

Re: patch / rfc

[Maciej W. Rozycki]

On Thu, 25 Aug 2005, Pete Popov wrote:

> This is an experimental (though tested) patch for early ioremap support
> on mips, before mem_init runs. Something like this is only needed on
> certain SoCs that have all of their I/O on high addresses such that they
> can't me ioremapped through kseg1.

 Hmm, wouldn't a temporary large page and a wired TLB entry be an easier 
solution?  Somebody designing these SoCs must have taken such an approach 
into account when deciding to put I/O devices outside the space that's 
directly accessible through unmapped spaces, so I'd expect them all to be 
reachable in a single page of the largest size supported by a given 
implementation.  Especially as not all software is expected to implement 
fully-featured page management.  This entry would of course be no longer 
available after the final paging setup (TLBs tend to be too small for 
entries to be wasted).

> I think the CONFIG_64 stuff needs to removed since we don't need it. The
> patch was tested on a MIPS32 CPU only. Some of the significant changes:

 Well, MIPS64 has XPHYS, so there is no need for going through paging for 
ioremap() at all.

> - trap_init() became early_trap_init() since too much stuff happens
> there that is needed to support early ioremap. The old trap_init() is
> now empty.

 That just provides a strong suggestion considering an alternative 
approach, such as one proposed above is not a bad idea -- this changes the 
order subsystems are initialized for the MIPS platform, which makes it 
different from all the others and therefore problematic.

> - added plat_setup_late() call. All ports would need to add that call,
> or at least a placeholder. Early ioremap is possible only at that point.
> However, most of the code in each plat_setup() can be moved to
> plat_setup_late()

 Which means it should rather be a function pointer initialized somewhere 
earlier, possibly in plat_setup() and then:

static void __init null_plat_setup_late(void) { }
void (*plat_setup_late)(void) __initdata = null_plat_setup_late;
[...]
	plat_setup_late()

or:

void (*plat_setup_late)(void);
[...]
	if (plat_setup_late)
		plat_setup_late()

or something like that.

  Maciej

Re: patch / rfc

[Pete Popov]

On Fri, 2005-08-26 at 14:10 +0100, Maciej W. Rozycki wrote:
> On Thu, 25 Aug 2005, Pete Popov wrote:
> 
> > This is an experimental (though tested) patch for early ioremap support
> > on mips, before mem_init runs. Something like this is only needed on
> > certain SoCs that have all of their I/O on high addresses such that they
> > can't me ioremapped through kseg1.
> 
>  Hmm, wouldn't a temporary large page and a wired TLB entry be an easier 
> solution?  

Yes, you can do that. It's just not as nice and the problem is that you
either have to remember these wired mappings, flush them later, and
properly ioremap them, or forever use up a few tlbs.

> Somebody designing these SoCs must have taken such an approach 
> into account when deciding to put I/O devices outside the space that's 
> directly accessible through unmapped spaces, so I'd expect them all to be 
> reachable in a single page of the largest size supported by a given 
> implementation.  

They may or may not be reachable with a single tlb. I've twice now seen
new SoCs with a huge I/O address range. There may or may not be a good
reason for this but we are not always hired early enough to change the
design.

> Especially as not all software is expected to implement 
> fully-featured page management.  This entry would of course be no longer 
> available after the final paging setup (TLBs tend to be too small for 
> entries to be wasted).

Right.

> > I think the CONFIG_64 stuff needs to removed since we don't need it. The
> > patch was tested on a MIPS32 CPU only. Some of the significant changes:
> 
>  Well, MIPS64 has XPHYS, so there is no need for going through paging for 
> ioremap() at all.

Right. I can easily change that.

> > - trap_init() became early_trap_init() since too much stuff happens
> > there that is needed to support early ioremap. The old trap_init() is
> > now empty.
> 
>  That just provides a strong suggestion considering an alternative 
> approach, such as one proposed above is not a bad idea -- this changes the 
> order subsystems are initialized for the MIPS platform, which makes it 
> different from all the others and therefore problematic.

Well, yes and no. On PowerPC, they do all this ahead of trap_init so
their trap_init is empty as well. Looking at the code, I just don't see
any reason why we can't do that init earlier.

> > - added plat_setup_late() call. All ports would need to add that call,
> > or at least a placeholder. Early ioremap is possible only at that point.
> > However, most of the code in each plat_setup() can be moved to
> > plat_setup_late()
> 
>  Which means it should rather be a function pointer initialized somewhere 
> earlier, possibly in plat_setup() and then:
> 
> static void __init null_plat_setup_late(void) { }
> void (*plat_setup_late)(void) __initdata = null_plat_setup_late;
> [...]
> 	plat_setup_late()
> 
> or:
> 
> void (*plat_setup_late)(void);
> [...]
> 	if (plat_setup_late)
> 		plat_setup_late()
> 
> or something like that.

Sure, we can do that.

Thanks,

Pete

Re: patch / rfc

[Dan Malek]

On Aug 26, 2005, at 11:24 AM, Pete Popov wrote:

>> void (*plat_setup_late)(void);
>> [...]
>> 	if (plat_setup_late)
>> 		plat_setup_late()
>>
>> or something like that.
>
> Sure, we can do that.

If you do this, I suggest using another PowerPC-ism.  They
have a ppc_md data structure that is filled with indirect function
pointers to machine dependent functions.  We could create
a mips_md that does this same thing.  The reason I like this
is it collects all machine dependent information in a single
place, so it's easy to see what functions/data are available
and what you may need to do.  It's also clear when used
that anything in this structure is a machine/board dependent
function.  In the proper places, you then do what is shown above:

	if (mips_md.plat_setup_late)
		mips_md.plat_setup_late();

Your earliest machine dependent set up can then fill this in
based upon board options, or you can statically set it up if
you need it even earlier and change it later.

Thanks.

	-- Dan

Re: patch / rfc

[Maciej W. Rozycki]

On Fri, 26 Aug 2005, Dan Malek wrote:

> If you do this, I suggest using another PowerPC-ism.  They
> have a ppc_md data structure that is filled with indirect function
> pointers to machine dependent functions.  We could create
> a mips_md that does this same thing.  The reason I like this
> is it collects all machine dependent information in a single
> place, so it's easy to see what functions/data are available
> and what you may need to do.  It's also clear when used
> that anything in this structure is a machine/board dependent
> function.  In the proper places, you then do what is shown above:

 Well, that's actually Alpha-ism, which has been that port since 1998 or 
so and is a long-term plan for our multi-platform support too. :-)  If you 
want to implement it right now, please go ahead!

  Maciej

Re: patch / rfc

[Dan Malek]

On Aug 26, 2005, at 11:53 AM, Maciej W. Rozycki wrote:

>  Well, that's actually Alpha-ism, which has been that port since 1998 
> or
> so and is a long-term plan for our multi-platform support too. :-)  If 
> you
> want to implement it right now, please go ahead!

If that's acceptable, I'll get started on it and it's something that
will grow over time.

Thanks.

	-- Dan