[RFC][PATCH] initial port of fixmap over from x86 for ppc32

[RFC][PATCH] initial port of fixmap over from x86 for ppc32

[Kumar Gala]

Wanted to get any feedback on this initial port of the fixmap support over
from x86.  There are a few TODOs:

* change HIGHMEM support to use fixmap
* fixup up VMALLOC config to respect fixmap

(after initial powerpc patch is in tree/accepted):
* rework a few bits of fixmap.h into an asm-generic/fixmap.h

The reason for introducing fixmap into ppc32 is it provides us with a
clean way of getting fixed compile time virtual addresses for things.

Beyond the HIGHMEM usage.  Ben and I have discussed cleaning up the PCIe
44x config code (and 83xx PCIe cfg) to use it.  Also, Dale's kexec/kdump
support on ppc32 can take advantage of it.  I'm also told this is useful
for hypervisor interactions.

One question for the guys looking at hypervisor.  The x86 code also has a
function called reserve_top_address (see arch/x86/mm/pgtable_32.c).  Is
this functionality something useful on ppc?  If so for what?

- k

---
 arch/powerpc/mm/pgtable_32.c |   18 ++++++
 include/asm-powerpc/fixmap.h |  123 ++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 141 insertions(+), 0 deletions(-)
 create mode 100644 include/asm-powerpc/fixmap.h

diff --git a/arch/powerpc/mm/pgtable_32.c b/arch/powerpc/mm/pgtable_32.c
index 64c44bc..fa0e48e 100644
--- a/arch/powerpc/mm/pgtable_32.c
+++ b/arch/powerpc/mm/pgtable_32.c
@@ -29,6 +29,7 @@

 #include <asm/pgtable.h>
 #include <asm/pgalloc.h>
+#include <asm/fixmap.h>
 #include <asm/io.h>

 #include "mmu_decl.h"
@@ -387,3 +388,20 @@ void kernel_map_pages(struct page *page, int numpages, int enable)
 	change_page_attr(page, numpages, enable ? PAGE_KERNEL : __pgprot(0));
 }
 #endif /* CONFIG_DEBUG_PAGEALLOC */
+
+static int fixmaps;
+unsigned long __FIXADDR_TOP = 0xfffff000;
+EXPORT_SYMBOL(__FIXADDR_TOP);
+
+void __set_fixmap (enum fixed_addresses idx, phys_addr_t phys, pgprot_t flags)
+{
+	unsigned long address = __fix_to_virt(idx);
+
+	if (idx >= __end_of_fixed_addresses) {
+		BUG();
+		return;
+	}
+
+	map_page(address, phys, flags);
+	fixmaps++;
+}
diff --git a/include/asm-powerpc/fixmap.h b/include/asm-powerpc/fixmap.h
new file mode 100644
index 0000000..7b570e5
--- /dev/null
+++ b/include/asm-powerpc/fixmap.h
@@ -0,0 +1,123 @@
+/*
+ * fixmap.h: compile-time virtual memory allocation
+ *
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License.  See the file "COPYING" in the main directory of this archive
+ * for more details.
+ *
+ * Copyright (C) 1998 Ingo Molnar
+ *
+ * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
+ *
+ * Copyright 2008 Freescale Semiconductor Inc.
+ *   Port to powerpc added by Kumar Gala
+ */
+
+#ifndef _ASM_FIXMAP_H
+#define _ASM_FIXMAP_H
+
+
+/* used by vmalloc.c, vsyscall.lds.S.
+ *
+ * Leave one empty page between vmalloc'ed areas and
+ * the start of the fixmap.
+ */
+extern unsigned long __FIXADDR_TOP;
+#define FIXADDR_USER_START     __fix_to_virt(FIX_VDSO)
+#define FIXADDR_USER_END       __fix_to_virt(FIX_VDSO - 1)
+
+#ifndef __ASSEMBLY__
+#include <linux/kernel.h>
+#include <asm/page.h>
+#ifdef CONFIG_HIGHMEM
+#include <linux/threads.h>
+#include <asm/kmap_types.h>
+#endif
+
+/*
+ * Here we define all the compile-time 'special' virtual
+ * addresses. The point is to have a constant address at
+ * compile time, but to set the physical address only
+ * in the boot process. We allocate these special addresses
+ * from the end of virtual memory (0xfffff000) backwards.
+ * Also this lets us do fail-safe vmalloc(), we
+ * can guarantee that these special addresses and
+ * vmalloc()-ed addresses never overlap.
+ *
+ * these 'compile-time allocated' memory buffers are
+ * fixed-size 4k pages. (or larger if used with an increment
+ * highger than 1) use fixmap_set(idx,phys) to associate
+ * physical memory with fixmap indices.
+ *
+ * TLB entries of such buffers will not be flushed across
+ * task switches.
+ */
+enum fixed_addresses {
+	FIX_HOLE,
+	FIX_VDSO,
+#ifdef CONFIG_HIGHMEM
+	FIX_KMAP_BEGIN,	/* reserved pte's for temporary kernel mappings */
+	FIX_KMAP_END = FIX_KMAP_BEGIN+(KM_TYPE_NR*NR_CPUS)-1,
+#endif
+#ifdef CONFIG_PCI_MMCONFIG
+	FIX_PCIE_MCFG,
+#endif
+	__end_of_fixed_addresses
+};
+
+extern void __set_fixmap (enum fixed_addresses idx,
+					phys_addr_t phys, pgprot_t flags);
+
+#define set_fixmap(idx, phys) \
+		__set_fixmap(idx, phys, PAGE_KERNEL)
+/*
+ * Some hardware wants to get fixmapped without caching.
+ */
+#define set_fixmap_nocache(idx, phys) \
+		__set_fixmap(idx, phys, PAGE_KERNEL_NOCACHE)
+
+#define clear_fixmap(idx) \
+		__set_fixmap(idx, 0, __pgprot(0))
+
+#define FIXADDR_TOP	((unsigned long)__FIXADDR_TOP)
+
+#define __FIXADDR_SIZE	(__end_of_fixed_addresses << PAGE_SHIFT)
+#define __FIXADDR_BOOT_SIZE	(__end_of_fixed_addresses << PAGE_SHIFT)
+#define FIXADDR_START		(FIXADDR_TOP - __FIXADDR_SIZE)
+#define FIXADDR_BOOT_START	(FIXADDR_TOP - __FIXADDR_BOOT_SIZE)
+
+#define __fix_to_virt(x)	(FIXADDR_TOP - ((x) << PAGE_SHIFT))
+#define __virt_to_fix(x)	((FIXADDR_TOP - ((x)&PAGE_MASK)) >> PAGE_SHIFT)
+
+extern void __this_fixmap_does_not_exist(void);
+
+/*
+ * 'index to address' translation. If anyone tries to use the idx
+ * directly without tranlation, we catch the bug with a NULL-deference
+ * kernel oops. Illegal ranges of incoming indices are caught too.
+ */
+static __always_inline unsigned long fix_to_virt(const unsigned int idx)
+{
+	/*
+	 * this branch gets completely eliminated after inlining,
+	 * except when someone tries to use fixaddr indices in an
+	 * illegal way. (such as mixing up address types or using
+	 * out-of-range indices).
+	 *
+	 * If it doesn't get removed, the linker will complain
+	 * loudly with a reasonably clear error message..
+	 */
+	if (idx >= __end_of_fixed_addresses)
+		__this_fixmap_does_not_exist();
+
+        return __fix_to_virt(idx);
+}
+
+static inline unsigned long virt_to_fix(const unsigned long vaddr)
+{
+	BUG_ON(vaddr >= FIXADDR_TOP || vaddr < FIXADDR_START);
+	return __virt_to_fix(vaddr);
+}
+
+#endif /* !__ASSEMBLY__ */
+#endif
-- 
1.5.4.1

Re: [RFC][PATCH] initial port of fixmap over from x86 for ppc32

[Hollis Blanchard]

On Thursday 03 April 2008 01:52:36 Kumar Gala wrote:
> Wanted to get any feedback on this initial port of the fixmap support over
> from x86.  There are a few TODOs:
>
> * change HIGHMEM support to use fixmap
> * fixup up VMALLOC config to respect fixmap
>
> (after initial powerpc patch is in tree/accepted):
> * rework a few bits of fixmap.h into an asm-generic/fixmap.h
>
> The reason for introducing fixmap into ppc32 is it provides us with a
> clean way of getting fixed compile time virtual addresses for things.
>
> Beyond the HIGHMEM usage.  Ben and I have discussed cleaning up the PCIe
> 44x config code (and 83xx PCIe cfg) to use it.  Also, Dale's kexec/kdump
> support on ppc32 can take advantage of it.  I'm also told this is useful
> for hypervisor interactions.
>
> One question for the guys looking at hypervisor.  The x86 code also has a
> function called reserve_top_address (see arch/x86/mm/pgtable_32.c).  Is
> this functionality something useful on ppc?  If so for what?

x86 virtualization implementations often needs a trampoline that's mapped into 
both host and guest virtual address space, so that's part of what you're 
seeing.

In general though, it can be very useful for the host to own a piece of the 
guest's virtual address space. For example, the host could rewrite 
problematic guest instructions to branch to host-optimized code which avoids 
hypercalls. However, this is impossible unless the host knows it can 
overwrite some portion of the guest's effective address space.

reserve_top_address() doesn't look complicated, so we might as well keep it?

-- 
Hollis Blanchard
IBM Linux Technology Center

Re: [RFC][PATCH] initial port of fixmap over from x86 for ppc32

[Benjamin Herrenschmidt]

On Thu, 2008-04-03 at 13:47 -0500, Hollis Blanchard wrote:
> 
> x86 virtualization implementations often needs a trampoline that's
> mapped into 
> both host and guest virtual address space, so that's part of what
> you're 
> seeing.
> 
> In general though, it can be very useful for the host to own a piece
> of the 
> guest's virtual address space. For example, the host could rewrite 
> problematic guest instructions to branch to host-optimized code which
> avoids 
> hypercalls. However, this is impossible unless the host knows it can 
> overwrite some portion of the guest's effective address space.
> 
> reserve_top_address() doesn't look complicated, so we might as well
> keep it ?

Agreed. In fact, using the top of the address space for that is a good
idea as you can do the branching there using absolute branch
instructions which is simpler.

Ben.

Re: [RFC][PATCH] initial port of fixmap over from x86 for ppc32

[Paul Mackerras]

Kumar Gala writes:

> Wanted to get any feedback on this initial port of the fixmap support over
> from x86.  There are a few TODOs:

I have no objection in principle, but your patch below imports a few
things that aren't (and won't be) needed on powerpc AFAICS -- for
example, we don't need FIX_VDSO, since our VDSO is mapped into user
space at the 1MB point (by default).

You have FIX_PCIE_MCFG in there too (keyed off CONFIG_PCI_MMCONFIG
which we don't have and don't want to have).  If you need to map in
PCIe config space, what's wrong with just using ioremap?  Why do you
need to have a fixed virtual address for it?

More generally, I think we need to take an overall look at what things
we are using fixed virtual addresses for, and why they need to be
fixed.  If there are indeed several such things then we can introduce
the fixmap stuff.

Paul.

Re: [RFC][PATCH] initial port of fixmap over from x86 for ppc32

[Benjamin Herrenschmidt]

> You have FIX_PCIE_MCFG in there too (keyed off CONFIG_PCI_MMCONFIG
> which we don't have and don't want to have).  If you need to map in
> PCIe config space, what's wrong with just using ioremap?  Why do you
> need to have a fixed virtual address for it?

Well, that was the whole point for doing the fixmap stuff actually, to
be able to have a non-HIGHMEM version of a quick kmap_atomic for ...
PCIe config space :-)

On some PCIe host brigdes like the ones used on 4xx or FSL chips, the
config space is physically mapped as a large linear space, too large to
ioremap permanently, and we can't ioremap from within the low level
config ops.

So the idea is to use a fixmap as a "window" to the config space,
possibly per-cpu. Ultimately, we should even be able to directly insert
TLB entries in kmap_atomic/fixmap to make it even faster.

> More generally, I think we need to take an overall look at what things
> we are using fixed virtual addresses for, and why they need to be
> fixed.  If there are indeed several such things then we can introduce
> the fixmap stuff.

The virtual address doesn't _need_ to be fixed in our case. It's more
like x86 builds kmap on top of fixmap and we were thinking about doing
the same, having it fixed makes it slightly easier to whack things but I
agree it's not necessarily the best option.

We could instead have something allocated a chunk of virtual space at
boot and provide a quick map/unmap.

But by using fixmap, we can use common code with kmap_atomic quickly and
easily.

Ben.

Re: [RFC][PATCH] initial port of fixmap over from x86 for ppc32

[Kumar Gala]

On Apr 6, 2008, at 6:48 PM, Paul Mackerras wrote:
> Kumar Gala writes:
>
>> Wanted to get any feedback on this initial port of the fixmap  
>> support over
>> from x86.  There are a few TODOs:
>
> I have no objection in principle, but your patch below imports a few
> things that aren't (and won't be) needed on powerpc AFAICS -- for
> example, we don't need FIX_VDSO, since our VDSO is mapped into user
> space at the 1MB point (by default).

Agreed.

> You have FIX_PCIE_MCFG in there too (keyed off CONFIG_PCI_MMCONFIG
> which we don't have and don't want to have).  If you need to map in
> PCIe config space, what's wrong with just using ioremap?  Why do you
> need to have a fixed virtual address for it?

Ben has commented on this.  I know on the 83xx systems we have two  
PCIe PHBs that would require ioremapping 512M of virtual address space  
which we don't have if the system has any large amount of memory.

> More generally, I think we need to take an overall look at what things
> we are using fixed virtual addresses for, and why they need to be
> fixed.  If there are indeed several such things then we can introduce
> the fixmap stuff.

The list as I see it:
* kmap
* pci-e config for 4xx/83xx
* kexec/kdump (ben commented on this when Dale posted his patches for  
ppc32 support)

future:
* possible usage by HV

since we already have three users and a possible fourth it seems like  
a useful change.

- k

Re: [RFC][PATCH] initial port of fixmap over from x86 for ppc32

[Scott Wood]

On Mon, Apr 07, 2008 at 08:09:57AM -0500, Kumar Gala wrote:
> On Apr 6, 2008, at 6:48 PM, Paul Mackerras wrote:
>> More generally, I think we need to take an overall look at what things
>> we are using fixed virtual addresses for, and why they need to be
>> fixed.  If there are indeed several such things then we can introduce
>> the fixmap stuff.
>
> The list as I see it:
> * kmap
> * pci-e config for 4xx/83xx
> * kexec/kdump (ben commented on this when Dale posted his patches for  
> ppc32 support)
>
> future:
> * possible usage by HV
>
> since we already have three users and a possible fourth it seems like a 
> useful change.

Another possible use is a BAT/TLB1 mapping for SoC registers (or anything
else on the board which is frequently accessed), which can be reused by
ioremap() to avoid wasting normal TLB entries, and to facilitate early
debugging.

-Scott

Re: [RFC][PATCH] initial port of fixmap over from x86 for ppc32

[Kumar Gala]

On Apr 7, 2008, at 11:20 AM, Scott Wood wrote:
> On Mon, Apr 07, 2008 at 08:09:57AM -0500, Kumar Gala wrote:
>> On Apr 6, 2008, at 6:48 PM, Paul Mackerras wrote:
>>> More generally, I think we need to take an overall look at what  
>>> things
>>> we are using fixed virtual addresses for, and why they need to be
>>> fixed.  If there are indeed several such things then we can  
>>> introduce
>>> the fixmap stuff.
>>
>> The list as I see it:
>> * kmap
>> * pci-e config for 4xx/83xx
>> * kexec/kdump (ben commented on this when Dale posted his patches for
>> ppc32 support)
>>
>> future:
>> * possible usage by HV
>>
>> since we already have three users and a possible fourth it seems  
>> like a
>> useful change.
>
> Another possible use is a BAT/TLB1 mapping for SoC registers (or  
> anything
> else on the board which is frequently accessed), which can be reused  
> by
> ioremap() to avoid wasting normal TLB entries, and to facilitate early
> debugging.

x86 has an early ioremap() that is implemented on top of their fixmap  
usage that we could also steal if desired :)

- k

Re: [RFC][PATCH] initial port of fixmap over from x86 for ppc32

[Benjamin Herrenschmidt]

On Mon, 2008-04-07 at 11:20 -0500, Scott Wood wrote:
> 
> Another possible use is a BAT/TLB1 mapping for SoC registers (or
> anything
> else on the board which is frequently accessed), which can be reused
> by
> ioremap() to avoid wasting normal TLB entries, and to facilitate early
> debugging.

As far as the above is concerned, I'm not too fan of fixed virtual
addresses. I'd rather dynamically assign it. But we can do it.

Ben.

Re: [RFC][PATCH] initial port of fixmap over from x86 for ppc32

[Kumar Gala]

On Apr 7, 2008, at 6:51 PM, Benjamin Herrenschmidt wrote:
>
> On Mon, 2008-04-07 at 11:20 -0500, Scott Wood wrote:
>>
>> Another possible use is a BAT/TLB1 mapping for SoC registers (or
>> anything
>> else on the board which is frequently accessed), which can be reused
>> by
>> ioremap() to avoid wasting normal TLB entries, and to facilitate  
>> early
>> debugging.
>
> As far as the above is concerned, I'm not too fan of fixed virtual
> addresses. I'd rather dynamically assign it. But we can do it.

I'm in agreement with Ben here.  On Freescale parts that could mean  
using up to 16M of virtual address space while in reality we may only  
need a handful of 4k pages to cover SoC registers that we are truly  
accessing at runtime.

However I think there might be some utility to have an early ioremap  
mappings for debug.

- k

Re: [RFC][PATCH] initial port of fixmap over from x86 for ppc32

[Scott Wood]

On Tue, Apr 08, 2008 at 09:28:12AM -0500, Kumar Gala wrote:
> On Apr 7, 2008, at 6:51 PM, Benjamin Herrenschmidt wrote:
>> As far as the above is concerned, I'm not too fan of fixed virtual
>> addresses. I'd rather dynamically assign it. But we can do it.
>
> I'm in agreement with Ben here.  On Freescale parts that could mean  
> using up to 16M of virtual address space while in reality we may only  
> need a handful of 4k pages to cover SoC registers that we are truly  
> accessing at runtime.

So make it configurable, and turn it off if you're low on address space. 

Most of the time, I'd think saving TLB0 entries would be more beneficial,
especially with heavy I/O workloads where registers get banged on a lot.

It's the early debugging that I'm really concerned with, though.  It gets
old hacking an early mapping in each time.

-Scott