* RESEND [PATCH 3/6] arm64: Refactor vDSO init/setup
From: Mark Salyzyn @ 2018-06-18 15:06 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20180618150613.10322-1-salyzyn@android.com>
From: Kevin Brodsky <kevin.brodsky@arm.com>
Move the logic for setting up mappings and pages for the vDSO into
static functions. This makes the vDSO setup code more consistent with
the compat side and will allow to reuse it for the future compat vDSO.
Signed-off-by: Kevin Brodsky <kevin.brodsky@arm.com>
Signed-off-by: Mark Salyzyn <salyzyn@android.com>
Cc: James Morse <james.morse@arm.com>
Cc: Russell King <linux@armlinux.org.uk>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dmitry Safonov <dsafonov@virtuozzo.com>
Cc: John Stultz <john.stultz@linaro.org>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Laura Abbott <labbott@redhat.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Andy Gross <andy.gross@linaro.org>
Cc: Andrew Pinski <apinski@cavium.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-kernel at vger.kernel.org
Cc: linux-arm-kernel at lists.infradead.org
Cc: Jeremy Linton <Jeremy.Linton@arm.com>
---
arch/arm64/kernel/vdso.c | 118 +++++++++++++++++++++++----------------
1 file changed, 70 insertions(+), 48 deletions(-)
diff --git a/arch/arm64/kernel/vdso.c b/arch/arm64/kernel/vdso.c
index 76a94bed4bd5..8529e85a521f 100644
--- a/arch/arm64/kernel/vdso.c
+++ b/arch/arm64/kernel/vdso.c
@@ -39,8 +39,11 @@
#include <asm/vdso.h>
#include <asm/vdso_datapage.h>
-extern char vdso_start[], vdso_end[];
-static unsigned long vdso_pages __ro_after_init;
+struct vdso_mappings {
+ unsigned long num_code_pages;
+ struct vm_special_mapping data_mapping;
+ struct vm_special_mapping code_mapping;
+};
/*
* The vDSO data page.
@@ -164,95 +167,114 @@ static int vdso_mremap(const struct vm_special_mapping *sm,
return 0;
}
-static struct vm_special_mapping vdso_spec[2] __ro_after_init = {
- {
- .name = "[vvar]",
- },
- {
- .name = "[vdso]",
- .mremap = vdso_mremap,
- },
-};
-
-static int __init vdso_init(void)
+static int __init vdso_mappings_init(const char *name,
+ const char *code_start,
+ const char *code_end,
+ struct vdso_mappings *mappings)
{
- int i;
+ unsigned long i, vdso_page;
struct page **vdso_pagelist;
unsigned long pfn;
- if (memcmp(vdso_start, "\177ELF", 4)) {
- pr_err("vDSO is not a valid ELF object!\n");
+ if (memcmp(code_start, "\177ELF", 4)) {
+ pr_err("%s is not a valid ELF object!\n", name);
return -EINVAL;
}
- vdso_pages = (vdso_end - vdso_start) >> PAGE_SHIFT;
- pr_info("vdso: %ld pages (%ld code @ %p, %ld data @ %p)\n",
- vdso_pages + 1, vdso_pages, vdso_start, 1L, vdso_data);
-
- /* Allocate the vDSO pagelist, plus a page for the data. */
- vdso_pagelist = kcalloc(vdso_pages + 1, sizeof(struct page *),
- GFP_KERNEL);
+ vdso_pages = (code_end - code_start) >> PAGE_SHIFT;
+ pr_info("%s: %ld pages (%ld code @ %p, %ld data @ %p)\n",
+ name, vdso_pages + 1, vdso_pages, code_start, 1L,
+ vdso_data);
+
+ /*
+ * Allocate space for storing pointers to the vDSO code pages + the
+ * data page. The pointers must have the same lifetime as the mappings,
+ * which are static, so there is no need to keep track of the pointer
+ * array to free it.
+ */
+ vdso_pagelist = kmalloc_array(vdso_pages + 1, sizeof(struct page *),
+ GFP_KERNEL);
if (vdso_pagelist == NULL)
return -ENOMEM;
/* Grab the vDSO data page. */
vdso_pagelist[0] = phys_to_page(__pa_symbol(vdso_data));
-
/* Grab the vDSO code pages. */
- pfn = sym_to_pfn(vdso_start);
+ pfn = sym_to_pfn(code_start);
for (i = 0; i < vdso_pages; i++)
vdso_pagelist[i + 1] = pfn_to_page(pfn + i);
- vdso_spec[0].pages = &vdso_pagelist[0];
- vdso_spec[1].pages = &vdso_pagelist[1];
+ /* Populate the special mapping structures */
+ mappings->data_mapping = (struct vm_special_mapping) {
+ .name = "[vvar]",
+ .pages = &vdso_pagelist[0],
+ };
+
+ mappings->code_mapping = (struct vm_special_mapping) {
+ .name = "[vdso]",
+ .pages = &vdso_pagelist[1],
+ };
+ mappings->num_code_pages = vdso_pages;
return 0;
}
+
+static struct vdso_mappings vdso_mappings __ro_after_init;
+
+static int __init vdso_init(void)
+{
+ extern char vdso_start[], vdso_end[];
+
+ return vdso_mappings_init("vdso", vdso_start, vdso_end,
+ &vdso_mappings);
+}
arch_initcall(vdso_init);
-int arch_setup_additional_pages(struct linux_binprm *bprm,
- int uses_interp)
+static int vdso_setup(struct mm_struct *mm,
+ const struct vdso_mappings *mappings)
{
- struct mm_struct *mm = current->mm;
unsigned long vdso_base, vdso_text_len, vdso_mapping_len;
void *ret;
- vdso_text_len = vdso_pages << PAGE_SHIFT;
+ vdso_text_len = mappings->num_code_pages << PAGE_SHIFT;
/* Be sure to map the data page */
vdso_mapping_len = vdso_text_len + PAGE_SIZE;
- if (down_write_killable(&mm->mmap_sem))
- return -EINTR;
vdso_base = get_unmapped_area(NULL, 0, vdso_mapping_len, 0, 0);
- if (IS_ERR_VALUE(vdso_base)) {
- ret = ERR_PTR(vdso_base);
- goto up_fail;
- }
+ if (IS_ERR_VALUE(vdso_base))
+ ret = PTR_ERR_OR_ZERO(ERR_PTR(vdso_base));
+
ret = _install_special_mapping(mm, vdso_base, PAGE_SIZE,
VM_READ|VM_MAYREAD,
- &vdso_spec[0]);
+ &mappings->data_mapping);
if (IS_ERR(ret))
- goto up_fail;
+ return PTR_ERR_OR_ZERO(ret);
vdso_base += PAGE_SIZE;
- mm->context.vdso = (void *)vdso_base;
ret = _install_special_mapping(mm, vdso_base, vdso_text_len,
VM_READ|VM_EXEC|
VM_MAYREAD|VM_MAYWRITE|VM_MAYEXEC,
- &vdso_spec[1]);
- if (IS_ERR(ret))
- goto up_fail;
+ &mappings->code_mapping);
+ if (!IS_ERR(ret))
+ mm->context.vdso = (void *)vdso_base;
+
+ return PTR_ERR_OR_ZERO(ret);
+}
+int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
+{
+ struct mm_struct *mm = current->mm;
+ int ret;
- up_write(&mm->mmap_sem);
- return 0;
+ if (down_write_killable(&mm->mmap_sem))
+ return -EINTR;
+
+ ret = vdso_setup(mm, &vdso_mappings);
-up_fail:
- mm->context.vdso = NULL;
up_write(&mm->mmap_sem);
- return PTR_ERR(ret);
+ return ret;
}
/*
--
2.18.0.rc1.244.gcf134e6275-goog
^ permalink raw reply related
* RESEND [PATCH 2/6] arm64: elf: Set AT_SYSINFO_EHDR in compat processes
From: Mark Salyzyn @ 2018-06-18 15:06 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20180618150613.10322-1-salyzyn@android.com>
From: Kevin Brodsky <kevin.brodsky@arm.com>
If the compat vDSO is enabled, we need to set AT_SYSINFO_EHDR in the
auxiliary vector of compat processes to the address of the vDSO code
page, so that the dynamic linker can find it (just like the regular vDSO).
Note that we cast context.vdso to Elf64_Off, instead of elf_addr_t,
because elf_addr_t is Elf32_Off in compat_binfmt_elf.c, and casting
context.vdso to u32 would trigger a pointer narrowing warning.
Signed-off-by: Kevin Brodsky <kevin.brodsky@arm.com>
Signed-off-by: Mark Salyzyn <salyzyn@android.com>
Cc: James Morse <james.morse@arm.com>
Cc: Russell King <linux@armlinux.org.uk>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dmitry Safonov <dsafonov@virtuozzo.com>
Cc: John Stultz <john.stultz@linaro.org>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Laura Abbott <labbott@redhat.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Andy Gross <andy.gross@linaro.org>
Cc: Andrew Pinski <apinski@cavium.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-kernel at vger.kernel.org
Cc: linux-arm-kernel at lists.infradead.org
Cc: Jeremy Linton <Jeremy.Linton@arm.com>
---
arch/arm64/include/asm/elf.h | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
diff --git a/arch/arm64/include/asm/elf.h b/arch/arm64/include/asm/elf.h
index 433b9554c6a1..bf4672a0491b 100644
--- a/arch/arm64/include/asm/elf.h
+++ b/arch/arm64/include/asm/elf.h
@@ -147,10 +147,10 @@ typedef struct user_fpsimd_state elf_fpregset_t;
})
/* update AT_VECTOR_SIZE_ARCH if the number of NEW_AUX_ENT entries changes */
-#define ARCH_DLINFO \
+#define _SET_AUX_ENT_VDSO \
do { \
NEW_AUX_ENT(AT_SYSINFO_EHDR, \
- (elf_addr_t)current->mm->context.vdso); \
+ (Elf64_Off)current->mm->context.vdso); \
\
/* \
* Should always be nonzero unless there's a kernel bug. \
@@ -162,6 +162,7 @@ do { \
else \
NEW_AUX_ENT(AT_IGNORE, 0); \
} while (0)
+#define ARCH_DLINFO _SET_AUX_ENT_VDSO
#define ARCH_HAS_SETUP_ADDITIONAL_PAGES
struct linux_binprm;
@@ -209,7 +210,11 @@ typedef compat_elf_greg_t compat_elf_gregset_t[COMPAT_ELF_NGREG];
({ \
set_thread_flag(TIF_32BIT); \
})
+#ifdef CONFIG_VDSO32
+#define COMPAT_ARCH_DLINFO _SET_AUX_ENT_VDSO
+#else
#define COMPAT_ARCH_DLINFO
+#endif
extern int aarch32_setup_vectors_page(struct linux_binprm *bprm,
int uses_interp);
#define compat_arch_setup_additional_pages \
--
2.18.0.rc1.244.gcf134e6275-goog
^ permalink raw reply related
* RESEND [PATCH 1/6] arm64: compat: Use vDSO sigreturn trampolines if available
From: Mark Salyzyn @ 2018-06-18 15:06 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20180618150613.10322-1-salyzyn@android.com>
From: Kevin Brodsky <kevin.brodsky@arm.com>
If the compat vDSO is enabled, it replaces the sigreturn page.
Therefore, we use the sigreturn trampolines the vDSO provides instead.
Signed-off-by: Kevin Brodsky <kevin.brodsky@arm.com>
Signed-off-by: Mark Salyzyn <salyzyn@android.com>
Cc: James Morse <james.morse@arm.com>
Cc: Russell King <linux@armlinux.org.uk>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dmitry Safonov <dsafonov@virtuozzo.com>
Cc: John Stultz <john.stultz@linaro.org>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Laura Abbott <labbott@redhat.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Andy Gross <andy.gross@linaro.org>
Cc: Andrew Pinski <apinski@cavium.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-kernel at vger.kernel.org
Cc: linux-arm-kernel at lists.infradead.org
Cc: Jeremy Linton <Jeremy.Linton@arm.com>
---
arch/arm64/include/asm/vdso.h | 3 +++
arch/arm64/kernel/signal32.c | 15 +++++++++++++++
2 files changed, 18 insertions(+)
diff --git a/arch/arm64/include/asm/vdso.h b/arch/arm64/include/asm/vdso.h
index 839ce0031bd5..f2a952338f1e 100644
--- a/arch/arm64/include/asm/vdso.h
+++ b/arch/arm64/include/asm/vdso.h
@@ -28,6 +28,9 @@
#ifndef __ASSEMBLY__
#include <generated/vdso-offsets.h>
+#ifdef CONFIG_VDSO32
+#include <generated/vdso32-offsets.h>
+#endif
#define VDSO_SYMBOL(base, name) \
({ \
diff --git a/arch/arm64/kernel/signal32.c b/arch/arm64/kernel/signal32.c
index 096c88731c56..5ee07da53fd1 100644
--- a/arch/arm64/kernel/signal32.c
+++ b/arch/arm64/kernel/signal32.c
@@ -29,6 +29,7 @@
#include <asm/traps.h>
#include <linux/uaccess.h>
#include <asm/unistd.h>
+#include <asm/vdso.h>
struct compat_vfp_sigframe {
compat_ulong_t magic;
@@ -347,6 +348,19 @@ static void compat_setup_return(struct pt_regs *regs, struct k_sigaction *ka,
retcode = ptr_to_compat(ka->sa.sa_restorer);
} else {
/* Set up sigreturn pointer */
+#ifdef CONFIG_VDSO32
+ void *vdso_base = current->mm->context.vdso;
+ void *trampoline =
+ (ka->sa.sa_flags & SA_SIGINFO
+ ? (thumb
+ ? VDSO_SYMBOL(vdso_base, compat_rt_sigreturn_thumb)
+ : VDSO_SYMBOL(vdso_base, compat_rt_sigreturn_arm))
+ : (thumb
+ ? VDSO_SYMBOL(vdso_base, compat_sigreturn_thumb)
+ : VDSO_SYMBOL(vdso_base, compat_sigreturn_arm)));
+
+ retcode = ptr_to_compat(trampoline) + thumb;
+#else
void *sigreturn_base = current->mm->context.vdso;
unsigned int idx = thumb << 1;
@@ -354,6 +368,7 @@ static void compat_setup_return(struct pt_regs *regs, struct k_sigaction *ka,
idx += 3;
retcode = ptr_to_compat(sigreturn_base) + (idx << 2) + thumb;
+#endif
}
regs->regs[0] = usig;
--
2.18.0.rc1.244.gcf134e6275-goog
^ permalink raw reply related
* RESEND [PATCH] arm64: compat: Expose offset to registers in sigframes
From: Mark Salyzyn @ 2018-06-18 15:06 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20180618150613.10322-1-salyzyn@android.com>
From: Kevin Brodsky <kevin.brodsky@arm.com>
This will be needed to provide unwinding information in compat
sigreturn trampolines, part of the future compat vDSO. There is no
obvious header the compat_sig* struct's should be moved to, so let's
put them in signal32.h.
Also fix minor style issues reported by checkpatch.
Signed-off-by: Kevin Brodsky <kevin.brodsky@arm.com>
Signed-off-by: Mark Salyzyn <salyzyn@android.com>
Cc: James Morse <james.morse@arm.com>
Cc: Russell King <linux@armlinux.org.uk>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dmitry Safonov <dsafonov@virtuozzo.com>
Cc: John Stultz <john.stultz@linaro.org>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Laura Abbott <labbott@redhat.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Andy Gross <andy.gross@linaro.org>
Cc: Kevin Brodsky <kevin.brodsky@arm.com>
Cc: Andrew Pinski <apinski@cavium.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-kernel at vger.kernel.org
Cc: linux-arm-kernel at lists.infradead.org
Cc: Dave Martin <Dave.Martin@arm.com>
Cc: Eric W. Biederman <ebiederm@xmission.com>
Cc: Jeremy Linton <Jeremy.Linton@arm.com>
NB:
Basically unchanged as part of a vDSO32 effort through 4 revisions,
Resubmitted as a standalone change for quciker approval.
---
arch/arm64/include/asm/signal32.h | 46 +++++++++++++++++++++++++++++++
arch/arm64/kernel/asm-offsets.c | 13 +++++++++
arch/arm64/kernel/signal32.c | 46 -------------------------------
3 files changed, 59 insertions(+), 46 deletions(-)
diff --git a/arch/arm64/include/asm/signal32.h b/arch/arm64/include/asm/signal32.h
index 58e288aaf0ba..bcd0e139ee4a 100644
--- a/arch/arm64/include/asm/signal32.h
+++ b/arch/arm64/include/asm/signal32.h
@@ -20,6 +20,52 @@
#ifdef CONFIG_COMPAT
#include <linux/compat.h>
+struct compat_sigcontext {
+ /* We always set these two fields to 0 */
+ compat_ulong_t trap_no;
+ compat_ulong_t error_code;
+
+ compat_ulong_t oldmask;
+ compat_ulong_t arm_r0;
+ compat_ulong_t arm_r1;
+ compat_ulong_t arm_r2;
+ compat_ulong_t arm_r3;
+ compat_ulong_t arm_r4;
+ compat_ulong_t arm_r5;
+ compat_ulong_t arm_r6;
+ compat_ulong_t arm_r7;
+ compat_ulong_t arm_r8;
+ compat_ulong_t arm_r9;
+ compat_ulong_t arm_r10;
+ compat_ulong_t arm_fp;
+ compat_ulong_t arm_ip;
+ compat_ulong_t arm_sp;
+ compat_ulong_t arm_lr;
+ compat_ulong_t arm_pc;
+ compat_ulong_t arm_cpsr;
+ compat_ulong_t fault_address;
+};
+
+struct compat_ucontext {
+ compat_ulong_t uc_flags;
+ compat_uptr_t uc_link;
+ compat_stack_t uc_stack;
+ struct compat_sigcontext uc_mcontext;
+ compat_sigset_t uc_sigmask;
+ int __unused[32 - (sizeof(compat_sigset_t) / sizeof(int))];
+ compat_ulong_t uc_regspace[128] __aligned(8);
+};
+
+struct compat_sigframe {
+ struct compat_ucontext uc;
+ compat_ulong_t retcode[2];
+};
+
+struct compat_rt_sigframe {
+ struct compat_siginfo info;
+ struct compat_sigframe sig;
+};
+
int compat_setup_frame(int usig, struct ksignal *ksig, sigset_t *set,
struct pt_regs *regs);
int compat_setup_rt_frame(int usig, struct ksignal *ksig, sigset_t *set,
diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c
index 8938a4223690..a79507c5d845 100644
--- a/arch/arm64/kernel/asm-offsets.c
+++ b/arch/arm64/kernel/asm-offsets.c
@@ -29,6 +29,7 @@
#include <asm/fixmap.h>
#include <asm/thread_info.h>
#include <asm/memory.h>
+#include <asm/signal32.h>
#include <asm/smp_plat.h>
#include <asm/suspend.h>
#include <asm/vdso_datapage.h>
@@ -81,6 +82,18 @@ int main(void)
DEFINE(S_STACKFRAME, offsetof(struct pt_regs, stackframe));
DEFINE(S_FRAME_SIZE, sizeof(struct pt_regs));
BLANK();
+#ifdef CONFIG_COMPAT
+ DEFINE(COMPAT_SIGFRAME_REGS_OFFSET,
+ offsetof(struct compat_sigframe, uc) +
+ offsetof(struct compat_ucontext, uc_mcontext) +
+ offsetof(struct compat_sigcontext, arm_r0));
+ DEFINE(COMPAT_RT_SIGFRAME_REGS_OFFSET,
+ offsetof(struct compat_rt_sigframe, sig) +
+ offsetof(struct compat_sigframe, uc) +
+ offsetof(struct compat_ucontext, uc_mcontext) +
+ offsetof(struct compat_sigcontext, arm_r0));
+ BLANK();
+#endif
DEFINE(MM_CONTEXT_ID, offsetof(struct mm_struct, context.id.counter));
BLANK();
DEFINE(VMA_VM_MM, offsetof(struct vm_area_struct, vm_mm));
diff --git a/arch/arm64/kernel/signal32.c b/arch/arm64/kernel/signal32.c
index 9c018878056b..096c88731c56 100644
--- a/arch/arm64/kernel/signal32.c
+++ b/arch/arm64/kernel/signal32.c
@@ -30,42 +30,6 @@
#include <linux/uaccess.h>
#include <asm/unistd.h>
-struct compat_sigcontext {
- /* We always set these two fields to 0 */
- compat_ulong_t trap_no;
- compat_ulong_t error_code;
-
- compat_ulong_t oldmask;
- compat_ulong_t arm_r0;
- compat_ulong_t arm_r1;
- compat_ulong_t arm_r2;
- compat_ulong_t arm_r3;
- compat_ulong_t arm_r4;
- compat_ulong_t arm_r5;
- compat_ulong_t arm_r6;
- compat_ulong_t arm_r7;
- compat_ulong_t arm_r8;
- compat_ulong_t arm_r9;
- compat_ulong_t arm_r10;
- compat_ulong_t arm_fp;
- compat_ulong_t arm_ip;
- compat_ulong_t arm_sp;
- compat_ulong_t arm_lr;
- compat_ulong_t arm_pc;
- compat_ulong_t arm_cpsr;
- compat_ulong_t fault_address;
-};
-
-struct compat_ucontext {
- compat_ulong_t uc_flags;
- compat_uptr_t uc_link;
- compat_stack_t uc_stack;
- struct compat_sigcontext uc_mcontext;
- compat_sigset_t uc_sigmask;
- int __unused[32 - (sizeof (compat_sigset_t) / sizeof (int))];
- compat_ulong_t uc_regspace[128] __attribute__((__aligned__(8)));
-};
-
struct compat_vfp_sigframe {
compat_ulong_t magic;
compat_ulong_t size;
@@ -92,16 +56,6 @@ struct compat_aux_sigframe {
unsigned long end_magic;
} __attribute__((__aligned__(8)));
-struct compat_sigframe {
- struct compat_ucontext uc;
- compat_ulong_t retcode[2];
-};
-
-struct compat_rt_sigframe {
- struct compat_siginfo info;
- struct compat_sigframe sig;
-};
-
#define _BLOCKABLE (~(sigmask(SIGKILL) | sigmask(SIGSTOP)))
static inline int put_sigset_t(compat_sigset_t __user *uset, sigset_t *set)
--
2.18.0.rc1.244.gcf134e6275-goog
^ permalink raw reply related
* RESEND [PATCH v2 3/3] arm64: compat: Add CONFIG_KUSER_HELPERS
From: Mark Salyzyn @ 2018-06-18 15:06 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20180618150613.10322-1-salyzyn@android.com>
From: Kevin Brodsky <kevin.brodsky@arm.com>
Make it possible to disable the kuser helpers by adding a KUSER_HELPERS
config option (enabled by default). When disabled, all kuser
helpers-related code is removed from the kernel and no mapping is done
at the fixed high address (0xffff0000); any attempt to use a kuser
helper from a 32-bit process will result in a segfault.
Signed-off-by: Kevin Brodsky <kevin.brodsky@arm.com>
Signed-off-by: Mark Salyzyn <salyzyn@android.com>
Cc: James Morse <james.morse@arm.com>
Cc: Russell King <linux@armlinux.org.uk>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dmitry Safonov <dsafonov@virtuozzo.com>
Cc: John Stultz <john.stultz@linaro.org>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Laura Abbott <labbott@redhat.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Andy Gross <andy.gross@linaro.org>
Cc: Andrew Pinski <apinski@cavium.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-kernel at vger.kernel.org
Cc: linux-arm-kernel at lists.infradead.org
Cc: Jeremy Linton <Jeremy.Linton@arm.com>
v2:
- split off assembler changes to a new previous patch in series to reduce churn
- modify slightly the feature documentation to reduce its reach
- modify slightly the feature documentation to rationalize the yes default.
- There are more ifdefs as a result of the rebase.
v3:
- rebase
---
arch/arm64/Kconfig | 30 ++++++++++++++++++++++++++++++
arch/arm64/kernel/Makefile | 2 +-
arch/arm64/kernel/vdso.c | 10 ++++++++++
3 files changed, 41 insertions(+), 1 deletion(-)
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index 42c090cf0292..11b4c6aef7d7 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -1296,6 +1296,36 @@ config COMPAT
If you want to execute 32-bit userspace applications, say Y.
+config KUSER_HELPERS
+ bool "Enable the kuser helpers page in 32-bit processes"
+ depends on COMPAT
+ default y
+ help
+ Warning: disabling this option may break 32-bit applications.
+
+ Provide kuser helpers in a special purpose fixed-address page. The
+ kernel provides helper code to userspace in read-only form at a fixed
+ location to allow userspace to be independent of the CPU type fitted
+ to the system. This permits 32-bit binaries to be run on ARMv6 through
+ to ARMv8 without modification.
+
+ See Documentation/arm/kernel_user_helpers.txt for details.
+
+ However, the fixed-address nature of these helpers can be used by ROP
+ (return-orientated programming) authors when creating exploits.
+
+ If all of the 32-bit binaries and libraries that run on your platform
+ are built specifically for your platform, and make no use of these
+ helpers, then you can turn this option off to hinder such exploits.
+ However, in that case, if a binary or library relying on those helpers
+ is run, it will receive a SIGSEGV signal, which will terminate the
+ program. Typically, binaries compiled for ARMv7 or later do not use
+ the kuser helpers.
+
+ Say N here only if you are absolutely certain that you do not need
+ these helpers; otherwise, the safe option is to say Y (the default
+ for now)
+
config SYSVIPC_COMPAT
def_bool y
depends on COMPAT && SYSVIPC
diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile
index 9851be3ef932..c16be3cf39bf 100644
--- a/arch/arm64/kernel/Makefile
+++ b/arch/arm64/kernel/Makefile
@@ -29,7 +29,7 @@ $(obj)/%.stub.o: $(obj)/%.o FORCE
arm64-obj-$(CONFIG_COMPAT) += sys32.o signal32.o \
sys_compat.o entry32.o
arm64-obj-$(CONFIG_COMPAT) += sigreturn32.o
-arm64-obj-$(CONFIG_COMPAT) += kuser32.o
+arm64-obj-$(CONFIG_KUSER_HELPERS) += kuser32.o
arm64-obj-$(CONFIG_FUNCTION_TRACER) += ftrace.o entry-ftrace.o
arm64-obj-$(CONFIG_MODULES) += arm64ksyms.o module.o
arm64-obj-$(CONFIG_ARM64_MODULE_PLTS) += module-plts.o
diff --git a/arch/arm64/kernel/vdso.c b/arch/arm64/kernel/vdso.c
index 5398f6454ce1..76a94bed4bd5 100644
--- a/arch/arm64/kernel/vdso.c
+++ b/arch/arm64/kernel/vdso.c
@@ -62,18 +62,22 @@ static const struct vm_special_mapping compat_vdso_spec[] = {
.name = "[sigpage]",
.pages = &vectors_page[0],
},
+#ifdef CONFIG_KUSER_HELPERS
{
.name = "[kuserhelpers]",
.pages = &vectors_page[1],
},
+#endif
};
static struct page *vectors_page[ARRAY_SIZE(compat_vdso_spec)] __ro_after_init;
static int __init alloc_vectors_page(void)
{
+#ifdef CONFIG_KUSER_HELPERS
extern char __kuser_helper_start[], __kuser_helper_end[];
size_t kuser_sz = __kuser_helper_end - __kuser_helper_start;
unsigned long kuser_vpage;
+#endif
extern char __aarch32_sigret_code_start[], __aarch32_sigret_code_end[];
size_t sigret_sz =
@@ -84,22 +88,26 @@ static int __init alloc_vectors_page(void)
if (!sigret_vpage)
return -ENOMEM;
+#ifdef CONFIG_KUSER_HELPERS
kuser_vpage = get_zeroed_page(GFP_ATOMIC);
if (!kuser_vpage) {
free_page(sigret_vpage);
return -ENOMEM;
}
+#endif
/* sigreturn code */
memcpy((void *)sigret_vpage, __aarch32_sigret_code_start, sigret_sz);
flush_icache_range(sigret_vpage, sigret_vpage + PAGE_SIZE);
vectors_page[0] = virt_to_page(sigret_vpage);
+#ifdef CONFIG_KUSER_HELPERS
/* kuser helpers */
memcpy((void *)kuser_vpage + 0x1000 - kuser_sz, __kuser_helper_start,
kuser_sz);
flush_icache_range(kuser_vpage, kuser_vpage + PAGE_SIZE);
vectors_page[1] = virt_to_page(kuser_vpage);
+#endif
return 0;
}
@@ -128,11 +136,13 @@ int aarch32_setup_vectors_page(struct linux_binprm *bprm, int uses_interp)
current->mm->context.vdso = (void *)addr;
+#ifdef CONFIG_KUSER_HELPERS
/* Map the kuser helpers at the ABI-defined high address. */
ret = _install_special_mapping(mm, AARCH32_KUSER_HELPERS_BASE,
PAGE_SIZE,
VM_READ|VM_EXEC|VM_MAYREAD|VM_MAYEXEC,
&compat_vdso_spec[1]);
+#endif
out:
up_write(&mm->mmap_sem);
--
2.18.0.rc1.244.gcf134e6275-goog
^ permalink raw reply related
* RESEND [PATCH v2 2/3] arm64: compat: Split the sigreturn trampolines and kuser helpers (assembler sources)
From: Mark Salyzyn @ 2018-06-18 15:06 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20180618150613.10322-1-salyzyn@android.com>
From: Kevin Brodsky <kevin.brodsky@arm.com>
AArch32 processes are currently installed a special [vectors] page that
contains the sigreturn trampolines and the kuser helpers, at the fixed
address mandated by the kuser helpers ABI.
Having both functionalities in the same page has become problematic,
because:
* It makes it impossible to disable the kuser helpers (the sigreturn
trampolines cannot be removed), which is possible on arm.
* A future 32-bit vDSO would provide the sigreturn trampolines itself,
making those in [vectors] redundant.
This patch addresses the problem by moving the sigreturn trampolines
sources to its own file. Wrapped the comments to reduce the wrath of
checkpatch.pl.
Signed-off-by: Kevin Brodsky <kevin.brodsky@arm.com>
Signed-off-by: Mark Salyzyn <salyzyn@android.com>
Cc: James Morse <james.morse@arm.com>
Cc: Russell King <linux@armlinux.org.uk>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dmitry Safonov <dsafonov@virtuozzo.com>
Cc: John Stultz <john.stultz@linaro.org>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Laura Abbott <labbott@redhat.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Andy Gross <andy.gross@linaro.org>
Cc: Andrew Pinski <apinski@cavium.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-kernel at vger.kernel.org
Cc: linux-arm-kernel at lists.infradead.org
Cc: Jeremy Linton <Jeremy.Linton@arm.com>
v2:
- split off from previous v1 'arm64: compat: Add CONFIG_KUSER_HELPERS'
- adjust makefile so one line for each of the assembler source modules
v3:
- rebase
---
arch/arm64/kernel/Makefile | 4 +-
arch/arm64/kernel/kuser32.S | 48 ++---------------------
arch/arm64/kernel/sigreturn32.S | 67 +++++++++++++++++++++++++++++++++
3 files changed, 73 insertions(+), 46 deletions(-)
create mode 100644 arch/arm64/kernel/sigreturn32.S
diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile
index 0025f8691046..9851be3ef932 100644
--- a/arch/arm64/kernel/Makefile
+++ b/arch/arm64/kernel/Makefile
@@ -26,8 +26,10 @@ OBJCOPYFLAGS := --prefix-symbols=__efistub_
$(obj)/%.stub.o: $(obj)/%.o FORCE
$(call if_changed,objcopy)
-arm64-obj-$(CONFIG_COMPAT) += sys32.o kuser32.o signal32.o \
+arm64-obj-$(CONFIG_COMPAT) += sys32.o signal32.o \
sys_compat.o entry32.o
+arm64-obj-$(CONFIG_COMPAT) += sigreturn32.o
+arm64-obj-$(CONFIG_COMPAT) += kuser32.o
arm64-obj-$(CONFIG_FUNCTION_TRACER) += ftrace.o entry-ftrace.o
arm64-obj-$(CONFIG_MODULES) += arm64ksyms.o module.o
arm64-obj-$(CONFIG_ARM64_MODULE_PLTS) += module-plts.o
diff --git a/arch/arm64/kernel/kuser32.S b/arch/arm64/kernel/kuser32.S
index 997e6b27ff6a..d15b5c2935b3 100644
--- a/arch/arm64/kernel/kuser32.S
+++ b/arch/arm64/kernel/kuser32.S
@@ -20,16 +20,13 @@
*
* AArch32 user helpers.
*
- * Each segment is 32-byte aligned and will be moved to the top of the high
- * vector page. New segments (if ever needed) must be added in front of
- * existing ones. This mechanism should be used only for things that are
- * really small and justified, and not be abused freely.
+ * These helpers are provided for compatibility with AArch32 binaries that
+ * still need them. They are installed at a fixed address by
+ * aarch32_setup_additional_pages().
*
* See Documentation/arm/kernel_user_helpers.txt for formal definitions.
*/
-#include <asm/unistd.h>
-
.align 5
.globl __kuser_helper_start
__kuser_helper_start:
@@ -77,42 +74,3 @@ __kuser_helper_version: // 0xffff0ffc
.word ((__kuser_helper_end - __kuser_helper_start) >> 5)
.globl __kuser_helper_end
__kuser_helper_end:
-
-/*
- * AArch32 sigreturn code
- *
- * For ARM syscalls, the syscall number has to be loaded into r7.
- * We do not support an OABI userspace.
- *
- * For Thumb syscalls, we also pass the syscall number via r7. We therefore
- * need two 16-bit instructions.
- */
- .globl __aarch32_sigret_code_start
-__aarch32_sigret_code_start:
-
- /*
- * ARM Code
- */
- .byte __NR_compat_sigreturn, 0x70, 0xa0, 0xe3 // mov r7, #__NR_compat_sigreturn
- .byte __NR_compat_sigreturn, 0x00, 0x00, 0xef // svc #__NR_compat_sigreturn
-
- /*
- * Thumb code
- */
- .byte __NR_compat_sigreturn, 0x27 // svc #__NR_compat_sigreturn
- .byte __NR_compat_sigreturn, 0xdf // mov r7, #__NR_compat_sigreturn
-
- /*
- * ARM code
- */
- .byte __NR_compat_rt_sigreturn, 0x70, 0xa0, 0xe3 // mov r7, #__NR_compat_rt_sigreturn
- .byte __NR_compat_rt_sigreturn, 0x00, 0x00, 0xef // svc #__NR_compat_rt_sigreturn
-
- /*
- * Thumb code
- */
- .byte __NR_compat_rt_sigreturn, 0x27 // svc #__NR_compat_rt_sigreturn
- .byte __NR_compat_rt_sigreturn, 0xdf // mov r7, #__NR_compat_rt_sigreturn
-
- .globl __aarch32_sigret_code_end
-__aarch32_sigret_code_end:
diff --git a/arch/arm64/kernel/sigreturn32.S b/arch/arm64/kernel/sigreturn32.S
new file mode 100644
index 000000000000..6ecda4d84cd5
--- /dev/null
+++ b/arch/arm64/kernel/sigreturn32.S
@@ -0,0 +1,67 @@
+/*
+ * sigreturn trampolines for AArch32.
+ *
+ * Copyright (C) 2005-2011 Nicolas Pitre <nico@fluxnic.net>
+ * Copyright (C) 2012 ARM Ltd.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ *
+ * AArch32 sigreturn code
+ *
+ * For ARM syscalls, the syscall number has to be loaded into r7.
+ * We do not support an OABI userspace.
+ *
+ * For Thumb syscalls, we also pass the syscall number via r7. We therefore
+ * need two 16-bit instructions.
+ */
+
+#include <asm/unistd.h>
+
+ .globl __aarch32_sigret_code_start
+__aarch32_sigret_code_start:
+
+ /*
+ * ARM Code
+ */
+ // mov r7, #__NR_compat_sigreturn
+ .byte __NR_compat_sigreturn, 0x70, 0xa0, 0xe3
+ // svc #__NR_compat_sigreturn
+ .byte __NR_compat_sigreturn, 0x00, 0x00, 0xef
+
+ /*
+ * Thumb code
+ */
+ // svc #__NR_compat_sigreturn
+ .byte __NR_compat_sigreturn, 0x27
+ // mov r7, #__NR_compat_sigreturn
+ .byte __NR_compat_sigreturn, 0xdf
+
+ /*
+ * ARM code
+ */
+ // mov r7, #__NR_compat_rt_sigreturn
+ .byte __NR_compat_rt_sigreturn, 0x70, 0xa0, 0xe3
+ // svc #__NR_compat_rt_sigreturn
+ .byte __NR_compat_rt_sigreturn, 0x00, 0x00, 0xef
+
+ /*
+ * Thumb code
+ */
+ // svc #__NR_compat_rt_sigreturn
+ .byte __NR_compat_rt_sigreturn, 0x27
+ // mov r7, #__NR_compat_rt_sigreturn
+ .byte __NR_compat_rt_sigreturn, 0xdf
+
+ .globl __aarch32_sigret_code_end
+__aarch32_sigret_code_end:
--
2.18.0.rc1.244.gcf134e6275-goog
^ permalink raw reply related
* RESEND [PATCH v2 1/3] arm64: compat: Split the sigreturn trampolines and kuser helpers (C sources)
From: Mark Salyzyn @ 2018-06-18 15:06 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20180618150613.10322-1-salyzyn@android.com>
From: Kevin Brodsky <kevin.brodsky@arm.com>
AArch32 processes are currently installed a special [vectors] page that
contains the sigreturn trampolines and the kuser helpers, at the fixed
address mandated by the kuser helpers ABI.
Having both functionalities in the same page has become problematic,
because:
* It makes it impossible to disable the kuser helpers (the sigreturn
trampolines cannot be removed), which is possible on arm.
* A future 32-bit vDSO would provide the sigreturn trampolines itself,
making those in [vectors] redundant.
This patch addresses the problem by moving the sigreturn trampolines to
a separate [sigpage] page, mirroring [sigpage] on arm.
Even though [vectors] has always been a misnomer on arm64/compat, as
there is no AArch32 vector there (and now only the kuser helpers),
its name has been left unchanged, for compatibility with arm (there
are reports of software relying on [vectors] being there as the last
mapping in /proc/maps).
mm->context.vdso used to point to the [vectors] page, which is
unnecessary (as its address is fixed). It now points to the [sigpage]
page (whose address is randomized like a vDSO).
Signed-off-by: Kevin Brodsky <kevin.brodsky@arm.com>
Signed-off-by: Mark Salyzyn <salyzyn@android.com>
Cc: James Morse <james.morse@arm.com>
Cc: Russell King <linux@armlinux.org.uk>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dmitry Safonov <dsafonov@virtuozzo.com>
Cc: John Stultz <john.stultz@linaro.org>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Laura Abbott <labbott@redhat.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Andy Gross <andy.gross@linaro.org>
Cc: Andrew Pinski <apinski@cavium.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-kernel at vger.kernel.org
Cc: linux-arm-kernel at lists.infradead.org
Cc: Jeremy Linton <Jeremy.Linton@arm.com>
v2:
- reduce churniness (and defer later to vDSO patches)
- vectors_page and compat_vdso_spec as array of 2
- free sigpage if vectors allocation failed
v3:
- rebase
---
arch/arm64/include/asm/processor.h | 4 +-
arch/arm64/include/asm/signal32.h | 2 -
arch/arm64/kernel/signal32.c | 5 +-
arch/arm64/kernel/vdso.c | 82 ++++++++++++++++++++----------
4 files changed, 60 insertions(+), 33 deletions(-)
diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h
index a73ae1e49200..3bcb897cb972 100644
--- a/arch/arm64/include/asm/processor.h
+++ b/arch/arm64/include/asm/processor.h
@@ -66,9 +66,9 @@
#define STACK_TOP_MAX TASK_SIZE_64
#ifdef CONFIG_COMPAT
-#define AARCH32_VECTORS_BASE 0xffff0000
+#define AARCH32_KUSER_HELPERS_BASE 0xffff0000
#define STACK_TOP (test_thread_flag(TIF_32BIT) ? \
- AARCH32_VECTORS_BASE : STACK_TOP_MAX)
+ AARCH32_KUSER_HELPERS_BASE : STACK_TOP_MAX)
#else
#define STACK_TOP STACK_TOP_MAX
#endif /* CONFIG_COMPAT */
diff --git a/arch/arm64/include/asm/signal32.h b/arch/arm64/include/asm/signal32.h
index 81abea0b7650..58e288aaf0ba 100644
--- a/arch/arm64/include/asm/signal32.h
+++ b/arch/arm64/include/asm/signal32.h
@@ -20,8 +20,6 @@
#ifdef CONFIG_COMPAT
#include <linux/compat.h>
-#define AARCH32_KERN_SIGRET_CODE_OFFSET 0x500
-
int compat_setup_frame(int usig, struct ksignal *ksig, sigset_t *set,
struct pt_regs *regs);
int compat_setup_rt_frame(int usig, struct ksignal *ksig, sigset_t *set,
diff --git a/arch/arm64/kernel/signal32.c b/arch/arm64/kernel/signal32.c
index 77b91f478995..9c018878056b 100644
--- a/arch/arm64/kernel/signal32.c
+++ b/arch/arm64/kernel/signal32.c
@@ -393,14 +393,13 @@ static void compat_setup_return(struct pt_regs *regs, struct k_sigaction *ka,
retcode = ptr_to_compat(ka->sa.sa_restorer);
} else {
/* Set up sigreturn pointer */
+ void *sigreturn_base = current->mm->context.vdso;
unsigned int idx = thumb << 1;
if (ka->sa.sa_flags & SA_SIGINFO)
idx += 3;
- retcode = AARCH32_VECTORS_BASE +
- AARCH32_KERN_SIGRET_CODE_OFFSET +
- (idx << 2) + thumb;
+ retcode = ptr_to_compat(sigreturn_base) + (idx << 2) + thumb;
}
regs->regs[0] = usig;
diff --git a/arch/arm64/kernel/vdso.c b/arch/arm64/kernel/vdso.c
index 8dd2ad220a0f..5398f6454ce1 100644
--- a/arch/arm64/kernel/vdso.c
+++ b/arch/arm64/kernel/vdso.c
@@ -1,5 +1,7 @@
/*
- * VDSO implementation for AArch64 and vector page setup for AArch32.
+ * Additional userspace pages setup for AArch64 and AArch32.
+ * - AArch64: vDSO pages setup, vDSO data page update.
+ * - AArch32: sigreturn and kuser helpers pages setup.
*
* Copyright (C) 2012 ARM Limited
*
@@ -53,32 +55,51 @@ struct vdso_data *vdso_data = &vdso_data_store.data;
/*
* Create and map the vectors page for AArch32 tasks.
*/
-static struct page *vectors_page[1] __ro_after_init;
+static struct page *vectors_page[] __ro_after_init;
+static const struct vm_special_mapping compat_vdso_spec[] = {
+ {
+ /* Must be named [sigpage] for compatibility with arm. */
+ .name = "[sigpage]",
+ .pages = &vectors_page[0],
+ },
+ {
+ .name = "[kuserhelpers]",
+ .pages = &vectors_page[1],
+ },
+};
+static struct page *vectors_page[ARRAY_SIZE(compat_vdso_spec)] __ro_after_init;
static int __init alloc_vectors_page(void)
{
extern char __kuser_helper_start[], __kuser_helper_end[];
- extern char __aarch32_sigret_code_start[], __aarch32_sigret_code_end[];
+ size_t kuser_sz = __kuser_helper_end - __kuser_helper_start;
+ unsigned long kuser_vpage;
- int kuser_sz = __kuser_helper_end - __kuser_helper_start;
- int sigret_sz = __aarch32_sigret_code_end - __aarch32_sigret_code_start;
- unsigned long vpage;
-
- vpage = get_zeroed_page(GFP_ATOMIC);
+ extern char __aarch32_sigret_code_start[], __aarch32_sigret_code_end[];
+ size_t sigret_sz =
+ __aarch32_sigret_code_end - __aarch32_sigret_code_start;
+ unsigned long sigret_vpage;
- if (!vpage)
+ sigret_vpage = get_zeroed_page(GFP_ATOMIC);
+ if (!sigret_vpage)
return -ENOMEM;
- /* kuser helpers */
- memcpy((void *)vpage + 0x1000 - kuser_sz, __kuser_helper_start,
- kuser_sz);
+ kuser_vpage = get_zeroed_page(GFP_ATOMIC);
+ if (!kuser_vpage) {
+ free_page(sigret_vpage);
+ return -ENOMEM;
+ }
/* sigreturn code */
- memcpy((void *)vpage + AARCH32_KERN_SIGRET_CODE_OFFSET,
- __aarch32_sigret_code_start, sigret_sz);
+ memcpy((void *)sigret_vpage, __aarch32_sigret_code_start, sigret_sz);
+ flush_icache_range(sigret_vpage, sigret_vpage + PAGE_SIZE);
+ vectors_page[0] = virt_to_page(sigret_vpage);
- flush_icache_range(vpage, vpage + PAGE_SIZE);
- vectors_page[0] = virt_to_page(vpage);
+ /* kuser helpers */
+ memcpy((void *)kuser_vpage + 0x1000 - kuser_sz, __kuser_helper_start,
+ kuser_sz);
+ flush_icache_range(kuser_vpage, kuser_vpage + PAGE_SIZE);
+ vectors_page[1] = virt_to_page(kuser_vpage);
return 0;
}
@@ -87,23 +108,32 @@ arch_initcall(alloc_vectors_page);
int aarch32_setup_vectors_page(struct linux_binprm *bprm, int uses_interp)
{
struct mm_struct *mm = current->mm;
- unsigned long addr = AARCH32_VECTORS_BASE;
- static const struct vm_special_mapping spec = {
- .name = "[vectors]",
- .pages = vectors_page,
-
- };
+ unsigned long addr;
void *ret;
if (down_write_killable(&mm->mmap_sem))
return -EINTR;
- current->mm->context.vdso = (void *)addr;
+ addr = get_unmapped_area(NULL, 0, PAGE_SIZE, 0, 0);
+ if (IS_ERR_VALUE(addr)) {
+ ret = ERR_PTR(addr);
+ goto out;
+ }
- /* Map vectors page at the high address. */
ret = _install_special_mapping(mm, addr, PAGE_SIZE,
- VM_READ|VM_EXEC|VM_MAYREAD|VM_MAYEXEC,
- &spec);
+ VM_READ|VM_EXEC|
+ VM_MAYREAD|VM_MAYWRITE|VM_MAYEXEC,
+ &compat_vdso_spec[0]);
+ if (IS_ERR(ret))
+ goto out;
+ current->mm->context.vdso = (void *)addr;
+
+ /* Map the kuser helpers at the ABI-defined high address. */
+ ret = _install_special_mapping(mm, AARCH32_KUSER_HELPERS_BASE,
+ PAGE_SIZE,
+ VM_READ|VM_EXEC|VM_MAYREAD|VM_MAYEXEC,
+ &compat_vdso_spec[1]);
+out:
up_write(&mm->mmap_sem);
return PTR_ERR_OR_ZERO(ret);
--
2.18.0.rc1.244.gcf134e6275-goog
^ permalink raw reply related
* RESEND [PATCH] lib: vdso: add support for time
From: Mark Salyzyn @ 2018-06-18 15:05 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20180618150613.10322-1-salyzyn@android.com>
From: Mark Salyzyn <salyzyn@google.com>
Add time() vdso support to match up with existing support in the x86's
vdso. Currently benefitting arm and arm64 which uses the common
vgettimeofday.c implementation. On arm provides about a ~14 fold
improvement in speed over the straight syscall, and about a ~5 fold
improvement in speed over an alternate library implementation that
relies on the vdso call to gettimeofday to fulfill the request.
We can provide __vdso_time even if we can not provide a speed
enhanced __vdso_gettimeofday. (arm, virtual counters).
Signed-off-by: Mark Salyzyn <salyzyn@android.com>
Cc: James Morse <james.morse@arm.com>
Cc: Russell King <linux@armlinux.org.uk>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dmitry Safonov <dsafonov@virtuozzo.com>
Cc: John Stultz <john.stultz@linaro.org>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Laura Abbott <labbott@redhat.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Andy Gross <andy.gross@linaro.org>
Cc: Kevin Brodsky <kevin.brodsky@arm.com>
Cc: Andrew Pinski <apinski@cavium.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-kernel at vger.kernel.org
Cc: linux-arm-kernel at lists.infradead.org
Cc: Jeremy Linton <Jeremy.Linton@arm.com>
---
arch/arm/kernel/vdso.c | 1 +
arch/arm/vdso/vdso.lds.S | 1 +
arch/arm64/kernel/vdso/compiler.h | 1 +
arch/arm64/kernel/vdso/vdso.lds.S | 1 +
lib/vdso/vgettimeofday.c | 10 ++++++++++
5 files changed, 14 insertions(+)
diff --git a/arch/arm/kernel/vdso.c b/arch/arm/kernel/vdso.c
index 51d8dcbd9952..6721854c5ae6 100644
--- a/arch/arm/kernel/vdso.c
+++ b/arch/arm/kernel/vdso.c
@@ -192,6 +192,7 @@ static void __init patch_vdso(void *ehdr)
vdso_nullpatch_one(&einfo, "__vdso_gettimeofday");
vdso_nullpatch_one(&einfo, "__vdso_clock_gettime");
vdso_nullpatch_one(&einfo, "__vdso_clock_getres");
+ /* do not zero out __vdso_time, no cntvct_ok dependency */
}
}
diff --git a/arch/arm/vdso/vdso.lds.S b/arch/arm/vdso/vdso.lds.S
index 1d81e8c3acf6..1eb577091d1f 100644
--- a/arch/arm/vdso/vdso.lds.S
+++ b/arch/arm/vdso/vdso.lds.S
@@ -83,6 +83,7 @@ VERSION
__vdso_clock_gettime;
__vdso_gettimeofday;
__vdso_clock_getres;
+ __vdso_time;
local: *;
};
}
diff --git a/arch/arm64/kernel/vdso/compiler.h b/arch/arm64/kernel/vdso/compiler.h
index 921a7191b497..fb27545640f2 100644
--- a/arch/arm64/kernel/vdso/compiler.h
+++ b/arch/arm64/kernel/vdso/compiler.h
@@ -65,5 +65,6 @@ static __always_inline notrace u64 arch_vdso_read_counter(void)
#define __vdso_clock_gettime __kernel_clock_gettime
#define __vdso_gettimeofday __kernel_gettimeofday
#define __vdso_clock_getres __kernel_clock_getres
+#define __vdso_time __kernel_time
#endif /* __VDSO_COMPILER_H */
diff --git a/arch/arm64/kernel/vdso/vdso.lds.S b/arch/arm64/kernel/vdso/vdso.lds.S
index beca249bc2f3..9de0ffc369c5 100644
--- a/arch/arm64/kernel/vdso/vdso.lds.S
+++ b/arch/arm64/kernel/vdso/vdso.lds.S
@@ -88,6 +88,7 @@ VERSION
__kernel_gettimeofday;
__kernel_clock_gettime;
__kernel_clock_getres;
+ __kernel_time;
local: *;
};
}
diff --git a/lib/vdso/vgettimeofday.c b/lib/vdso/vgettimeofday.c
index 54e519c99c4b..dfced9608cd3 100644
--- a/lib/vdso/vgettimeofday.c
+++ b/lib/vdso/vgettimeofday.c
@@ -386,3 +386,13 @@ int __vdso_clock_getres(clockid_t clock, struct timespec *res)
return 0;
}
+
+notrace time_t __vdso_time(time_t *t)
+{
+ const struct vdso_data *vd = __get_datapage();
+ time_t result = READ_ONCE(vd->xtime_coarse_sec);
+
+ if (t)
+ *t = result;
+ return result;
+}
--
2.18.0.rc1.244.gcf134e6275-goog
^ permalink raw reply related
* RESEND [PATCH v5 12/12] lib: vdso: do not expose gettimeofday, if no arch supported timer
From: Mark Salyzyn @ 2018-06-18 15:05 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20180618150613.10322-1-salyzyn@android.com>
Take an effort to recode the arm64 vdso code from assembler to C
previously submitted by Andrew Pinski <apinski@cavium.com>, rework
it for use in both arm and arm64, overlapping any optimizations
for each architecture. But instead of landing it in arm64, land the
result into lib/vdso and unify both implementations to simplify
future maintenance.
If ARCH_PROVIDES_TIMER is not defined, do not expose gettimeofday.
libc will default directly to syscall. Also ifdef clock_gettime
switch cases and stubs if not supported and other unused components.
Signed-off-by: Mark Salyzyn <salyzyn@android.com>
Cc: James Morse <james.morse@arm.com>
Cc: Russell King <linux@armlinux.org.uk>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dmitry Safonov <dsafonov@virtuozzo.com>
Cc: John Stultz <john.stultz@linaro.org>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Laura Abbott <labbott@redhat.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Andy Gross <andy.gross@linaro.org>
Cc: Kevin Brodsky <kevin.brodsky@arm.com>
Cc: Andrew Pinski <apinski@cavium.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-kernel at vger.kernel.org
Cc: linux-arm-kernel at lists.infradead.org
Cc: Jeremy Linton <Jeremy.Linton@arm.com>
v3:
- do not expose gettimeofday if arch does not support user space timer.
v4:
- update commit message to reflect overall reasoning of patch series.
v5:
- rebase
---
lib/vdso/vgettimeofday.c | 52 ++++++++++++++++------------------------
1 file changed, 20 insertions(+), 32 deletions(-)
diff --git a/lib/vdso/vgettimeofday.c b/lib/vdso/vgettimeofday.c
index 4c3af7bc6499..54e519c99c4b 100644
--- a/lib/vdso/vgettimeofday.c
+++ b/lib/vdso/vgettimeofday.c
@@ -30,7 +30,9 @@
#include "compiler.h"
#include "datapage.h"
+#ifdef ARCH_PROVIDES_TIMER
DEFINE_FALLBACK(gettimeofday, struct timeval *, tv, struct timezone *, tz)
+#endif
DEFINE_FALLBACK(clock_gettime, clockid_t, clock, struct timespec *, ts)
DEFINE_FALLBACK(clock_getres, clockid_t, clock, struct timespec *, ts)
@@ -292,30 +294,6 @@ static notrace int do_boottime(const struct vdso_data *vd, struct timespec *ts)
return 0;
}
-#else /* ARCH_PROVIDES_TIMER */
-
-static notrace int do_realtime(const struct vdso_data *vd, struct timespec *ts)
-{
- return -1;
-}
-
-static notrace int do_monotonic(const struct vdso_data *vd, struct timespec *ts)
-{
- return -1;
-}
-
-static notrace int do_monotonic_raw(const struct vdso_data *vd,
- struct timespec *ts)
-{
- return -1;
-}
-
-static notrace int do_boottime(const struct vdso_data *vd,
- struct timespec *ts)
-{
- return -1;
-}
-
#endif /* ARCH_PROVIDES_TIMER */
notrace int __vdso_clock_gettime(clockid_t clock, struct timespec *ts)
@@ -329,6 +307,7 @@ notrace int __vdso_clock_gettime(clockid_t clock, struct timespec *ts)
case CLOCK_MONOTONIC_COARSE:
do_monotonic_coarse(vd, ts);
break;
+#ifdef ARCH_PROVIDES_TIMER
case CLOCK_REALTIME:
if (do_realtime(vd, ts))
goto fallback;
@@ -345,6 +324,7 @@ notrace int __vdso_clock_gettime(clockid_t clock, struct timespec *ts)
if (do_boottime(vd, ts))
goto fallback;
break;
+#endif
default:
goto fallback;
}
@@ -354,6 +334,7 @@ notrace int __vdso_clock_gettime(clockid_t clock, struct timespec *ts)
return clock_gettime_fallback(clock, ts);
}
+#ifdef ARCH_PROVIDES_TIMER
notrace int __vdso_gettimeofday(struct timeval *tv, struct timezone *tz)
{
const struct vdso_data *vd = __get_datapage();
@@ -375,21 +356,28 @@ notrace int __vdso_gettimeofday(struct timeval *tv, struct timezone *tz)
return 0;
}
+#endif
int __vdso_clock_getres(clockid_t clock, struct timespec *res)
{
long nsec;
- if (clock == CLOCK_REALTIME ||
- clock == CLOCK_BOOTTIME ||
- clock == CLOCK_MONOTONIC ||
- clock == CLOCK_MONOTONIC_RAW)
- nsec = MONOTONIC_RES_NSEC;
- else if (clock == CLOCK_REALTIME_COARSE ||
- clock == CLOCK_MONOTONIC_COARSE)
+ switch (clock) {
+ case CLOCK_REALTIME_COARSE:
+ case CLOCK_MONOTONIC_COARSE:
nsec = LOW_RES_NSEC;
- else
+ break;
+#ifdef ARCH_PROVIDES_TIMER
+ case CLOCK_REALTIME:
+ case CLOCK_MONOTONIC:
+ case CLOCK_MONOTONIC_RAW:
+ case CLOCK_BOOTTIME:
+ nsec = MONOTONIC_RES_NSEC;
+ break;
+#endif
+ default:
return clock_getres_fallback(clock, res);
+ }
if (likely(res != NULL)) {
res->tv_sec = 0;
--
2.18.0.rc1.244.gcf134e6275-goog
^ permalink raw reply related
* RESEND [PATCH v6 11/12] lib: vdso: Add support for CLOCK_BOOTTIME
From: Mark Salyzyn @ 2018-06-18 15:05 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20180618150613.10322-1-salyzyn@android.com>
Take an effort to recode the arm64 vdso code from assembler to C
previously submitted by Andrew Pinski <apinski@cavium.com>, rework
it for use in both arm and arm64, overlapping any optimizations
for each architecture. But instead of landing it in arm64, land the
result into lib/vdso and unify both implementations to simplify
future maintenance.
Add a case for CLOCK_BOOTTIME as it is popular for measuring
relative time on systems expected to suspend() or hibernate().
Android uses CLOCK_BOOTTIME for all relative time measurements
and timeouts. Switching to vdso reduced CPU utilization and improves
accuracy. There is also a desire by some partners to switch all
logging over to CLOCK_BOOTTIME, and thus this operation alone would
contribute to a near percentile CPU load.
Signed-off-by: Mark Salyzyn <salyzyn@android.com>
Cc: James Morse <james.morse@arm.com>
Cc: Russell King <linux@armlinux.org.uk>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dmitry Safonov <dsafonov@virtuozzo.com>
Cc: John Stultz <john.stultz@linaro.org>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Laura Abbott <labbott@redhat.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Andy Gross <andy.gross@linaro.org>
Cc: Kevin Brodsky <kevin.brodsky@arm.com>
Cc: Andrew Pinski <apinski@cavium.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-kernel at vger.kernel.org
Cc: linux-arm-kernel at lists.infradead.org
Cc: Jeremy Linton <Jeremy.Linton@arm.com>
v2:
- rebased and changed from 3/3 to 10/10, fortified commit message.
v3:
- move arch/arm/vdso/vgettimeofday.c to lib/vdso/vgettimeofday.c.
v4:
- update commit message to reflect specific, and overall reasoning
of patch series.
- drop forced inline operations.
- switch typeof() with __kernel_time_t.
v5:
- added comment about open coded timepsec_add_ns() for clarity.
v6:
- fix issue with __iter_div_u64_rem scaling by splitting sec & nsec
---
arch/arm/include/asm/vdso_datapage.h | 2 +
arch/arm/kernel/vdso.c | 4 ++
arch/arm64/include/asm/vdso_datapage.h | 2 +
arch/arm64/kernel/vdso.c | 4 ++
lib/vdso/vgettimeofday.c | 56 ++++++++++++++++++++++++++
5 files changed, 68 insertions(+)
diff --git a/arch/arm/include/asm/vdso_datapage.h b/arch/arm/include/asm/vdso_datapage.h
index 1c6e6a5d5d9d..0120852b6b12 100644
--- a/arch/arm/include/asm/vdso_datapage.h
+++ b/arch/arm/include/asm/vdso_datapage.h
@@ -64,6 +64,8 @@ struct vdso_data {
u32 tz_minuteswest; /* timezone info for gettimeofday(2) */
u32 tz_dsttime;
+ u32 btm_sec; /* monotonic to boot time */
+ u32 btm_nsec;
/* Raw clocksource multipler */
u32 cs_raw_mult;
/* Raw time */
diff --git a/arch/arm/kernel/vdso.c b/arch/arm/kernel/vdso.c
index c299967df63c..51d8dcbd9952 100644
--- a/arch/arm/kernel/vdso.c
+++ b/arch/arm/kernel/vdso.c
@@ -337,6 +337,8 @@ void update_vsyscall(struct timekeeper *tk)
vdso_data->wtm_clock_nsec = wtm->tv_nsec;
if (!vdso_data->use_syscall) {
+ struct timespec btm = ktime_to_timespec(tk->offs_boot);
+
vdso_data->cs_cycle_last = tk->tkr_mono.cycle_last;
vdso_data->raw_time_sec = tk->raw_sec;
vdso_data->raw_time_nsec = tk->tkr_raw.xtime_nsec;
@@ -347,6 +349,8 @@ void update_vsyscall(struct timekeeper *tk)
/* tkr_mono.shift == tkr_raw.shift */
vdso_data->cs_shift = tk->tkr_mono.shift;
vdso_data->cs_mask = tk->tkr_mono.mask;
+ vdso_data->btm_sec = btm.tv_sec;
+ vdso_data->btm_nsec = btm.tv_nsec;
}
vdso_write_end(vdso_data);
diff --git a/arch/arm64/include/asm/vdso_datapage.h b/arch/arm64/include/asm/vdso_datapage.h
index 95f4a7abab80..348b9be9efe7 100644
--- a/arch/arm64/include/asm/vdso_datapage.h
+++ b/arch/arm64/include/asm/vdso_datapage.h
@@ -45,6 +45,8 @@ struct vdso_data {
__u64 xtime_coarse_nsec;
__u64 wtm_clock_sec; /* Wall to monotonic time */
vdso_wtm_clock_nsec_t wtm_clock_nsec;
+ __u32 btm_sec; /* monotonic to boot time */
+ __u32 btm_nsec;
__u32 tb_seq_count; /* Timebase sequence counter */
/* cs_* members must be adjacent and in this order (ldp accesses) */
__u32 cs_mono_mult; /* NTP-adjusted clocksource multiplier */
diff --git a/arch/arm64/kernel/vdso.c b/arch/arm64/kernel/vdso.c
index 59f150c25889..8dd2ad220a0f 100644
--- a/arch/arm64/kernel/vdso.c
+++ b/arch/arm64/kernel/vdso.c
@@ -233,6 +233,8 @@ void update_vsyscall(struct timekeeper *tk)
vdso_data->wtm_clock_nsec = tk->wall_to_monotonic.tv_nsec;
if (!use_syscall) {
+ struct timespec btm = ktime_to_timespec(tk->offs_boot);
+
/* tkr_mono.cycle_last == tkr_raw.cycle_last */
vdso_data->cs_cycle_last = tk->tkr_mono.cycle_last;
vdso_data->raw_time_sec = tk->raw_sec;
@@ -243,6 +245,8 @@ void update_vsyscall(struct timekeeper *tk)
vdso_data->cs_raw_mult = tk->tkr_raw.mult;
/* tkr_mono.shift == tkr_raw.shift */
vdso_data->cs_shift = tk->tkr_mono.shift;
+ vdso_data->btm_sec = btm.tv_sec;
+ vdso_data->btm_nsec = btm.tv_nsec;
}
smp_wmb();
diff --git a/lib/vdso/vgettimeofday.c b/lib/vdso/vgettimeofday.c
index 33c5917fe9f8..4c3af7bc6499 100644
--- a/lib/vdso/vgettimeofday.c
+++ b/lib/vdso/vgettimeofday.c
@@ -247,6 +247,51 @@ static notrace int do_monotonic_raw(const struct vdso_data *vd,
return 0;
}
+static notrace int do_boottime(const struct vdso_data *vd, struct timespec *ts)
+{
+ u32 seq, mult, shift;
+ u64 nsec, cycle_last;
+ vdso_wtm_clock_nsec_t wtm_nsec;
+#ifdef ARCH_CLOCK_FIXED_MASK
+ static const u64 mask = ARCH_CLOCK_FIXED_MASK;
+#else
+ u64 mask;
+#endif
+ __kernel_time_t sec;
+
+ do {
+ seq = vdso_read_begin(vd);
+
+ if (vd->use_syscall)
+ return -1;
+
+ cycle_last = vd->cs_cycle_last;
+
+ mult = vd->cs_mono_mult;
+ shift = vd->cs_shift;
+#ifndef ARCH_CLOCK_FIXED_MASK
+ mask = vd->cs_mask;
+#endif
+
+ sec = vd->xtime_clock_sec;
+ nsec = vd->xtime_clock_snsec;
+
+ sec += vd->wtm_clock_sec + vd->btm_sec;
+ wtm_nsec = vd->wtm_clock_nsec + vd->btm_nsec;
+
+ } while (unlikely(vdso_read_retry(vd, seq)));
+
+ nsec += get_clock_shifted_nsec(cycle_last, mult, mask);
+ nsec >>= shift;
+ nsec += wtm_nsec;
+
+ /* open coding timespec_add_ns to save a ts->tv_nsec = 0 */
+ ts->tv_sec = sec + __iter_div_u64_rem(nsec, NSEC_PER_SEC, &nsec);
+ ts->tv_nsec = nsec;
+
+ return 0;
+}
+
#else /* ARCH_PROVIDES_TIMER */
static notrace int do_realtime(const struct vdso_data *vd, struct timespec *ts)
@@ -265,6 +310,12 @@ static notrace int do_monotonic_raw(const struct vdso_data *vd,
return -1;
}
+static notrace int do_boottime(const struct vdso_data *vd,
+ struct timespec *ts)
+{
+ return -1;
+}
+
#endif /* ARCH_PROVIDES_TIMER */
notrace int __vdso_clock_gettime(clockid_t clock, struct timespec *ts)
@@ -290,6 +341,10 @@ notrace int __vdso_clock_gettime(clockid_t clock, struct timespec *ts)
if (do_monotonic_raw(vd, ts))
goto fallback;
break;
+ case CLOCK_BOOTTIME:
+ if (do_boottime(vd, ts))
+ goto fallback;
+ break;
default:
goto fallback;
}
@@ -326,6 +381,7 @@ int __vdso_clock_getres(clockid_t clock, struct timespec *res)
long nsec;
if (clock == CLOCK_REALTIME ||
+ clock == CLOCK_BOOTTIME ||
clock == CLOCK_MONOTONIC ||
clock == CLOCK_MONOTONIC_RAW)
nsec = MONOTONIC_RES_NSEC;
--
2.18.0.rc1.244.gcf134e6275-goog
^ permalink raw reply related
* RESEND [PATCH v5 10/12] arm64: vdso: replace gettimeofday.S with global vgettimeofday.C
From: Mark Salyzyn @ 2018-06-18 15:05 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20180618150613.10322-1-salyzyn@android.com>
Take an effort from the previous 9 patches to recode the arm64 vdso
code from assembler to C previously submitted by
Andrew Pinski <apinski@cavium.com>, rework it for use in both arm and
arm64, overlapping any optimizations for each architecture. But
instead of landing it in arm64, land the result into lib/vdso and
unify both implementations to simplify future maintenance.
apinski at cavium.com makes the following claims in the original patch:
This allows the compiler to optimize the divide by 1000 and remove
the other divides.
On ThunderX, gettimeofday improves by 32%. On ThunderX 2,
gettimeofday improves by 18%.
Note I noticed a bug in the old (arm64) implementation of
__kernel_clock_getres; it was checking only the lower 32bits of the
pointer; this would work for most cases but could fail in a few.
<end of claim>
Signed-off-by: Mark Salyzyn <salyzyn@android.com>
Cc: James Morse <james.morse@arm.com>
Cc: Russell King <linux@armlinux.org.uk>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dmitry Safonov <dsafonov@virtuozzo.com>
Cc: John Stultz <john.stultz@linaro.org>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Laura Abbott <labbott@redhat.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Andy Gross <andy.gross@linaro.org>
Cc: Kevin Brodsky <kevin.brodsky@arm.com>
Cc: Andrew Pinski <apinski@cavium.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-kernel at vger.kernel.org
Cc: linux-arm-kernel at lists.infradead.org
Cc: Jeremy Linton <Jeremy.Linton@arm.com>
v2:
- turned off profiling, restored quiet_cmd_vdsoas.
v3:
- move arch/arm/vdso/vgettimeofday.c to lib/vdso/vgettimeofday.c.
- adjust vgettimeofday.c to be a better global candidate, switch to
using ARCH_PROVIDES_TIMER and __arch_counter_get() as more generic.
v4:
- simplify arch_vdso_read_counter, use read_sysreg.
- Use GENMASK_ULL macro for ARCH_CLOCK_FIXED_MASK.
- update commit message.
- add vdso_wtm_clock_nsec_t, vdso_xtime_clock_sec_t and
vdso_raw_time_nsec_t.
v5:
- comment about asm/processor.h and asm/sysreg.h regarding why
they have been added in compiler.h for porting clarity.
- add linux/hrtimer.h to compiler.h and comment for porting clarity.
- strip out excess unused definitions in asm-offsets.c.
---
arch/arm64/include/asm/vdso_datapage.h | 23 +-
arch/arm64/kernel/asm-offsets.c | 36 ---
arch/arm64/kernel/vdso.c | 2 +-
arch/arm64/kernel/vdso/Makefile | 29 ++-
arch/arm64/kernel/vdso/compiler.h | 69 ++++++
arch/arm64/kernel/vdso/datapage.h | 59 +++++
arch/arm64/kernel/vdso/gettimeofday.S | 328 -------------------------
arch/arm64/kernel/vdso/vgettimeofday.c | 3 +
8 files changed, 175 insertions(+), 374 deletions(-)
create mode 100644 arch/arm64/kernel/vdso/compiler.h
create mode 100644 arch/arm64/kernel/vdso/datapage.h
delete mode 100644 arch/arm64/kernel/vdso/gettimeofday.S
create mode 100644 arch/arm64/kernel/vdso/vgettimeofday.c
diff --git a/arch/arm64/include/asm/vdso_datapage.h b/arch/arm64/include/asm/vdso_datapage.h
index 2b9a63771eda..95f4a7abab80 100644
--- a/arch/arm64/include/asm/vdso_datapage.h
+++ b/arch/arm64/include/asm/vdso_datapage.h
@@ -20,16 +20,31 @@
#ifndef __ASSEMBLY__
+#ifndef _VDSO_WTM_CLOCK_SEC_T
+#define _VDSO_WTM_CLOCK_SEC_T
+typedef __u64 vdso_wtm_clock_nsec_t;
+#endif
+
+#ifndef _VDSO_XTIME_CLOCK_SEC_T
+#define _VDSO_XTIME_CLOCK_SEC_T
+typedef __u64 vdso_xtime_clock_sec_t;
+#endif
+
+#ifndef _VDSO_RAW_TIME_SEC_T
+#define _VDSO_RAW_TIME_SEC_T
+typedef __u64 vdso_raw_time_sec_t;
+#endif
+
struct vdso_data {
__u64 cs_cycle_last; /* Timebase at clocksource init */
- __u64 raw_time_sec; /* Raw time */
+ vdso_raw_time_sec_t raw_time_sec; /* Raw time */
__u64 raw_time_nsec;
- __u64 xtime_clock_sec; /* Kernel time */
- __u64 xtime_clock_nsec;
+ vdso_xtime_clock_sec_t xtime_clock_sec; /* Kernel time */
+ __u64 xtime_clock_snsec;
__u64 xtime_coarse_sec; /* Coarse time */
__u64 xtime_coarse_nsec;
__u64 wtm_clock_sec; /* Wall to monotonic time */
- __u64 wtm_clock_nsec;
+ vdso_wtm_clock_nsec_t wtm_clock_nsec;
__u32 tb_seq_count; /* Timebase sequence counter */
/* cs_* members must be adjacent and in this order (ldp accesses) */
__u32 cs_mono_mult; /* NTP-adjusted clocksource multiplier */
diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c
index 323aeb5f2fe6..8938a4223690 100644
--- a/arch/arm64/kernel/asm-offsets.c
+++ b/arch/arm64/kernel/asm-offsets.c
@@ -94,42 +94,6 @@ int main(void)
DEFINE(DMA_TO_DEVICE, DMA_TO_DEVICE);
DEFINE(DMA_FROM_DEVICE, DMA_FROM_DEVICE);
BLANK();
- DEFINE(PREEMPT_DISABLE_OFFSET, PREEMPT_DISABLE_OFFSET);
- BLANK();
- DEFINE(CLOCK_REALTIME, CLOCK_REALTIME);
- DEFINE(CLOCK_MONOTONIC, CLOCK_MONOTONIC);
- DEFINE(CLOCK_MONOTONIC_RAW, CLOCK_MONOTONIC_RAW);
- DEFINE(CLOCK_REALTIME_RES, MONOTONIC_RES_NSEC);
- DEFINE(CLOCK_REALTIME_COARSE, CLOCK_REALTIME_COARSE);
- DEFINE(CLOCK_MONOTONIC_COARSE,CLOCK_MONOTONIC_COARSE);
- DEFINE(CLOCK_COARSE_RES, LOW_RES_NSEC);
- DEFINE(NSEC_PER_SEC, NSEC_PER_SEC);
- BLANK();
- DEFINE(VDSO_CS_CYCLE_LAST, offsetof(struct vdso_data, cs_cycle_last));
- DEFINE(VDSO_RAW_TIME_SEC, offsetof(struct vdso_data, raw_time_sec));
- DEFINE(VDSO_RAW_TIME_NSEC, offsetof(struct vdso_data, raw_time_nsec));
- DEFINE(VDSO_XTIME_CLK_SEC, offsetof(struct vdso_data, xtime_clock_sec));
- DEFINE(VDSO_XTIME_CLK_NSEC, offsetof(struct vdso_data, xtime_clock_nsec));
- DEFINE(VDSO_XTIME_CRS_SEC, offsetof(struct vdso_data, xtime_coarse_sec));
- DEFINE(VDSO_XTIME_CRS_NSEC, offsetof(struct vdso_data, xtime_coarse_nsec));
- DEFINE(VDSO_WTM_CLK_SEC, offsetof(struct vdso_data, wtm_clock_sec));
- DEFINE(VDSO_WTM_CLK_NSEC, offsetof(struct vdso_data, wtm_clock_nsec));
- DEFINE(VDSO_TB_SEQ_COUNT, offsetof(struct vdso_data, tb_seq_count));
- DEFINE(VDSO_CS_MONO_MULT, offsetof(struct vdso_data, cs_mono_mult));
- DEFINE(VDSO_CS_RAW_MULT, offsetof(struct vdso_data, cs_raw_mult));
- DEFINE(VDSO_CS_SHIFT, offsetof(struct vdso_data, cs_shift));
- DEFINE(VDSO_TZ_MINWEST, offsetof(struct vdso_data, tz_minuteswest));
- DEFINE(VDSO_TZ_DSTTIME, offsetof(struct vdso_data, tz_dsttime));
- DEFINE(VDSO_USE_SYSCALL, offsetof(struct vdso_data, use_syscall));
- BLANK();
- DEFINE(TVAL_TV_SEC, offsetof(struct timeval, tv_sec));
- DEFINE(TVAL_TV_USEC, offsetof(struct timeval, tv_usec));
- DEFINE(TSPEC_TV_SEC, offsetof(struct timespec, tv_sec));
- DEFINE(TSPEC_TV_NSEC, offsetof(struct timespec, tv_nsec));
- BLANK();
- DEFINE(TZ_MINWEST, offsetof(struct timezone, tz_minuteswest));
- DEFINE(TZ_DSTTIME, offsetof(struct timezone, tz_dsttime));
- BLANK();
DEFINE(CPU_BOOT_STACK, offsetof(struct secondary_data, stack));
DEFINE(CPU_BOOT_TASK, offsetof(struct secondary_data, task));
BLANK();
diff --git a/arch/arm64/kernel/vdso.c b/arch/arm64/kernel/vdso.c
index 2d419006ad43..59f150c25889 100644
--- a/arch/arm64/kernel/vdso.c
+++ b/arch/arm64/kernel/vdso.c
@@ -238,7 +238,7 @@ void update_vsyscall(struct timekeeper *tk)
vdso_data->raw_time_sec = tk->raw_sec;
vdso_data->raw_time_nsec = tk->tkr_raw.xtime_nsec;
vdso_data->xtime_clock_sec = tk->xtime_sec;
- vdso_data->xtime_clock_nsec = tk->tkr_mono.xtime_nsec;
+ vdso_data->xtime_clock_snsec = tk->tkr_mono.xtime_nsec;
vdso_data->cs_mono_mult = tk->tkr_mono.mult;
vdso_data->cs_raw_mult = tk->tkr_raw.mult;
/* tkr_mono.shift == tkr_raw.shift */
diff --git a/arch/arm64/kernel/vdso/Makefile b/arch/arm64/kernel/vdso/Makefile
index b215c712d897..21cad81a4f40 100644
--- a/arch/arm64/kernel/vdso/Makefile
+++ b/arch/arm64/kernel/vdso/Makefile
@@ -6,18 +6,32 @@
# Heavily based on the vDSO Makefiles for other archs.
#
-obj-vdso := gettimeofday.o note.o sigreturn.o
+obj-vdso-s := note.o sigreturn.o
+obj-vdso-c := vgettimeofday.o
# Build rules
-targets := $(obj-vdso) vdso.so vdso.so.dbg
-obj-vdso := $(addprefix $(obj)/, $(obj-vdso))
+targets := $(obj-vdso-s) $(obj-vdso-c) vdso.so vdso.so.dbg
+obj-vdso-s := $(addprefix $(obj)/, $(obj-vdso-s))
+obj-vdso-c := $(addprefix $(obj)/, $(obj-vdso-c))
+obj-vdso := $(obj-vdso-c) $(obj-vdso-s)
-ccflags-y := -shared -fno-common -fno-builtin
+ccflags-y := -shared -fno-common -fno-builtin -fno-stack-protector
+ccflags-y += -DDISABLE_BRANCH_PROFILING
ccflags-y += -nostdlib -Wl,-soname=linux-vdso.so.1 \
$(call cc-ldoption, -Wl$(comma)--hash-style=sysv)
+# Force -O2 to avoid libgcc dependencies
+CFLAGS_REMOVE_vgettimeofday.o = -pg -Os
+CFLAGS_vgettimeofday.o = -O2 -fPIC
+ifneq ($(cc-name),clang)
+CFLAGS_vgettimeofday.o += -mcmodel=tiny
+endif
+
# Disable gcov profiling for VDSO code
GCOV_PROFILE := n
+KASAN_SANITIZE := n
+UBSAN_SANITIZE := n
+KCOV_INSTRUMENT := n
# Workaround for bare-metal (ELF) toolchains that neglect to pass -shared
# down to collect2, resulting in silent corruption of the vDSO image.
@@ -50,12 +64,17 @@ include/generated/vdso-offsets.h: $(obj)/vdso.so.dbg FORCE
$(call if_changed,vdsosym)
# Assembly rules for the .S files
-$(obj-vdso): %.o: %.S FORCE
+$(obj-vdso-s): %.o: %.S FORCE
$(call if_changed_dep,vdsoas)
+$(obj-vdso-c): %.o: %.c FORCE
+ $(call if_changed_dep,vdsocc)
+
# Actual build commands
quiet_cmd_vdsold = VDSOL $@
cmd_vdsold = $(CC) $(c_flags) -Wl,-n -Wl,-T $^ -o $@
+quiet_cmd_vdsocc = VDSOC $@
+ cmd_vdsocc = ${CC} $(c_flags) -c -o $@ $<
quiet_cmd_vdsoas = VDSOA $@
cmd_vdsoas = $(CC) $(a_flags) -c -o $@ $<
diff --git a/arch/arm64/kernel/vdso/compiler.h b/arch/arm64/kernel/vdso/compiler.h
new file mode 100644
index 000000000000..921a7191b497
--- /dev/null
+++ b/arch/arm64/kernel/vdso/compiler.h
@@ -0,0 +1,69 @@
+/*
+ * Userspace implementations of fallback calls
+ *
+ * Copyright (C) 2017 Cavium, Inc.
+ * Copyright (C) 2012 ARM Limited
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ * Author: Will Deacon <will.deacon@arm.com>
+ * Rewriten into C by: Andrew Pinski <apinski@cavium.com>
+ */
+
+#ifndef __VDSO_COMPILER_H
+#define __VDSO_COMPILER_H
+
+#include <asm/processor.h> /* for cpu_relax() */
+#include <asm/sysreg.h> /* for read_sysreg() */
+#include <asm/unistd.h>
+#include <linux/compiler.h>
+#include <linux/hrtimer.h> /* for LOW_RES_NSEC and MONOTONIC_RES_NSEC */
+
+#ifdef CONFIG_ARM_ARCH_TIMER
+#define ARCH_PROVIDES_TIMER
+#endif
+
+#define DEFINE_FALLBACK(name, type_arg1, name_arg1, type_arg2, name_arg2) \
+static notrace long name##_fallback(type_arg1 _##name_arg1, \
+ type_arg2 _##name_arg2) \
+{ \
+ register type_arg1 name_arg1 asm("x0") = _##name_arg1; \
+ register type_arg2 name_arg2 asm("x1") = _##name_arg2; \
+ register long ret asm ("x0"); \
+ register long nr asm("x8") = __NR_##name; \
+ \
+ asm volatile( \
+ " svc #0\n" \
+ : "=r" (ret) \
+ : "r" (name_arg1), "r" (name_arg2), "r" (nr) \
+ : "memory"); \
+ \
+ return ret; \
+}
+
+/*
+ * AArch64 implementation of arch_counter_get_cntvct() suitable for vdso
+ */
+static __always_inline notrace u64 arch_vdso_read_counter(void)
+{
+ /* Read the virtual counter. */
+ isb();
+ return read_sysreg(cntvct_el0);
+}
+
+/* Rename exported vdso functions */
+#define __vdso_clock_gettime __kernel_clock_gettime
+#define __vdso_gettimeofday __kernel_gettimeofday
+#define __vdso_clock_getres __kernel_clock_getres
+
+#endif /* __VDSO_COMPILER_H */
diff --git a/arch/arm64/kernel/vdso/datapage.h b/arch/arm64/kernel/vdso/datapage.h
new file mode 100644
index 000000000000..be86a6074cf8
--- /dev/null
+++ b/arch/arm64/kernel/vdso/datapage.h
@@ -0,0 +1,59 @@
+/*
+ * Userspace implementations of __get_datapage
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef __VDSO_DATAPAGE_H
+#define __VDSO_DATAPAGE_H
+
+#include <linux/bitops.h>
+#include <linux/types.h>
+#include <asm/vdso_datapage.h>
+
+/*
+ * We use the hidden visibility to prevent the compiler from generating a GOT
+ * relocation. Not only is going through a GOT useless (the entry couldn't and
+ * mustn't be overridden by another library), it does not even work: the linker
+ * cannot generate an absolute address to the data page.
+ *
+ * With the hidden visibility, the compiler simply generates a PC-relative
+ * relocation (R_ARM_REL32), and this is what we need.
+ */
+extern const struct vdso_data _vdso_data __attribute__((visibility("hidden")));
+
+static inline const struct vdso_data *__get_datapage(void)
+{
+ const struct vdso_data *ret;
+ /*
+ * This simply puts &_vdso_data into ret. The reason why we don't use
+ * `ret = &_vdso_data` is that the compiler tends to optimise this in a
+ * very suboptimal way: instead of keeping &_vdso_data in a register,
+ * it goes through a relocation almost every time _vdso_data must be
+ * accessed (even in subfunctions). This is both time and space
+ * consuming: each relocation uses a word in the code section, and it
+ * has to be loaded at runtime.
+ *
+ * This trick hides the assignment from the compiler. Since it cannot
+ * track where the pointer comes from, it will only use one relocation
+ * where __get_datapage() is called, and then keep the result in a
+ * register.
+ */
+ asm("" : "=r"(ret) : "0"(&_vdso_data));
+ return ret;
+}
+
+/* We can only guarantee 56 bits of precision. */
+#define ARCH_CLOCK_FIXED_MASK GENMASK_ULL(55, 0)
+
+#endif /* __VDSO_DATAPAGE_H */
diff --git a/arch/arm64/kernel/vdso/gettimeofday.S b/arch/arm64/kernel/vdso/gettimeofday.S
deleted file mode 100644
index c39872a7b03c..000000000000
--- a/arch/arm64/kernel/vdso/gettimeofday.S
+++ /dev/null
@@ -1,328 +0,0 @@
-/*
- * Userspace implementations of gettimeofday() and friends.
- *
- * Copyright (C) 2012 ARM Limited
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- *
- * Author: Will Deacon <will.deacon@arm.com>
- */
-
-#include <linux/linkage.h>
-#include <asm/asm-offsets.h>
-#include <asm/unistd.h>
-
-#define NSEC_PER_SEC_LO16 0xca00
-#define NSEC_PER_SEC_HI16 0x3b9a
-
-vdso_data .req x6
-seqcnt .req w7
-w_tmp .req w8
-x_tmp .req x8
-
-/*
- * Conventions for macro arguments:
- * - An argument is write-only if its name starts with "res".
- * - All other arguments are read-only, unless otherwise specified.
- */
-
- .macro seqcnt_acquire
-9999: ldr seqcnt, [vdso_data, #VDSO_TB_SEQ_COUNT]
- tbnz seqcnt, #0, 9999b
- dmb ishld
- .endm
-
- .macro seqcnt_check fail
- dmb ishld
- ldr w_tmp, [vdso_data, #VDSO_TB_SEQ_COUNT]
- cmp w_tmp, seqcnt
- b.ne \fail
- .endm
-
- .macro syscall_check fail
- ldr w_tmp, [vdso_data, #VDSO_USE_SYSCALL]
- cbnz w_tmp, \fail
- .endm
-
- .macro get_nsec_per_sec res
- mov \res, #NSEC_PER_SEC_LO16
- movk \res, #NSEC_PER_SEC_HI16, lsl #16
- .endm
-
- /*
- * Returns the clock delta, in nanoseconds left-shifted by the clock
- * shift.
- */
- .macro get_clock_shifted_nsec res, cycle_last, mult
- /* Read the virtual counter. */
- isb
- mrs x_tmp, cntvct_el0
- /* Calculate cycle delta and convert to ns. */
- sub \res, x_tmp, \cycle_last
- /* We can only guarantee 56 bits of precision. */
- movn x_tmp, #0xff00, lsl #48
- and \res, x_tmp, \res
- mul \res, \res, \mult
- .endm
-
- /*
- * Returns in res_{sec,nsec} the REALTIME timespec, based on the
- * "wall time" (xtime) and the clock_mono delta.
- */
- .macro get_ts_realtime res_sec, res_nsec, \
- clock_nsec, xtime_sec, xtime_nsec, nsec_to_sec
- add \res_nsec, \clock_nsec, \xtime_nsec
- udiv x_tmp, \res_nsec, \nsec_to_sec
- add \res_sec, \xtime_sec, x_tmp
- msub \res_nsec, x_tmp, \nsec_to_sec, \res_nsec
- .endm
-
- /*
- * Returns in res_{sec,nsec} the timespec based on the clock_raw delta,
- * used for CLOCK_MONOTONIC_RAW.
- */
- .macro get_ts_clock_raw res_sec, res_nsec, clock_nsec, nsec_to_sec
- udiv \res_sec, \clock_nsec, \nsec_to_sec
- msub \res_nsec, \res_sec, \nsec_to_sec, \clock_nsec
- .endm
-
- /* sec and nsec are modified in place. */
- .macro add_ts sec, nsec, ts_sec, ts_nsec, nsec_to_sec
- /* Add timespec. */
- add \sec, \sec, \ts_sec
- add \nsec, \nsec, \ts_nsec
-
- /* Normalise the new timespec. */
- cmp \nsec, \nsec_to_sec
- b.lt 9999f
- sub \nsec, \nsec, \nsec_to_sec
- add \sec, \sec, #1
-9999:
- cmp \nsec, #0
- b.ge 9998f
- add \nsec, \nsec, \nsec_to_sec
- sub \sec, \sec, #1
-9998:
- .endm
-
- .macro clock_gettime_return, shift=0
- .if \shift == 1
- lsr x11, x11, x12
- .endif
- stp x10, x11, [x1, #TSPEC_TV_SEC]
- mov x0, xzr
- ret
- .endm
-
- .macro jump_slot jumptable, index, label
- .if (. - \jumptable) != 4 * (\index)
- .error "Jump slot index mismatch"
- .endif
- b \label
- .endm
-
- .text
-
-/* int __kernel_gettimeofday(struct timeval *tv, struct timezone *tz); */
-ENTRY(__kernel_gettimeofday)
- .cfi_startproc
- adr vdso_data, _vdso_data
- /* If tv is NULL, skip to the timezone code. */
- cbz x0, 2f
-
- /* Compute the time of day. */
-1: seqcnt_acquire
- syscall_check fail=4f
- ldr x10, [vdso_data, #VDSO_CS_CYCLE_LAST]
- /* w11 = cs_mono_mult, w12 = cs_shift */
- ldp w11, w12, [vdso_data, #VDSO_CS_MONO_MULT]
- ldp x13, x14, [vdso_data, #VDSO_XTIME_CLK_SEC]
- seqcnt_check fail=1b
-
- get_nsec_per_sec res=x9
- lsl x9, x9, x12
-
- get_clock_shifted_nsec res=x15, cycle_last=x10, mult=x11
- get_ts_realtime res_sec=x10, res_nsec=x11, \
- clock_nsec=x15, xtime_sec=x13, xtime_nsec=x14, nsec_to_sec=x9
-
- /* Convert ns to us. */
- mov x13, #1000
- lsl x13, x13, x12
- udiv x11, x11, x13
- stp x10, x11, [x0, #TVAL_TV_SEC]
-2:
- /* If tz is NULL, return 0. */
- cbz x1, 3f
- ldp w4, w5, [vdso_data, #VDSO_TZ_MINWEST]
- stp w4, w5, [x1, #TZ_MINWEST]
-3:
- mov x0, xzr
- ret
-4:
- /* Syscall fallback. */
- mov x8, #__NR_gettimeofday
- svc #0
- ret
- .cfi_endproc
-ENDPROC(__kernel_gettimeofday)
-
-#define JUMPSLOT_MAX CLOCK_MONOTONIC_COARSE
-
-/* int __kernel_clock_gettime(clockid_t clock_id, struct timespec *tp); */
-ENTRY(__kernel_clock_gettime)
- .cfi_startproc
- cmp w0, #JUMPSLOT_MAX
- b.hi syscall
- adr vdso_data, _vdso_data
- adr x_tmp, jumptable
- add x_tmp, x_tmp, w0, uxtw #2
- br x_tmp
-
- ALIGN
-jumptable:
- jump_slot jumptable, CLOCK_REALTIME, realtime
- jump_slot jumptable, CLOCK_MONOTONIC, monotonic
- b syscall
- b syscall
- jump_slot jumptable, CLOCK_MONOTONIC_RAW, monotonic_raw
- jump_slot jumptable, CLOCK_REALTIME_COARSE, realtime_coarse
- jump_slot jumptable, CLOCK_MONOTONIC_COARSE, monotonic_coarse
-
- .if (. - jumptable) != 4 * (JUMPSLOT_MAX + 1)
- .error "Wrong jumptable size"
- .endif
-
- ALIGN
-realtime:
- seqcnt_acquire
- syscall_check fail=syscall
- ldr x10, [vdso_data, #VDSO_CS_CYCLE_LAST]
- /* w11 = cs_mono_mult, w12 = cs_shift */
- ldp w11, w12, [vdso_data, #VDSO_CS_MONO_MULT]
- ldp x13, x14, [vdso_data, #VDSO_XTIME_CLK_SEC]
- seqcnt_check fail=realtime
-
- /* All computations are done with left-shifted nsecs. */
- get_nsec_per_sec res=x9
- lsl x9, x9, x12
-
- get_clock_shifted_nsec res=x15, cycle_last=x10, mult=x11
- get_ts_realtime res_sec=x10, res_nsec=x11, \
- clock_nsec=x15, xtime_sec=x13, xtime_nsec=x14, nsec_to_sec=x9
- clock_gettime_return, shift=1
-
- ALIGN
-monotonic:
- seqcnt_acquire
- syscall_check fail=syscall
- ldr x10, [vdso_data, #VDSO_CS_CYCLE_LAST]
- /* w11 = cs_mono_mult, w12 = cs_shift */
- ldp w11, w12, [vdso_data, #VDSO_CS_MONO_MULT]
- ldp x13, x14, [vdso_data, #VDSO_XTIME_CLK_SEC]
- ldp x3, x4, [vdso_data, #VDSO_WTM_CLK_SEC]
- seqcnt_check fail=monotonic
-
- /* All computations are done with left-shifted nsecs. */
- lsl x4, x4, x12
- get_nsec_per_sec res=x9
- lsl x9, x9, x12
-
- get_clock_shifted_nsec res=x15, cycle_last=x10, mult=x11
- get_ts_realtime res_sec=x10, res_nsec=x11, \
- clock_nsec=x15, xtime_sec=x13, xtime_nsec=x14, nsec_to_sec=x9
-
- add_ts sec=x10, nsec=x11, ts_sec=x3, ts_nsec=x4, nsec_to_sec=x9
- clock_gettime_return, shift=1
-
- ALIGN
-monotonic_raw:
- seqcnt_acquire
- syscall_check fail=syscall
- ldr x10, [vdso_data, #VDSO_CS_CYCLE_LAST]
- /* w11 = cs_raw_mult, w12 = cs_shift */
- ldp w12, w11, [vdso_data, #VDSO_CS_SHIFT]
- ldp x13, x14, [vdso_data, #VDSO_RAW_TIME_SEC]
- seqcnt_check fail=monotonic_raw
-
- /* All computations are done with left-shifted nsecs. */
- get_nsec_per_sec res=x9
- lsl x9, x9, x12
-
- get_clock_shifted_nsec res=x15, cycle_last=x10, mult=x11
- get_ts_clock_raw res_sec=x10, res_nsec=x11, \
- clock_nsec=x15, nsec_to_sec=x9
-
- add_ts sec=x10, nsec=x11, ts_sec=x13, ts_nsec=x14, nsec_to_sec=x9
- clock_gettime_return, shift=1
-
- ALIGN
-realtime_coarse:
- seqcnt_acquire
- ldp x10, x11, [vdso_data, #VDSO_XTIME_CRS_SEC]
- seqcnt_check fail=realtime_coarse
- clock_gettime_return
-
- ALIGN
-monotonic_coarse:
- seqcnt_acquire
- ldp x10, x11, [vdso_data, #VDSO_XTIME_CRS_SEC]
- ldp x13, x14, [vdso_data, #VDSO_WTM_CLK_SEC]
- seqcnt_check fail=monotonic_coarse
-
- /* Computations are done in (non-shifted) nsecs. */
- get_nsec_per_sec res=x9
- add_ts sec=x10, nsec=x11, ts_sec=x13, ts_nsec=x14, nsec_to_sec=x9
- clock_gettime_return
-
- ALIGN
-syscall: /* Syscall fallback. */
- mov x8, #__NR_clock_gettime
- svc #0
- ret
- .cfi_endproc
-ENDPROC(__kernel_clock_gettime)
-
-/* int __kernel_clock_getres(clockid_t clock_id, struct timespec *res); */
-ENTRY(__kernel_clock_getres)
- .cfi_startproc
- cmp w0, #CLOCK_REALTIME
- ccmp w0, #CLOCK_MONOTONIC, #0x4, ne
- ccmp w0, #CLOCK_MONOTONIC_RAW, #0x4, ne
- b.ne 1f
-
- ldr x2, 5f
- b 2f
-1:
- cmp w0, #CLOCK_REALTIME_COARSE
- ccmp w0, #CLOCK_MONOTONIC_COARSE, #0x4, ne
- b.ne 4f
- ldr x2, 6f
-2:
- cbz x1, 3f
- stp xzr, x2, [x1]
-
-3: /* res == NULL. */
- mov w0, wzr
- ret
-
-4: /* Syscall fallback. */
- mov x8, #__NR_clock_getres
- svc #0
- ret
-5:
- .quad CLOCK_REALTIME_RES
-6:
- .quad CLOCK_COARSE_RES
- .cfi_endproc
-ENDPROC(__kernel_clock_getres)
diff --git a/arch/arm64/kernel/vdso/vgettimeofday.c b/arch/arm64/kernel/vdso/vgettimeofday.c
new file mode 100644
index 000000000000..b73d4011993d
--- /dev/null
+++ b/arch/arm64/kernel/vdso/vgettimeofday.c
@@ -0,0 +1,3 @@
+#include "compiler.h"
+#include "datapage.h"
+#include "../../../../lib/vdso/vgettimeofday.c"
--
2.18.0.rc1.244.gcf134e6275-goog
^ permalink raw reply related
* RESEND [PATCH v5 09/12] arm: vdso: move vgettimeofday.c to lib/vdso/
From: Mark Salyzyn @ 2018-06-18 15:05 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20180618150613.10322-1-salyzyn@android.com>
Take an effort to recode the arm64 vdso code from assembler to C
previously submitted by Andrew Pinski <apinski@cavium.com>, rework
it for use in both arm and arm64, overlapping any optimizations
for each architecture. But instead of landing it in arm64, land the
result into lib/vdso and unify both implementations to simplify
future maintenance.
Declare arch/arm/vdso/vgettimeofday.c to be a candidate for a global
implementation of the vdso timer calls. The hope is that new
architectures can take advantage of the current unification of
arm and arm64 implementations.
We urge future efforts to merge their implementations into the
global vgettimeofday.c file and thus provide functional parity.
Signed-off-by: Mark Salyzyn <salyzyn@android.com>
Cc: James Morse <james.morse@arm.com>
Cc: Russell King <linux@armlinux.org.uk>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dmitry Safonov <dsafonov@virtuozzo.com>
Cc: John Stultz <john.stultz@linaro.org>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Laura Abbott <labbott@redhat.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Andy Gross <andy.gross@linaro.org>
Cc: Kevin Brodsky <kevin.brodsky@arm.com>
Cc: Andrew Pinski <apinski@cavium.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-kernel at vger.kernel.org
Cc: linux-arm-kernel at lists.infradead.org
Cc: Jeremy Linton <Jeremy.Linton@arm.com>
v3:
- added this change
- move arch/arm/vdso/vgettimeofday.c to lib/vdso/vgettimeofday.c
- adjust vgettimeofday.c to be a better global candidate, switch to
using ARCH_PROVIDES_TIMER and __arch_counter_get() as more generic.
v4:
- update commit message to reflect overall reasoning
- adjust to reflect dropping of any forced inline
v5:
- rebase
---
arch/arm/vdso/compiler.h | 4 +
arch/arm/vdso/vgettimeofday.c | 343 +--------------------------------
lib/vdso/compiler.h | 24 +++
lib/vdso/datapage.h | 24 +++
lib/vdso/vgettimeofday.c | 344 ++++++++++++++++++++++++++++++++++
5 files changed, 397 insertions(+), 342 deletions(-)
create mode 100644 lib/vdso/compiler.h
create mode 100644 lib/vdso/datapage.h
create mode 100644 lib/vdso/vgettimeofday.c
diff --git a/arch/arm/vdso/compiler.h b/arch/arm/vdso/compiler.h
index c7751019246a..6fd88be2ff0e 100644
--- a/arch/arm/vdso/compiler.h
+++ b/arch/arm/vdso/compiler.h
@@ -34,6 +34,10 @@
#error This code depends on AEABI system call conventions
#endif
+#ifdef CONFIG_ARM_ARCH_TIMER
+#define ARCH_PROVIDES_TIMER
+#endif
+
#define DEFINE_FALLBACK(name, type_arg1, name_arg1, type_arg2, name_arg2) \
static notrace long name##_fallback(type_arg1 _##name_arg1, \
type_arg2 _##name_arg2) \
diff --git a/arch/arm/vdso/vgettimeofday.c b/arch/arm/vdso/vgettimeofday.c
index 3005479efbe8..4b241fe60d17 100644
--- a/arch/arm/vdso/vgettimeofday.c
+++ b/arch/arm/vdso/vgettimeofday.c
@@ -1,344 +1,3 @@
-/*
- * Userspace implementations of gettimeofday() and friends.
- *
- * Copyright (C) 2017 Cavium, Inc.
- * Copyright (C) 2015 Mentor Graphics Corporation
- * Copyright (C) 2012 ARM Limited
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- *
- * Author: Will Deacon <will.deacon@arm.com>
- * Rewriten from arch64 version into C by: Andrew Pinski <apinski@cavium.com>
- * Reworked and rebased over arm version by: Mark Salyzyn <salyzyn@android.com>
- */
-
-#include <asm/barrier.h>
-#include <linux/compiler.h> /* for notrace */
-#include <linux/math64.h> /* for __iter_div_u64_rem() */
-#include <uapi/linux/time.h> /* for struct timespec */
-
#include "compiler.h"
#include "datapage.h"
-
-DEFINE_FALLBACK(gettimeofday, struct timeval *, tv, struct timezone *, tz)
-DEFINE_FALLBACK(clock_gettime, clockid_t, clock, struct timespec *, ts)
-DEFINE_FALLBACK(clock_getres, clockid_t, clock, struct timespec *, ts)
-
-static notrace u32 vdso_read_begin(const struct vdso_data *vd)
-{
- u32 seq;
-
- do {
- seq = READ_ONCE(vd->tb_seq_count);
-
- if ((seq & 1) == 0)
- break;
-
- cpu_relax();
- } while (true);
-
- smp_rmb(); /* Pairs with second smp_wmb in update_vsyscall */
- return seq;
-}
-
-static notrace int vdso_read_retry(const struct vdso_data *vd, u32 start)
-{
- u32 seq;
-
- smp_rmb(); /* Pairs with first smp_wmb in update_vsyscall */
- seq = READ_ONCE(vd->tb_seq_count);
- return seq != start;
-}
-
-static notrace int do_realtime_coarse(const struct vdso_data *vd,
- struct timespec *ts)
-{
- u32 seq;
-
- do {
- seq = vdso_read_begin(vd);
-
- ts->tv_sec = vd->xtime_coarse_sec;
- ts->tv_nsec = vd->xtime_coarse_nsec;
-
- } while (vdso_read_retry(vd, seq));
-
- return 0;
-}
-
-static notrace int do_monotonic_coarse(const struct vdso_data *vd,
- struct timespec *ts)
-{
- struct timespec tomono;
- u32 seq;
- u64 nsec;
-
- do {
- seq = vdso_read_begin(vd);
-
- ts->tv_sec = vd->xtime_coarse_sec;
- ts->tv_nsec = vd->xtime_coarse_nsec;
-
- tomono.tv_sec = vd->wtm_clock_sec;
- tomono.tv_nsec = vd->wtm_clock_nsec;
-
- } while (vdso_read_retry(vd, seq));
-
- ts->tv_sec += tomono.tv_sec;
- /* open coding timespec_add_ns */
- ts->tv_sec += __iter_div_u64_rem(ts->tv_nsec + tomono.tv_nsec,
- NSEC_PER_SEC, &nsec);
- ts->tv_nsec = nsec;
-
- return 0;
-}
-
-#ifdef CONFIG_ARM_ARCH_TIMER
-
-/*
- * Returns the clock delta, in nanoseconds left-shifted by the clock
- * shift.
- */
-static notrace u64 get_clock_shifted_nsec(const u64 cycle_last,
- const u32 mult,
- const u64 mask)
-{
- u64 res;
-
- /* Read the virtual counter. */
- res = arch_vdso_read_counter();
-
- res = res - cycle_last;
-
- res &= mask;
- return res * mult;
-}
-
-static notrace int do_realtime(const struct vdso_data *vd, struct timespec *ts)
-{
- u32 seq, mult, shift;
- u64 nsec, cycle_last;
-#ifdef ARCH_CLOCK_FIXED_MASK
- static const u64 mask = ARCH_CLOCK_FIXED_MASK;
-#else
- u64 mask;
-#endif
- vdso_xtime_clock_sec_t sec;
-
- do {
- seq = vdso_read_begin(vd);
-
- if (vd->use_syscall)
- return -1;
-
- cycle_last = vd->cs_cycle_last;
-
- mult = vd->cs_mono_mult;
- shift = vd->cs_shift;
-#ifndef ARCH_CLOCK_FIXED_MASK
- mask = vd->cs_mask;
-#endif
-
- sec = vd->xtime_clock_sec;
- nsec = vd->xtime_clock_snsec;
-
- } while (unlikely(vdso_read_retry(vd, seq)));
-
- nsec += get_clock_shifted_nsec(cycle_last, mult, mask);
- nsec >>= shift;
- /* open coding timespec_add_ns to save a ts->tv_nsec = 0 */
- ts->tv_sec = sec + __iter_div_u64_rem(nsec, NSEC_PER_SEC, &nsec);
- ts->tv_nsec = nsec;
-
- return 0;
-}
-
-static notrace int do_monotonic(const struct vdso_data *vd, struct timespec *ts)
-{
- u32 seq, mult, shift;
- u64 nsec, cycle_last;
-#ifdef ARCH_CLOCK_FIXED_MASK
- static const u64 mask = ARCH_CLOCK_FIXED_MASK;
-#else
- u64 mask;
-#endif
- vdso_wtm_clock_nsec_t wtm_nsec;
- __kernel_time_t sec;
-
- do {
- seq = vdso_read_begin(vd);
-
- if (vd->use_syscall)
- return -1;
-
- cycle_last = vd->cs_cycle_last;
-
- mult = vd->cs_mono_mult;
- shift = vd->cs_shift;
-#ifndef ARCH_CLOCK_FIXED_MASK
- mask = vd->cs_mask;
-#endif
-
- sec = vd->xtime_clock_sec;
- nsec = vd->xtime_clock_snsec;
-
- sec += vd->wtm_clock_sec;
- wtm_nsec = vd->wtm_clock_nsec;
-
- } while (unlikely(vdso_read_retry(vd, seq)));
-
- nsec += get_clock_shifted_nsec(cycle_last, mult, mask);
- nsec >>= shift;
- nsec += wtm_nsec;
- /* open coding timespec_add_ns to save a ts->tv_nsec = 0 */
- ts->tv_sec = sec + __iter_div_u64_rem(nsec, NSEC_PER_SEC, &nsec);
- ts->tv_nsec = nsec;
-
- return 0;
-}
-
-static notrace int do_monotonic_raw(const struct vdso_data *vd,
- struct timespec *ts)
-{
- u32 seq, mult, shift;
- u64 nsec, cycle_last;
-#ifdef ARCH_CLOCK_FIXED_MASK
- static const u64 mask = ARCH_CLOCK_FIXED_MASK;
-#else
- u64 mask;
-#endif
- vdso_raw_time_sec_t sec;
-
- do {
- seq = vdso_read_begin(vd);
-
- if (vd->use_syscall)
- return -1;
-
- cycle_last = vd->cs_cycle_last;
-
- mult = vd->cs_raw_mult;
- shift = vd->cs_shift;
-#ifndef ARCH_CLOCK_FIXED_MASK
- mask = vd->cs_mask;
-#endif
-
- sec = vd->raw_time_sec;
- nsec = vd->raw_time_nsec;
-
- } while (unlikely(vdso_read_retry(vd, seq)));
-
- nsec += get_clock_shifted_nsec(cycle_last, mult, mask);
- nsec >>= shift;
- /* open coding timespec_add_ns to save a ts->tv_nsec = 0 */
- ts->tv_sec = sec + __iter_div_u64_rem(nsec, NSEC_PER_SEC, &nsec);
- ts->tv_nsec = nsec;
-
- return 0;
-}
-
-#else /* CONFIG_ARM_ARCH_TIMER */
-
-static notrace int do_realtime(const struct vdso_data *vd, struct timespec *ts)
-{
- return -1;
-}
-
-static notrace int do_monotonic(const struct vdso_data *vd, struct timespec *ts)
-{
- return -1;
-}
-
-static notrace int do_monotonic_raw(const struct vdso_data *vd,
- struct timespec *ts)
-{
- return -1;
-}
-
-#endif /* CONFIG_ARM_ARCH_TIMER */
-
-notrace int __vdso_clock_gettime(clockid_t clock, struct timespec *ts)
-{
- const struct vdso_data *vd = __get_datapage();
-
- switch (clock) {
- case CLOCK_REALTIME_COARSE:
- do_realtime_coarse(vd, ts);
- break;
- case CLOCK_MONOTONIC_COARSE:
- do_monotonic_coarse(vd, ts);
- break;
- case CLOCK_REALTIME:
- if (do_realtime(vd, ts))
- goto fallback;
- break;
- case CLOCK_MONOTONIC:
- if (do_monotonic(vd, ts))
- goto fallback;
- break;
- case CLOCK_MONOTONIC_RAW:
- if (do_monotonic_raw(vd, ts))
- goto fallback;
- break;
- default:
- goto fallback;
- }
-
- return 0;
-fallback:
- return clock_gettime_fallback(clock, ts);
-}
-
-notrace int __vdso_gettimeofday(struct timeval *tv, struct timezone *tz)
-{
- const struct vdso_data *vd = __get_datapage();
-
- if (likely(tv != NULL)) {
- struct timespec ts;
-
- if (do_realtime(vd, &ts))
- return gettimeofday_fallback(tv, tz);
-
- tv->tv_sec = ts.tv_sec;
- tv->tv_usec = ts.tv_nsec / 1000;
- }
-
- if (unlikely(tz != NULL)) {
- tz->tz_minuteswest = vd->tz_minuteswest;
- tz->tz_dsttime = vd->tz_dsttime;
- }
-
- return 0;
-}
-
-int __vdso_clock_getres(clockid_t clock, struct timespec *res)
-{
- long nsec;
-
- if (clock == CLOCK_REALTIME ||
- clock == CLOCK_MONOTONIC ||
- clock == CLOCK_MONOTONIC_RAW)
- nsec = MONOTONIC_RES_NSEC;
- else if (clock == CLOCK_REALTIME_COARSE ||
- clock == CLOCK_MONOTONIC_COARSE)
- nsec = LOW_RES_NSEC;
- else
- return clock_getres_fallback(clock, res);
-
- if (likely(res != NULL)) {
- res->tv_sec = 0;
- res->tv_nsec = nsec;
- }
-
- return 0;
-}
+#include "../../../lib/vdso/vgettimeofday.c"
diff --git a/lib/vdso/compiler.h b/lib/vdso/compiler.h
new file mode 100644
index 000000000000..0e618b73e064
--- /dev/null
+++ b/lib/vdso/compiler.h
@@ -0,0 +1,24 @@
+/*
+ * Userspace implementations of fallback calls
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef __VDSO_COMPILER_H
+#define __VDSO_COMPILER_H
+
+#error "vdso: Provide architectural overrides such as ARCH_PROVIDES_TIMER,"
+#error " DEFINE_FALLBACK and __arch_counter_get or any overrides. eg:"
+#error " vdso entry points or compilation time helpers."
+
+#endif /* __VDSO_COMPILER_H */
diff --git a/lib/vdso/datapage.h b/lib/vdso/datapage.h
new file mode 100644
index 000000000000..df4427e42d51
--- /dev/null
+++ b/lib/vdso/datapage.h
@@ -0,0 +1,24 @@
+/*
+ * Userspace implementations of __get_datapage
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef __VDSO_DATAPAGE_H
+#define __VDSO_DATAPAGE_H
+
+#error "vdso: Provide a user space architecture specific definition or"
+#error "prototype for struct vdso_data *__get_datapage(void). Also define"
+#error "ARCH_CLOCK_FIXED_MASK if not provided by cs_mask."
+
+#endif /* __VDSO_DATAPAGE_H */
diff --git a/lib/vdso/vgettimeofday.c b/lib/vdso/vgettimeofday.c
new file mode 100644
index 000000000000..33c5917fe9f8
--- /dev/null
+++ b/lib/vdso/vgettimeofday.c
@@ -0,0 +1,344 @@
+/*
+ * Userspace implementations of gettimeofday() and friends.
+ *
+ * Copyright (C) 2017 Cavium, Inc.
+ * Copyright (C) 2015 Mentor Graphics Corporation
+ * Copyright (C) 2012 ARM Limited
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ * Author: Will Deacon <will.deacon@arm.com>
+ * Rewriten from arch64 version into C by: Andrew Pinski <apinski@cavium.com>
+ * Reworked and rebased over arm version by: Mark Salyzyn <salyzyn@android.com>
+ */
+
+#include <asm/barrier.h>
+#include <linux/compiler.h> /* for notrace */
+#include <linux/math64.h> /* for __iter_div_u64_rem() */
+#include <uapi/linux/time.h> /* for struct timespec */
+
+#include "compiler.h"
+#include "datapage.h"
+
+DEFINE_FALLBACK(gettimeofday, struct timeval *, tv, struct timezone *, tz)
+DEFINE_FALLBACK(clock_gettime, clockid_t, clock, struct timespec *, ts)
+DEFINE_FALLBACK(clock_getres, clockid_t, clock, struct timespec *, ts)
+
+static notrace u32 vdso_read_begin(const struct vdso_data *vd)
+{
+ u32 seq;
+
+ do {
+ seq = READ_ONCE(vd->tb_seq_count);
+
+ if ((seq & 1) == 0)
+ break;
+
+ cpu_relax();
+ } while (true);
+
+ smp_rmb(); /* Pairs with second smp_wmb in update_vsyscall */
+ return seq;
+}
+
+static notrace int vdso_read_retry(const struct vdso_data *vd, u32 start)
+{
+ u32 seq;
+
+ smp_rmb(); /* Pairs with first smp_wmb in update_vsyscall */
+ seq = READ_ONCE(vd->tb_seq_count);
+ return seq != start;
+}
+
+static notrace int do_realtime_coarse(const struct vdso_data *vd,
+ struct timespec *ts)
+{
+ u32 seq;
+
+ do {
+ seq = vdso_read_begin(vd);
+
+ ts->tv_sec = vd->xtime_coarse_sec;
+ ts->tv_nsec = vd->xtime_coarse_nsec;
+
+ } while (vdso_read_retry(vd, seq));
+
+ return 0;
+}
+
+static notrace int do_monotonic_coarse(const struct vdso_data *vd,
+ struct timespec *ts)
+{
+ struct timespec tomono;
+ u32 seq;
+ u64 nsec;
+
+ do {
+ seq = vdso_read_begin(vd);
+
+ ts->tv_sec = vd->xtime_coarse_sec;
+ ts->tv_nsec = vd->xtime_coarse_nsec;
+
+ tomono.tv_sec = vd->wtm_clock_sec;
+ tomono.tv_nsec = vd->wtm_clock_nsec;
+
+ } while (vdso_read_retry(vd, seq));
+
+ ts->tv_sec += tomono.tv_sec;
+ /* open coding timespec_add_ns */
+ ts->tv_sec += __iter_div_u64_rem(ts->tv_nsec + tomono.tv_nsec,
+ NSEC_PER_SEC, &nsec);
+ ts->tv_nsec = nsec;
+
+ return 0;
+}
+
+#ifdef ARCH_PROVIDES_TIMER
+
+/*
+ * Returns the clock delta, in nanoseconds left-shifted by the clock
+ * shift.
+ */
+static notrace u64 get_clock_shifted_nsec(const u64 cycle_last,
+ const u32 mult,
+ const u64 mask)
+{
+ u64 res;
+
+ /* Read the virtual counter. */
+ res = arch_vdso_read_counter();
+
+ res = res - cycle_last;
+
+ res &= mask;
+ return res * mult;
+}
+
+static notrace int do_realtime(const struct vdso_data *vd, struct timespec *ts)
+{
+ u32 seq, mult, shift;
+ u64 nsec, cycle_last;
+#ifdef ARCH_CLOCK_FIXED_MASK
+ static const u64 mask = ARCH_CLOCK_FIXED_MASK;
+#else
+ u64 mask;
+#endif
+ vdso_xtime_clock_sec_t sec;
+
+ do {
+ seq = vdso_read_begin(vd);
+
+ if (vd->use_syscall)
+ return -1;
+
+ cycle_last = vd->cs_cycle_last;
+
+ mult = vd->cs_mono_mult;
+ shift = vd->cs_shift;
+#ifndef ARCH_CLOCK_FIXED_MASK
+ mask = vd->cs_mask;
+#endif
+
+ sec = vd->xtime_clock_sec;
+ nsec = vd->xtime_clock_snsec;
+
+ } while (unlikely(vdso_read_retry(vd, seq)));
+
+ nsec += get_clock_shifted_nsec(cycle_last, mult, mask);
+ nsec >>= shift;
+ /* open coding timespec_add_ns to save a ts->tv_nsec = 0 */
+ ts->tv_sec = sec + __iter_div_u64_rem(nsec, NSEC_PER_SEC, &nsec);
+ ts->tv_nsec = nsec;
+
+ return 0;
+}
+
+static notrace int do_monotonic(const struct vdso_data *vd, struct timespec *ts)
+{
+ u32 seq, mult, shift;
+ u64 nsec, cycle_last;
+#ifdef ARCH_CLOCK_FIXED_MASK
+ static const u64 mask = ARCH_CLOCK_FIXED_MASK;
+#else
+ u64 mask;
+#endif
+ vdso_wtm_clock_nsec_t wtm_nsec;
+ __kernel_time_t sec;
+
+ do {
+ seq = vdso_read_begin(vd);
+
+ if (vd->use_syscall)
+ return -1;
+
+ cycle_last = vd->cs_cycle_last;
+
+ mult = vd->cs_mono_mult;
+ shift = vd->cs_shift;
+#ifndef ARCH_CLOCK_FIXED_MASK
+ mask = vd->cs_mask;
+#endif
+
+ sec = vd->xtime_clock_sec;
+ nsec = vd->xtime_clock_snsec;
+
+ sec += vd->wtm_clock_sec;
+ wtm_nsec = vd->wtm_clock_nsec;
+
+ } while (unlikely(vdso_read_retry(vd, seq)));
+
+ nsec += get_clock_shifted_nsec(cycle_last, mult, mask);
+ nsec >>= shift;
+ nsec += wtm_nsec;
+ /* open coding timespec_add_ns to save a ts->tv_nsec = 0 */
+ ts->tv_sec = sec + __iter_div_u64_rem(nsec, NSEC_PER_SEC, &nsec);
+ ts->tv_nsec = nsec;
+
+ return 0;
+}
+
+static notrace int do_monotonic_raw(const struct vdso_data *vd,
+ struct timespec *ts)
+{
+ u32 seq, mult, shift;
+ u64 nsec, cycle_last;
+#ifdef ARCH_CLOCK_FIXED_MASK
+ static const u64 mask = ARCH_CLOCK_FIXED_MASK;
+#else
+ u64 mask;
+#endif
+ vdso_raw_time_sec_t sec;
+
+ do {
+ seq = vdso_read_begin(vd);
+
+ if (vd->use_syscall)
+ return -1;
+
+ cycle_last = vd->cs_cycle_last;
+
+ mult = vd->cs_raw_mult;
+ shift = vd->cs_shift;
+#ifndef ARCH_CLOCK_FIXED_MASK
+ mask = vd->cs_mask;
+#endif
+
+ sec = vd->raw_time_sec;
+ nsec = vd->raw_time_nsec;
+
+ } while (unlikely(vdso_read_retry(vd, seq)));
+
+ nsec += get_clock_shifted_nsec(cycle_last, mult, mask);
+ nsec >>= shift;
+ /* open coding timespec_add_ns to save a ts->tv_nsec = 0 */
+ ts->tv_sec = sec + __iter_div_u64_rem(nsec, NSEC_PER_SEC, &nsec);
+ ts->tv_nsec = nsec;
+
+ return 0;
+}
+
+#else /* ARCH_PROVIDES_TIMER */
+
+static notrace int do_realtime(const struct vdso_data *vd, struct timespec *ts)
+{
+ return -1;
+}
+
+static notrace int do_monotonic(const struct vdso_data *vd, struct timespec *ts)
+{
+ return -1;
+}
+
+static notrace int do_monotonic_raw(const struct vdso_data *vd,
+ struct timespec *ts)
+{
+ return -1;
+}
+
+#endif /* ARCH_PROVIDES_TIMER */
+
+notrace int __vdso_clock_gettime(clockid_t clock, struct timespec *ts)
+{
+ const struct vdso_data *vd = __get_datapage();
+
+ switch (clock) {
+ case CLOCK_REALTIME_COARSE:
+ do_realtime_coarse(vd, ts);
+ break;
+ case CLOCK_MONOTONIC_COARSE:
+ do_monotonic_coarse(vd, ts);
+ break;
+ case CLOCK_REALTIME:
+ if (do_realtime(vd, ts))
+ goto fallback;
+ break;
+ case CLOCK_MONOTONIC:
+ if (do_monotonic(vd, ts))
+ goto fallback;
+ break;
+ case CLOCK_MONOTONIC_RAW:
+ if (do_monotonic_raw(vd, ts))
+ goto fallback;
+ break;
+ default:
+ goto fallback;
+ }
+
+ return 0;
+fallback:
+ return clock_gettime_fallback(clock, ts);
+}
+
+notrace int __vdso_gettimeofday(struct timeval *tv, struct timezone *tz)
+{
+ const struct vdso_data *vd = __get_datapage();
+
+ if (likely(tv != NULL)) {
+ struct timespec ts;
+
+ if (do_realtime(vd, &ts))
+ return gettimeofday_fallback(tv, tz);
+
+ tv->tv_sec = ts.tv_sec;
+ tv->tv_usec = ts.tv_nsec / 1000;
+ }
+
+ if (unlikely(tz != NULL)) {
+ tz->tz_minuteswest = vd->tz_minuteswest;
+ tz->tz_dsttime = vd->tz_dsttime;
+ }
+
+ return 0;
+}
+
+int __vdso_clock_getres(clockid_t clock, struct timespec *res)
+{
+ long nsec;
+
+ if (clock == CLOCK_REALTIME ||
+ clock == CLOCK_MONOTONIC ||
+ clock == CLOCK_MONOTONIC_RAW)
+ nsec = MONOTONIC_RES_NSEC;
+ else if (clock == CLOCK_REALTIME_COARSE ||
+ clock == CLOCK_MONOTONIC_COARSE)
+ nsec = LOW_RES_NSEC;
+ else
+ return clock_getres_fallback(clock, res);
+
+ if (likely(res != NULL)) {
+ res->tv_sec = 0;
+ res->tv_nsec = nsec;
+ }
+
+ return 0;
+}
--
2.18.0.rc1.244.gcf134e6275-goog
^ permalink raw reply related
* RESEND [PATCH v5 08/12] arm: vdso: Add ARCH_CLOCK_FIXED_MASK
From: Mark Salyzyn @ 2018-06-18 15:05 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20180618150613.10322-1-salyzyn@android.com>
Take an effort to recode the arm64 vdso code from assembler to C
previously submitted by Andrew Pinski <apinski@cavium.com>, rework
it for use in both arm and arm64, overlapping any optimizations
for each architecture. But instead of landing it in arm64, land the
result into lib/vdso and unify both implementations to simplify
future maintenance.
Add ARCH_CLOCK_FIXED_MASK as an efficiency since arm64 has no
purpose for cs_mask vdso_data variable.
Signed-off-by: Mark Salyzyn <salyzyn@android.com>
Cc: James Morse <james.morse@arm.com>
Cc: Russell King <linux@armlinux.org.uk>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dmitry Safonov <dsafonov@virtuozzo.com>
Cc: John Stultz <john.stultz@linaro.org>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Laura Abbott <labbott@redhat.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Andy Gross <andy.gross@linaro.org>
Cc: Kevin Brodsky <kevin.brodsky@arm.com>
Cc: Andrew Pinski <apinski@cavium.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-kernel at vger.kernel.org
Cc: linux-arm-kernel at lists.infradead.org
Cc: Jeremy Linton <Jeremy.Linton@arm.com>
v4:
- New in series, split off from earlier.
v5:
- rebase
---
arch/arm/vdso/vgettimeofday.c | 18 ++++++++++++++++++
1 file changed, 18 insertions(+)
diff --git a/arch/arm/vdso/vgettimeofday.c b/arch/arm/vdso/vgettimeofday.c
index a354586f8a65..3005479efbe8 100644
--- a/arch/arm/vdso/vgettimeofday.c
+++ b/arch/arm/vdso/vgettimeofday.c
@@ -128,7 +128,11 @@ static notrace int do_realtime(const struct vdso_data *vd, struct timespec *ts)
{
u32 seq, mult, shift;
u64 nsec, cycle_last;
+#ifdef ARCH_CLOCK_FIXED_MASK
+ static const u64 mask = ARCH_CLOCK_FIXED_MASK;
+#else
u64 mask;
+#endif
vdso_xtime_clock_sec_t sec;
do {
@@ -141,7 +145,9 @@ static notrace int do_realtime(const struct vdso_data *vd, struct timespec *ts)
mult = vd->cs_mono_mult;
shift = vd->cs_shift;
+#ifndef ARCH_CLOCK_FIXED_MASK
mask = vd->cs_mask;
+#endif
sec = vd->xtime_clock_sec;
nsec = vd->xtime_clock_snsec;
@@ -161,7 +167,11 @@ static notrace int do_monotonic(const struct vdso_data *vd, struct timespec *ts)
{
u32 seq, mult, shift;
u64 nsec, cycle_last;
+#ifdef ARCH_CLOCK_FIXED_MASK
+ static const u64 mask = ARCH_CLOCK_FIXED_MASK;
+#else
u64 mask;
+#endif
vdso_wtm_clock_nsec_t wtm_nsec;
__kernel_time_t sec;
@@ -175,7 +185,9 @@ static notrace int do_monotonic(const struct vdso_data *vd, struct timespec *ts)
mult = vd->cs_mono_mult;
shift = vd->cs_shift;
+#ifndef ARCH_CLOCK_FIXED_MASK
mask = vd->cs_mask;
+#endif
sec = vd->xtime_clock_sec;
nsec = vd->xtime_clock_snsec;
@@ -200,7 +212,11 @@ static notrace int do_monotonic_raw(const struct vdso_data *vd,
{
u32 seq, mult, shift;
u64 nsec, cycle_last;
+#ifdef ARCH_CLOCK_FIXED_MASK
+ static const u64 mask = ARCH_CLOCK_FIXED_MASK;
+#else
u64 mask;
+#endif
vdso_raw_time_sec_t sec;
do {
@@ -213,7 +229,9 @@ static notrace int do_monotonic_raw(const struct vdso_data *vd,
mult = vd->cs_raw_mult;
shift = vd->cs_shift;
+#ifndef ARCH_CLOCK_FIXED_MASK
mask = vd->cs_mask;
+#endif
sec = vd->raw_time_sec;
nsec = vd->raw_time_nsec;
--
2.18.0.rc1.244.gcf134e6275-goog
^ permalink raw reply related
* RESEND [PATCH v5 07/12] arm: vdso: disable profiling
From: Mark Salyzyn @ 2018-06-18 15:05 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20180618150613.10322-1-salyzyn@android.com>
Take an effort to recode the arm64 vdso code from assembler to C
previously submitted by Andrew Pinski <apinski@cavium.com>, rework
it for use in both arm and arm64, overlapping any optimizations
for each architecture. But instead of landing it in arm64, land the
result into lib/vdso and unify both implementations to simplify
future maintenance.
Make sure kasan and ubsan profiling, and kcov instrumentation,
is turned off for VDSO code.
Signed-off-by: Mark Salyzyn <salyzyn@android.com>
Cc: James Morse <james.morse@arm.com>
Cc: Russell King <linux@armlinux.org.uk>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dmitry Safonov <dsafonov@virtuozzo.com>
Cc: John Stultz <john.stultz@linaro.org>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Laura Abbott <labbott@redhat.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Andy Gross <andy.gross@linaro.org>
Cc: Kevin Brodsky <kevin.brodsky@arm.com>
Cc: Andrew Pinski <apinski@cavium.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-kernel at vger.kernel.org
Cc: linux-arm-kernel at lists.infradead.org
Cc: Jeremy Linton <Jeremy.Linton@arm.com>
v2:
- Added this to the split up of first CL, based on comments in second CL
v4:
- update commit message to reflect overall reasoning
v5:
- rebase
---
arch/arm/vdso/Makefile | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/arch/arm/vdso/Makefile b/arch/arm/vdso/Makefile
index f4efff9d3afb..1125e3fb8006 100644
--- a/arch/arm/vdso/Makefile
+++ b/arch/arm/vdso/Makefile
@@ -27,8 +27,11 @@ CFLAGS_REMOVE_vdso.o = -pg
CFLAGS_REMOVE_vgettimeofday.o = -pg -Os
CFLAGS_vgettimeofday.o = -O2
-# Disable gcov profiling for VDSO code
+# Disable gcov, kasan, ubsan and kcov profiling for VDSO code
GCOV_PROFILE := n
+KASAN_SANITIZE := n
+UBSAN_SANITIZE := n
+KCOV_INSTRUMENT := n
# Prevents link failures: __sanitizer_cov_trace_pc() is not linked in.
KCOV_INSTRUMENT := n
--
2.18.0.rc1.244.gcf134e6275-goog
^ permalink raw reply related
* RESEND [PATCH v5 06/12] arm: vdso: add support for clock_getres
From: Mark Salyzyn @ 2018-06-18 15:05 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20180618150613.10322-1-salyzyn@android.com>
Take an effort to recode the arm64 vdso code from assembler to C
previously submitted by Andrew Pinski <apinski@cavium.com>, rework
it for use in both arm and arm64, overlapping any optimizations
for each architecture. But instead of landing it in arm64, land the
result into lib/vdso and unify both implementations to simplify
future maintenance.
Add clock_getres vdso support to match up with existing support in
the arm64's vdso.
Signed-off-by: Mark Salyzyn <salyzyn@android.com>
Cc: James Morse <james.morse@arm.com>
Cc: Russell King <linux@armlinux.org.uk>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dmitry Safonov <dsafonov@virtuozzo.com>
Cc: John Stultz <john.stultz@linaro.org>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Laura Abbott <labbott@redhat.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Andy Gross <andy.gross@linaro.org>
Cc: Kevin Brodsky <kevin.brodsky@arm.com>
Cc: Andrew Pinski <apinski@cavium.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-kernel at vger.kernel.org
Cc: linux-arm-kernel at lists.infradead.org
Cc: Jeremy Linton <Jeremy.Linton@arm.com>
v2:
- split first CL into 7 of 7 pieces
v4:
- update commit message to reflect overall reasoning.
- replace typeof() with long for nsec
- replace clock_id with clock to match usage elsewhere.
v5:
- add linux/hrtimer.h to compiler.h to supply for LOW_RES_NSEC and
MONOTONIC_RES_NSEC.
---
arch/arm/kernel/vdso.c | 1 +
arch/arm/vdso/compiler.h | 1 +
arch/arm/vdso/vdso.lds.S | 1 +
arch/arm/vdso/vgettimeofday.c | 23 +++++++++++++++++++++++
4 files changed, 26 insertions(+)
diff --git a/arch/arm/kernel/vdso.c b/arch/arm/kernel/vdso.c
index 9c13a32fa5f0..c299967df63c 100644
--- a/arch/arm/kernel/vdso.c
+++ b/arch/arm/kernel/vdso.c
@@ -191,6 +191,7 @@ static void __init patch_vdso(void *ehdr)
if (!cntvct_ok) {
vdso_nullpatch_one(&einfo, "__vdso_gettimeofday");
vdso_nullpatch_one(&einfo, "__vdso_clock_gettime");
+ vdso_nullpatch_one(&einfo, "__vdso_clock_getres");
}
}
diff --git a/arch/arm/vdso/compiler.h b/arch/arm/vdso/compiler.h
index 3edddb705a1b..c7751019246a 100644
--- a/arch/arm/vdso/compiler.h
+++ b/arch/arm/vdso/compiler.h
@@ -27,6 +27,7 @@
#include <asm/processor.h> /* for cpu_relax() */
#include <asm/unistd.h>
#include <linux/compiler.h>
+#include <linux/hrtimer.h> /* for LOW_RES_NSEC and MONOTONIC_RES_NSEC */
#include <linux/time.h> /* for NSEC_PER_SEC */
#ifndef CONFIG_AEABI
diff --git a/arch/arm/vdso/vdso.lds.S b/arch/arm/vdso/vdso.lds.S
index 89ca89f12d23..1d81e8c3acf6 100644
--- a/arch/arm/vdso/vdso.lds.S
+++ b/arch/arm/vdso/vdso.lds.S
@@ -82,6 +82,7 @@ VERSION
global:
__vdso_clock_gettime;
__vdso_gettimeofday;
+ __vdso_clock_getres;
local: *;
};
}
diff --git a/arch/arm/vdso/vgettimeofday.c b/arch/arm/vdso/vgettimeofday.c
index a2c4db83edc4..a354586f8a65 100644
--- a/arch/arm/vdso/vgettimeofday.c
+++ b/arch/arm/vdso/vgettimeofday.c
@@ -32,6 +32,7 @@
DEFINE_FALLBACK(gettimeofday, struct timeval *, tv, struct timezone *, tz)
DEFINE_FALLBACK(clock_gettime, clockid_t, clock, struct timespec *, ts)
+DEFINE_FALLBACK(clock_getres, clockid_t, clock, struct timespec *, ts)
static notrace u32 vdso_read_begin(const struct vdso_data *vd)
{
@@ -301,3 +302,25 @@ notrace int __vdso_gettimeofday(struct timeval *tv, struct timezone *tz)
return 0;
}
+
+int __vdso_clock_getres(clockid_t clock, struct timespec *res)
+{
+ long nsec;
+
+ if (clock == CLOCK_REALTIME ||
+ clock == CLOCK_MONOTONIC ||
+ clock == CLOCK_MONOTONIC_RAW)
+ nsec = MONOTONIC_RES_NSEC;
+ else if (clock == CLOCK_REALTIME_COARSE ||
+ clock == CLOCK_MONOTONIC_COARSE)
+ nsec = LOW_RES_NSEC;
+ else
+ return clock_getres_fallback(clock, res);
+
+ if (likely(res != NULL)) {
+ res->tv_sec = 0;
+ res->tv_nsec = nsec;
+ }
+
+ return 0;
+}
--
2.18.0.rc1.244.gcf134e6275-goog
^ permalink raw reply related
* RESEND [PATCH v6 05/12] arm: vdso: Add support for CLOCK_MONOTONIC_RAW
From: Mark Salyzyn @ 2018-06-18 15:05 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20180618150613.10322-1-salyzyn@android.com>
Take an effort to recode the arm64 vdso code from assembler to C
previously submitted by Andrew Pinski <apinski@cavium.com>, rework
it for use in both arm and arm64, overlapping any optimizations
for each architecture. But instead of landing it in arm64, land the
result into lib/vdso and unify both implementations to simplify
future maintenance.
Add a case for CLOCK_MONOTONIC_RAW to match up with support that
is available in arm64's vdso.
Signed-off-by: Mark Salyzyn <salyzyn@android.com>
Cc: James Morse <james.morse@arm.com>
Cc: Russell King <linux@armlinux.org.uk>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dmitry Safonov <dsafonov@virtuozzo.com>
Cc: John Stultz <john.stultz@linaro.org>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Laura Abbott <labbott@redhat.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Andy Gross <andy.gross@linaro.org>
Cc: Kevin Brodsky <kevin.brodsky@arm.com>
Cc: Andrew Pinski <apinski@cavium.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-kernel at vger.kernel.org
Cc: linux-arm-kernel at lists.infradead.org
Cc: Jeremy Linton <Jeremy.Linton@arm.com>
v2:
- split first CL into 6 of 7 pieces
v4:
- Move out ARCH_CLOCK_FIXED_MASK to later adjustment.
- update commit message to reflect overall reasoning.
- replace typeof() with type vdso_raw_time_sec_t.
v5:
- replace erroneous tk->raw_time.shift with tk->tkr_raw.shift
v6:
- fixup raw_time_sec and raw_time_nsec in vdso.c
---
arch/arm/include/asm/vdso_datapage.h | 11 +++++++
arch/arm/kernel/vdso.c | 3 ++
arch/arm/vdso/vgettimeofday.c | 44 ++++++++++++++++++++++++++++
3 files changed, 58 insertions(+)
diff --git a/arch/arm/include/asm/vdso_datapage.h b/arch/arm/include/asm/vdso_datapage.h
index 8dd7303db4ec..1c6e6a5d5d9d 100644
--- a/arch/arm/include/asm/vdso_datapage.h
+++ b/arch/arm/include/asm/vdso_datapage.h
@@ -34,6 +34,11 @@ typedef u32 vdso_wtm_clock_nsec_t;
typedef u32 vdso_xtime_clock_sec_t;
#endif
+#ifndef _VDSO_RAW_TIME_SEC_T
+#define _VDSO_RAW_TIME_SEC_T
+typedef u32 vdso_raw_time_sec_t;
+#endif
+
/* Try to be cache-friendly on systems that don't implement the
* generic timer: fit the unconditionally updated fields in the first
* 32 bytes.
@@ -58,6 +63,12 @@ struct vdso_data {
u64 xtime_clock_snsec; /* CLOCK_REALTIME sub-ns base */
u32 tz_minuteswest; /* timezone info for gettimeofday(2) */
u32 tz_dsttime;
+
+ /* Raw clocksource multipler */
+ u32 cs_raw_mult;
+ /* Raw time */
+ vdso_raw_time_sec_t raw_time_sec;
+ u32 raw_time_nsec;
};
union vdso_data_store {
diff --git a/arch/arm/kernel/vdso.c b/arch/arm/kernel/vdso.c
index c2c57f6b8c60..9c13a32fa5f0 100644
--- a/arch/arm/kernel/vdso.c
+++ b/arch/arm/kernel/vdso.c
@@ -337,9 +337,12 @@ void update_vsyscall(struct timekeeper *tk)
if (!vdso_data->use_syscall) {
vdso_data->cs_cycle_last = tk->tkr_mono.cycle_last;
+ vdso_data->raw_time_sec = tk->raw_sec;
+ vdso_data->raw_time_nsec = tk->tkr_raw.xtime_nsec;
vdso_data->xtime_clock_sec = tk->xtime_sec;
vdso_data->xtime_clock_snsec = tk->tkr_mono.xtime_nsec;
vdso_data->cs_mono_mult = tk->tkr_mono.mult;
+ vdso_data->cs_raw_mult = tk->tkr_raw.mult;
/* tkr_mono.shift == tkr_raw.shift */
vdso_data->cs_shift = tk->tkr_mono.shift;
vdso_data->cs_mask = tk->tkr_mono.mask;
diff --git a/arch/arm/vdso/vgettimeofday.c b/arch/arm/vdso/vgettimeofday.c
index 59893fca03b3..a2c4db83edc4 100644
--- a/arch/arm/vdso/vgettimeofday.c
+++ b/arch/arm/vdso/vgettimeofday.c
@@ -194,6 +194,40 @@ static notrace int do_monotonic(const struct vdso_data *vd, struct timespec *ts)
return 0;
}
+static notrace int do_monotonic_raw(const struct vdso_data *vd,
+ struct timespec *ts)
+{
+ u32 seq, mult, shift;
+ u64 nsec, cycle_last;
+ u64 mask;
+ vdso_raw_time_sec_t sec;
+
+ do {
+ seq = vdso_read_begin(vd);
+
+ if (vd->use_syscall)
+ return -1;
+
+ cycle_last = vd->cs_cycle_last;
+
+ mult = vd->cs_raw_mult;
+ shift = vd->cs_shift;
+ mask = vd->cs_mask;
+
+ sec = vd->raw_time_sec;
+ nsec = vd->raw_time_nsec;
+
+ } while (unlikely(vdso_read_retry(vd, seq)));
+
+ nsec += get_clock_shifted_nsec(cycle_last, mult, mask);
+ nsec >>= shift;
+ /* open coding timespec_add_ns to save a ts->tv_nsec = 0 */
+ ts->tv_sec = sec + __iter_div_u64_rem(nsec, NSEC_PER_SEC, &nsec);
+ ts->tv_nsec = nsec;
+
+ return 0;
+}
+
#else /* CONFIG_ARM_ARCH_TIMER */
static notrace int do_realtime(const struct vdso_data *vd, struct timespec *ts)
@@ -206,6 +240,12 @@ static notrace int do_monotonic(const struct vdso_data *vd, struct timespec *ts)
return -1;
}
+static notrace int do_monotonic_raw(const struct vdso_data *vd,
+ struct timespec *ts)
+{
+ return -1;
+}
+
#endif /* CONFIG_ARM_ARCH_TIMER */
notrace int __vdso_clock_gettime(clockid_t clock, struct timespec *ts)
@@ -227,6 +267,10 @@ notrace int __vdso_clock_gettime(clockid_t clock, struct timespec *ts)
if (do_monotonic(vd, ts))
goto fallback;
break;
+ case CLOCK_MONOTONIC_RAW:
+ if (do_monotonic_raw(vd, ts))
+ goto fallback;
+ break;
default:
goto fallback;
}
--
2.18.0.rc1.244.gcf134e6275-goog
^ permalink raw reply related
* RESEND [PATCH v5 04/12] arm: vdso: do calculations outside reader loops
From: Mark Salyzyn @ 2018-06-18 15:05 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20180618150613.10322-1-salyzyn@android.com>
Take an effort to recode the arm64 vdso code from assembler to C
previously submitted by Andrew Pinski <apinski@cavium.com>, rework
it for use in both arm and arm64, overlapping any optimizations
for each architecture. But instead of landing it in arm64, land the
result into lib/vdso and unify both implementations to simplify
future maintenance.
In variable timer reading loops, pick up just the values until all
are synchronized, then outside of loop pick up cntvct and perform
calculations to determine final offset, shifted and multiplied
output value.
This replaces get_ns with get_clock_shifted_nsec as cntvct reader.
Signed-off-by: Mark Salyzyn <salyzyn@android.com>
Cc: James Morse <james.morse@arm.com>
Cc: Russell King <linux@armlinux.org.uk>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dmitry Safonov <dsafonov@virtuozzo.com>
Cc: John Stultz <john.stultz@linaro.org>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Laura Abbott <labbott@redhat.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Andy Gross <andy.gross@linaro.org>
Cc: Kevin Brodsky <kevin.brodsky@arm.com>
Cc: Andrew Pinski <apinski@cavium.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-kernel at vger.kernel.org
Cc: linux-arm-kernel at lists.infradead.org
Cc: Jeremy Linton <Jeremy.Linton@arm.com>
v2:
- split first CL into 5 of 7 pieces
v4:
- split into two, moving ARCH_CLOCK_FIXED_MASK to later
- update commit message to reflect overall reasoning
- adjust for dropping forced inline
- replace typeof() with types vdso_wtm_clock_nsec_t,
vdso_xtime_clock_sec and __kernel_time_t.
v5:
- drop including linux/time.h in favour of uapi/linux/time.h in
vgettimeofday.c to limit architectural includes.
- add linux/time.h to compiler.h for NSEC_PER_SEC definition.
- replace last timespec_add_ns with __iter_div_u64_rem.
---
arch/arm/include/asm/vdso_datapage.h | 18 +++++-
arch/arm/vdso/compiler.h | 1 +
arch/arm/vdso/vgettimeofday.c | 90 ++++++++++++++++++----------
3 files changed, 76 insertions(+), 33 deletions(-)
diff --git a/arch/arm/include/asm/vdso_datapage.h b/arch/arm/include/asm/vdso_datapage.h
index fa3e1856244d..8dd7303db4ec 100644
--- a/arch/arm/include/asm/vdso_datapage.h
+++ b/arch/arm/include/asm/vdso_datapage.h
@@ -24,6 +24,16 @@
#include <asm/page.h>
+#ifndef _VDSO_WTM_CLOCK_SEC_T
+#define _VDSO_WTM_CLOCK_SEC_T
+typedef u32 vdso_wtm_clock_nsec_t;
+#endif
+
+#ifndef _VDSO_XTIME_CLOCK_SEC_T
+#define _VDSO_XTIME_CLOCK_SEC_T
+typedef u32 vdso_xtime_clock_sec_t;
+#endif
+
/* Try to be cache-friendly on systems that don't implement the
* generic timer: fit the unconditionally updated fields in the first
* 32 bytes.
@@ -35,9 +45,11 @@ struct vdso_data {
u32 xtime_coarse_sec; /* coarse time */
u32 xtime_coarse_nsec;
- u32 wtm_clock_sec; /* wall to monotonic offset */
- u32 wtm_clock_nsec;
- u32 xtime_clock_sec; /* CLOCK_REALTIME - seconds */
+ /* wall to monotonic offset */
+ u32 wtm_clock_sec;
+ vdso_wtm_clock_nsec_t wtm_clock_nsec;
+ /* CLOCK_REALTIME - seconds */
+ vdso_xtime_clock_sec_t xtime_clock_sec;
u32 cs_mono_mult; /* clocksource multiplier */
u64 cs_cycle_last; /* last cycle value */
diff --git a/arch/arm/vdso/compiler.h b/arch/arm/vdso/compiler.h
index af24502797e8..3edddb705a1b 100644
--- a/arch/arm/vdso/compiler.h
+++ b/arch/arm/vdso/compiler.h
@@ -27,6 +27,7 @@
#include <asm/processor.h> /* for cpu_relax() */
#include <asm/unistd.h>
#include <linux/compiler.h>
+#include <linux/time.h> /* for NSEC_PER_SEC */
#ifndef CONFIG_AEABI
#error This code depends on AEABI system call conventions
diff --git a/arch/arm/vdso/vgettimeofday.c b/arch/arm/vdso/vgettimeofday.c
index 522094b147a2..59893fca03b3 100644
--- a/arch/arm/vdso/vgettimeofday.c
+++ b/arch/arm/vdso/vgettimeofday.c
@@ -24,7 +24,8 @@
#include <asm/barrier.h>
#include <linux/compiler.h> /* for notrace */
-#include <linux/time.h>
+#include <linux/math64.h> /* for __iter_div_u64_rem() */
+#include <uapi/linux/time.h> /* for struct timespec */
#include "compiler.h"
#include "datapage.h"
@@ -79,6 +80,7 @@ static notrace int do_monotonic_coarse(const struct vdso_data *vd,
{
struct timespec tomono;
u32 seq;
+ u64 nsec;
do {
seq = vdso_read_begin(vd);
@@ -92,33 +94,41 @@ static notrace int do_monotonic_coarse(const struct vdso_data *vd,
} while (vdso_read_retry(vd, seq));
ts->tv_sec += tomono.tv_sec;
- timespec_add_ns(ts, tomono.tv_nsec);
+ /* open coding timespec_add_ns */
+ ts->tv_sec += __iter_div_u64_rem(ts->tv_nsec + tomono.tv_nsec,
+ NSEC_PER_SEC, &nsec);
+ ts->tv_nsec = nsec;
return 0;
}
#ifdef CONFIG_ARM_ARCH_TIMER
-static notrace u64 get_ns(const struct vdso_data *vd)
+/*
+ * Returns the clock delta, in nanoseconds left-shifted by the clock
+ * shift.
+ */
+static notrace u64 get_clock_shifted_nsec(const u64 cycle_last,
+ const u32 mult,
+ const u64 mask)
{
- u64 cycle_delta;
- u64 cycle_now;
- u64 nsec;
-
- cycle_now = arch_vdso_read_counter();
+ u64 res;
- cycle_delta = (cycle_now - vd->cs_cycle_last) & vd->cs_mask;
+ /* Read the virtual counter. */
+ res = arch_vdso_read_counter();
- nsec = (cycle_delta * vd->cs_mono_mult) + vd->xtime_clock_snsec;
- nsec >>= vd->cs_shift;
+ res = res - cycle_last;
- return nsec;
+ res &= mask;
+ return res * mult;
}
static notrace int do_realtime(const struct vdso_data *vd, struct timespec *ts)
{
- u64 nsecs;
- u32 seq;
+ u32 seq, mult, shift;
+ u64 nsec, cycle_last;
+ u64 mask;
+ vdso_xtime_clock_sec_t sec;
do {
seq = vdso_read_begin(vd);
@@ -126,22 +136,33 @@ static notrace int do_realtime(const struct vdso_data *vd, struct timespec *ts)
if (vd->use_syscall)
return -1;
- ts->tv_sec = vd->xtime_clock_sec;
- nsecs = get_ns(vd);
+ cycle_last = vd->cs_cycle_last;
- } while (vdso_read_retry(vd, seq));
+ mult = vd->cs_mono_mult;
+ shift = vd->cs_shift;
+ mask = vd->cs_mask;
+
+ sec = vd->xtime_clock_sec;
+ nsec = vd->xtime_clock_snsec;
+
+ } while (unlikely(vdso_read_retry(vd, seq)));
- ts->tv_nsec = 0;
- timespec_add_ns(ts, nsecs);
+ nsec += get_clock_shifted_nsec(cycle_last, mult, mask);
+ nsec >>= shift;
+ /* open coding timespec_add_ns to save a ts->tv_nsec = 0 */
+ ts->tv_sec = sec + __iter_div_u64_rem(nsec, NSEC_PER_SEC, &nsec);
+ ts->tv_nsec = nsec;
return 0;
}
static notrace int do_monotonic(const struct vdso_data *vd, struct timespec *ts)
{
- struct timespec tomono;
- u64 nsecs;
- u32 seq;
+ u32 seq, mult, shift;
+ u64 nsec, cycle_last;
+ u64 mask;
+ vdso_wtm_clock_nsec_t wtm_nsec;
+ __kernel_time_t sec;
do {
seq = vdso_read_begin(vd);
@@ -149,17 +170,26 @@ static notrace int do_monotonic(const struct vdso_data *vd, struct timespec *ts)
if (vd->use_syscall)
return -1;
- ts->tv_sec = vd->xtime_clock_sec;
- nsecs = get_ns(vd);
+ cycle_last = vd->cs_cycle_last;
- tomono.tv_sec = vd->wtm_clock_sec;
- tomono.tv_nsec = vd->wtm_clock_nsec;
+ mult = vd->cs_mono_mult;
+ shift = vd->cs_shift;
+ mask = vd->cs_mask;
- } while (vdso_read_retry(vd, seq));
+ sec = vd->xtime_clock_sec;
+ nsec = vd->xtime_clock_snsec;
- ts->tv_sec += tomono.tv_sec;
- ts->tv_nsec = 0;
- timespec_add_ns(ts, nsecs + tomono.tv_nsec);
+ sec += vd->wtm_clock_sec;
+ wtm_nsec = vd->wtm_clock_nsec;
+
+ } while (unlikely(vdso_read_retry(vd, seq)));
+
+ nsec += get_clock_shifted_nsec(cycle_last, mult, mask);
+ nsec >>= shift;
+ nsec += wtm_nsec;
+ /* open coding timespec_add_ns to save a ts->tv_nsec = 0 */
+ ts->tv_sec = sec + __iter_div_u64_rem(nsec, NSEC_PER_SEC, &nsec);
+ ts->tv_nsec = nsec;
return 0;
}
--
2.18.0.rc1.244.gcf134e6275-goog
^ permalink raw reply related
* RESEND [PATCH v5 03/12] arm: vdso: inline assembler operations to compiler.h
From: Mark Salyzyn @ 2018-06-18 15:05 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20180618150613.10322-1-salyzyn@android.com>
Take an effort to recode the arm64 vdso code from assembler to C
previously submitted by Andrew Pinski <apinski@cavium.com>, rework
it for use in both arm and arm64, overlapping any optimizations
for each architecture. But instead of landing it in arm64, land the
result into lib/vdso and unify both implementations to simplify
future maintenance.
Move compiler-specific code to a local compiler.h file:
- CONFIG_AEABI dependency check.
- System call fallback functions standardized into a
DEFINE_FALLBACK macro.
- Replace arch_counter_get_cntvct() with arch_vdso_read_counter.
- Deal with architecture specific unresolved references emitted
by GCC.
- Optimize handling of fallback calls in callers.
- For time functions that always return success, do not waste time
checking return value for switch to fallback.
- Optimize unlikely nullptr checking in __vdso_gettimeofday,
if tv null no need to proceed to fallback, as vdso is still
capable of filling in the tv values.
Signed-off-by: Mark Salyzyn <salyzyn@android.com>
Cc: James Morse <james.morse@arm.com>
Cc: Russell King <linux@armlinux.org.uk>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dmitry Safonov <dsafonov@virtuozzo.com>
Cc: John Stultz <john.stultz@linaro.org>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Laura Abbott <labbott@redhat.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Andy Gross <andy.gross@linaro.org>
Cc: Kevin Brodsky <kevin.brodsky@arm.com>
Cc: Andrew Pinski <apinski@cavium.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-kernel at vger.kernel.org
Cc: linux-arm-kernel at lists.infradead.org
Cc: Jeremy Linton <Jeremy.Linton@arm.com>
v2:
- split first CL into 3 of 7 pieces
v4:
- switch to arch_vdso_read_counter as common API
- update commit message to reflect overall reasoning
v5:
- comment about asm/arch_timer.h and asm/processor.h regarding why
they have been added in compiler.h for porting clarity.
- add linux/compiler.h in vgettimeofday.c because of notrace.
- remove unnecessary dependency on linux/hrtimer.h
---
arch/arm/vdso/compiler.h | 69 +++++++++++++++++++++
arch/arm/vdso/vgettimeofday.c | 109 +++++++++-------------------------
2 files changed, 96 insertions(+), 82 deletions(-)
create mode 100644 arch/arm/vdso/compiler.h
diff --git a/arch/arm/vdso/compiler.h b/arch/arm/vdso/compiler.h
new file mode 100644
index 000000000000..af24502797e8
--- /dev/null
+++ b/arch/arm/vdso/compiler.h
@@ -0,0 +1,69 @@
+/*
+ * Userspace implementations of fallback calls
+ *
+ * Copyright (C) 2017 Cavium, Inc.
+ * Copyright (C) 2012 ARM Limited
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ * Author: Will Deacon <will.deacon@arm.com>
+ * Rewriten into C by: Andrew Pinski <apinski@cavium.com>
+ */
+
+#ifndef __VDSO_COMPILER_H
+#define __VDSO_COMPILER_H
+
+#include <asm/arch_timer.h> /* for arch_counter_get_cntvct() */
+#include <asm/processor.h> /* for cpu_relax() */
+#include <asm/unistd.h>
+#include <linux/compiler.h>
+
+#ifndef CONFIG_AEABI
+#error This code depends on AEABI system call conventions
+#endif
+
+#define DEFINE_FALLBACK(name, type_arg1, name_arg1, type_arg2, name_arg2) \
+static notrace long name##_fallback(type_arg1 _##name_arg1, \
+ type_arg2 _##name_arg2) \
+{ \
+ register type_arg1 name_arg1 asm("r0") = _##name_arg1; \
+ register type_arg2 name_arg2 asm("r1") = _##name_arg2; \
+ register long ret asm ("r0"); \
+ register long nr asm("r7") = __NR_##name; \
+ \
+ asm volatile( \
+ " swi #0\n" \
+ : "=r" (ret) \
+ : "r" (name_arg1), "r" (name_arg2), "r" (nr) \
+ : "memory"); \
+ \
+ return ret; \
+}
+
+#define arch_vdso_read_counter() arch_counter_get_cntvct()
+
+/* Avoid unresolved references emitted by GCC */
+
+void __aeabi_unwind_cpp_pr0(void)
+{
+}
+
+void __aeabi_unwind_cpp_pr1(void)
+{
+}
+
+void __aeabi_unwind_cpp_pr2(void)
+{
+}
+
+#endif /* __VDSO_COMPILER_H */
diff --git a/arch/arm/vdso/vgettimeofday.c b/arch/arm/vdso/vgettimeofday.c
index 2474c17dc356..522094b147a2 100644
--- a/arch/arm/vdso/vgettimeofday.c
+++ b/arch/arm/vdso/vgettimeofday.c
@@ -22,21 +22,16 @@
* Reworked and rebased over arm version by: Mark Salyzyn <salyzyn@android.com>
*/
-#include <linux/compiler.h>
-#include <linux/hrtimer.h>
-#include <linux/time.h>
-#include <asm/arch_timer.h>
#include <asm/barrier.h>
-#include <asm/bug.h>
-#include <asm/page.h>
-#include <asm/unistd.h>
-
-#ifndef CONFIG_AEABI
-#error This code depends on AEABI system call conventions
-#endif
+#include <linux/compiler.h> /* for notrace */
+#include <linux/time.h>
+#include "compiler.h"
#include "datapage.h"
+DEFINE_FALLBACK(gettimeofday, struct timeval *, tv, struct timezone *, tz)
+DEFINE_FALLBACK(clock_gettime, clockid_t, clock, struct timespec *, ts)
+
static notrace u32 vdso_read_begin(const struct vdso_data *vd)
{
u32 seq;
@@ -63,23 +58,6 @@ static notrace int vdso_read_retry(const struct vdso_data *vd, u32 start)
return seq != start;
}
-static notrace long clock_gettime_fallback(clockid_t _clkid,
- struct timespec *_ts)
-{
- register struct timespec *ts asm("r1") = _ts;
- register clockid_t clkid asm("r0") = _clkid;
- register long ret asm ("r0");
- register long nr asm("r7") = __NR_clock_gettime;
-
- asm volatile(
- " swi #0\n"
- : "=r" (ret)
- : "r" (clkid), "r" (ts), "r" (nr)
- : "memory");
-
- return ret;
-}
-
static notrace int do_realtime_coarse(const struct vdso_data *vd,
struct timespec *ts)
{
@@ -127,7 +105,7 @@ static notrace u64 get_ns(const struct vdso_data *vd)
u64 cycle_now;
u64 nsec;
- cycle_now = arch_counter_get_cntvct();
+ cycle_now = arch_vdso_read_counter();
cycle_delta = (cycle_now - vd->cs_cycle_last) & vd->cs_mask;
@@ -200,85 +178,52 @@ static notrace int do_monotonic(const struct vdso_data *vd, struct timespec *ts)
#endif /* CONFIG_ARM_ARCH_TIMER */
-notrace int __vdso_clock_gettime(clockid_t clkid, struct timespec *ts)
+notrace int __vdso_clock_gettime(clockid_t clock, struct timespec *ts)
{
- int ret = -1;
-
const struct vdso_data *vd = __get_datapage();
- switch (clkid) {
+ switch (clock) {
case CLOCK_REALTIME_COARSE:
- ret = do_realtime_coarse(vd, ts);
+ do_realtime_coarse(vd, ts);
break;
case CLOCK_MONOTONIC_COARSE:
- ret = do_monotonic_coarse(vd, ts);
+ do_monotonic_coarse(vd, ts);
break;
case CLOCK_REALTIME:
- ret = do_realtime(vd, ts);
+ if (do_realtime(vd, ts))
+ goto fallback;
break;
case CLOCK_MONOTONIC:
- ret = do_monotonic(vd, ts);
+ if (do_monotonic(vd, ts))
+ goto fallback;
break;
default:
- break;
+ goto fallback;
}
- if (ret)
- ret = clock_gettime_fallback(clkid, ts);
-
- return ret;
-}
-
-static notrace long gettimeofday_fallback(struct timeval *_tv,
- struct timezone *_tz)
-{
- register struct timezone *tz asm("r1") = _tz;
- register struct timeval *tv asm("r0") = _tv;
- register long ret asm ("r0");
- register long nr asm("r7") = __NR_gettimeofday;
-
- asm volatile(
- " swi #0\n"
- : "=r" (ret)
- : "r" (tv), "r" (tz), "r" (nr)
- : "memory");
-
- return ret;
+ return 0;
+fallback:
+ return clock_gettime_fallback(clock, ts);
}
notrace int __vdso_gettimeofday(struct timeval *tv, struct timezone *tz)
{
- struct timespec ts;
- int ret;
-
const struct vdso_data *vd = __get_datapage();
- ret = do_realtime(vd, &ts);
- if (ret)
- return gettimeofday_fallback(tv, tz);
+ if (likely(tv != NULL)) {
+ struct timespec ts;
+
+ if (do_realtime(vd, &ts))
+ return gettimeofday_fallback(tv, tz);
- if (tv) {
tv->tv_sec = ts.tv_sec;
tv->tv_usec = ts.tv_nsec / 1000;
}
- if (tz) {
+
+ if (unlikely(tz != NULL)) {
tz->tz_minuteswest = vd->tz_minuteswest;
tz->tz_dsttime = vd->tz_dsttime;
}
- return ret;
-}
-
-/* Avoid unresolved references emitted by GCC */
-
-void __aeabi_unwind_cpp_pr0(void)
-{
-}
-
-void __aeabi_unwind_cpp_pr1(void)
-{
-}
-
-void __aeabi_unwind_cpp_pr2(void)
-{
+ return 0;
}
--
2.18.0.rc1.244.gcf134e6275-goog
^ permalink raw reply related
* RESEND [PATCH v5 02/12] arm: vdso: add include file defining __get_datapage()
From: Mark Salyzyn @ 2018-06-18 15:05 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20180618150613.10322-1-salyzyn@android.com>
Take an effort to recode the arm64 vdso code from assembler to C
previously submitted by Andrew Pinski <apinski@cavium.com>, rework
it for use in both arm and arm64, overlapping any optimizations
for each architecture. But instead of landing it in arm64, land the
result into lib/vdso and unify both implementations to simplify
future maintenance.
Define the prototype for __get_datapage() in local datapage.h header.
Rename all vdata variable that point to the datapage shortened to vd
to relect a consistent and concise style. Make sure that all
references to the datapage in vdso operations are readonly (const).
Make sure datapage is first parameter to all subroutines to also
be consistent.
Signed-off-by: Mark Salyzyn <salyzyn@android.com>
Cc: James Morse <james.morse@arm.com>
Cc: Russell King <linux@armlinux.org.uk>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dmitry Safonov <dsafonov@virtuozzo.com>
Cc: John Stultz <john.stultz@linaro.org>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Laura Abbott <labbott@redhat.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Andy Gross <andy.gross@linaro.org>
Cc: Kevin Brodsky <kevin.brodsky@arm.com>
Cc: Andrew Pinski <apinski@cavium.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-kernel at vger.kernel.org
Cc: linux-arm-kernel at lists.infradead.org
Cc: Jeremy Linton <Jeremy.Linton@arm.com>
v2:
- split first CL into 2 of 7 pieces
v4:
- update commit message to reflect overall reasoning
v5:
- rebase
---
arch/arm/vdso/datapage.h | 25 +++++++++
arch/arm/vdso/vgettimeofday.c | 99 +++++++++++++++++------------------
2 files changed, 73 insertions(+), 51 deletions(-)
create mode 100644 arch/arm/vdso/datapage.h
diff --git a/arch/arm/vdso/datapage.h b/arch/arm/vdso/datapage.h
new file mode 100644
index 000000000000..e3088bdfb946
--- /dev/null
+++ b/arch/arm/vdso/datapage.h
@@ -0,0 +1,25 @@
+/*
+ * Userspace implementations of __get_datapage
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef __VDSO_DATAPAGE_H
+#define __VDSO_DATAPAGE_H
+
+#include <linux/types.h>
+#include <asm/vdso_datapage.h>
+
+extern const struct vdso_data *__get_datapage(void);
+
+#endif /* __VDSO_DATAPAGE_H */
diff --git a/arch/arm/vdso/vgettimeofday.c b/arch/arm/vdso/vgettimeofday.c
index 8cf13af1323c..2474c17dc356 100644
--- a/arch/arm/vdso/vgettimeofday.c
+++ b/arch/arm/vdso/vgettimeofday.c
@@ -30,20 +30,19 @@
#include <asm/bug.h>
#include <asm/page.h>
#include <asm/unistd.h>
-#include <asm/vdso_datapage.h>
#ifndef CONFIG_AEABI
#error This code depends on AEABI system call conventions
#endif
-extern struct vdso_data *__get_datapage(void);
+#include "datapage.h"
-static notrace u32 vdso_read_begin(const struct vdso_data *vdata)
+static notrace u32 vdso_read_begin(const struct vdso_data *vd)
{
u32 seq;
do {
- seq = READ_ONCE(vdata->tb_seq_count);
+ seq = READ_ONCE(vd->tb_seq_count);
if ((seq & 1) == 0)
break;
@@ -55,12 +54,12 @@ static notrace u32 vdso_read_begin(const struct vdso_data *vdata)
return seq;
}
-static notrace int vdso_read_retry(const struct vdso_data *vdata, u32 start)
+static notrace int vdso_read_retry(const struct vdso_data *vd, u32 start)
{
u32 seq;
smp_rmb(); /* Pairs with first smp_wmb in update_vsyscall */
- seq = READ_ONCE(vdata->tb_seq_count);
+ seq = READ_ONCE(vd->tb_seq_count);
return seq != start;
}
@@ -81,38 +80,38 @@ static notrace long clock_gettime_fallback(clockid_t _clkid,
return ret;
}
-static notrace int do_realtime_coarse(struct timespec *ts,
- struct vdso_data *vdata)
+static notrace int do_realtime_coarse(const struct vdso_data *vd,
+ struct timespec *ts)
{
u32 seq;
do {
- seq = vdso_read_begin(vdata);
+ seq = vdso_read_begin(vd);
- ts->tv_sec = vdata->xtime_coarse_sec;
- ts->tv_nsec = vdata->xtime_coarse_nsec;
+ ts->tv_sec = vd->xtime_coarse_sec;
+ ts->tv_nsec = vd->xtime_coarse_nsec;
- } while (vdso_read_retry(vdata, seq));
+ } while (vdso_read_retry(vd, seq));
return 0;
}
-static notrace int do_monotonic_coarse(struct timespec *ts,
- struct vdso_data *vdata)
+static notrace int do_monotonic_coarse(const struct vdso_data *vd,
+ struct timespec *ts)
{
struct timespec tomono;
u32 seq;
do {
- seq = vdso_read_begin(vdata);
+ seq = vdso_read_begin(vd);
- ts->tv_sec = vdata->xtime_coarse_sec;
- ts->tv_nsec = vdata->xtime_coarse_nsec;
+ ts->tv_sec = vd->xtime_coarse_sec;
+ ts->tv_nsec = vd->xtime_coarse_nsec;
- tomono.tv_sec = vdata->wtm_clock_sec;
- tomono.tv_nsec = vdata->wtm_clock_nsec;
+ tomono.tv_sec = vd->wtm_clock_sec;
+ tomono.tv_nsec = vd->wtm_clock_nsec;
- } while (vdso_read_retry(vdata, seq));
+ } while (vdso_read_retry(vd, seq));
ts->tv_sec += tomono.tv_sec;
timespec_add_ns(ts, tomono.tv_nsec);
@@ -122,7 +121,7 @@ static notrace int do_monotonic_coarse(struct timespec *ts,
#ifdef CONFIG_ARM_ARCH_TIMER
-static notrace u64 get_ns(struct vdso_data *vdata)
+static notrace u64 get_ns(const struct vdso_data *vd)
{
u64 cycle_delta;
u64 cycle_now;
@@ -130,29 +129,29 @@ static notrace u64 get_ns(struct vdso_data *vdata)
cycle_now = arch_counter_get_cntvct();
- cycle_delta = (cycle_now - vdata->cs_cycle_last) & vdata->cs_mask;
+ cycle_delta = (cycle_now - vd->cs_cycle_last) & vd->cs_mask;
- nsec = (cycle_delta * vdata->cs_mono_mult) + vdata->xtime_clock_snsec;
- nsec >>= vdata->cs_shift;
+ nsec = (cycle_delta * vd->cs_mono_mult) + vd->xtime_clock_snsec;
+ nsec >>= vd->cs_shift;
return nsec;
}
-static notrace int do_realtime(struct timespec *ts, struct vdso_data *vdata)
+static notrace int do_realtime(const struct vdso_data *vd, struct timespec *ts)
{
u64 nsecs;
u32 seq;
do {
- seq = vdso_read_begin(vdata);
+ seq = vdso_read_begin(vd);
- if (vdata->use_syscall)
+ if (vd->use_syscall)
return -1;
- ts->tv_sec = vdata->xtime_clock_sec;
- nsecs = get_ns(vdata);
+ ts->tv_sec = vd->xtime_clock_sec;
+ nsecs = get_ns(vd);
- } while (vdso_read_retry(vdata, seq));
+ } while (vdso_read_retry(vd, seq));
ts->tv_nsec = 0;
timespec_add_ns(ts, nsecs);
@@ -160,25 +159,25 @@ static notrace int do_realtime(struct timespec *ts, struct vdso_data *vdata)
return 0;
}
-static notrace int do_monotonic(struct timespec *ts, struct vdso_data *vdata)
+static notrace int do_monotonic(const struct vdso_data *vd, struct timespec *ts)
{
struct timespec tomono;
u64 nsecs;
u32 seq;
do {
- seq = vdso_read_begin(vdata);
+ seq = vdso_read_begin(vd);
- if (vdata->use_syscall)
+ if (vd->use_syscall)
return -1;
- ts->tv_sec = vdata->xtime_clock_sec;
- nsecs = get_ns(vdata);
+ ts->tv_sec = vd->xtime_clock_sec;
+ nsecs = get_ns(vd);
- tomono.tv_sec = vdata->wtm_clock_sec;
- tomono.tv_nsec = vdata->wtm_clock_nsec;
+ tomono.tv_sec = vd->wtm_clock_sec;
+ tomono.tv_nsec = vd->wtm_clock_nsec;
- } while (vdso_read_retry(vdata, seq));
+ } while (vdso_read_retry(vd, seq));
ts->tv_sec += tomono.tv_sec;
ts->tv_nsec = 0;
@@ -189,12 +188,12 @@ static notrace int do_monotonic(struct timespec *ts, struct vdso_data *vdata)
#else /* CONFIG_ARM_ARCH_TIMER */
-static notrace int do_realtime(struct timespec *ts, struct vdso_data *vdata)
+static notrace int do_realtime(const struct vdso_data *vd, struct timespec *ts)
{
return -1;
}
-static notrace int do_monotonic(struct timespec *ts, struct vdso_data *vdata)
+static notrace int do_monotonic(const struct vdso_data *vd, struct timespec *ts)
{
return -1;
}
@@ -203,23 +202,22 @@ static notrace int do_monotonic(struct timespec *ts, struct vdso_data *vdata)
notrace int __vdso_clock_gettime(clockid_t clkid, struct timespec *ts)
{
- struct vdso_data *vdata;
int ret = -1;
- vdata = __get_datapage();
+ const struct vdso_data *vd = __get_datapage();
switch (clkid) {
case CLOCK_REALTIME_COARSE:
- ret = do_realtime_coarse(ts, vdata);
+ ret = do_realtime_coarse(vd, ts);
break;
case CLOCK_MONOTONIC_COARSE:
- ret = do_monotonic_coarse(ts, vdata);
+ ret = do_monotonic_coarse(vd, ts);
break;
case CLOCK_REALTIME:
- ret = do_realtime(ts, vdata);
+ ret = do_realtime(vd, ts);
break;
case CLOCK_MONOTONIC:
- ret = do_monotonic(ts, vdata);
+ ret = do_monotonic(vd, ts);
break;
default:
break;
@@ -251,12 +249,11 @@ static notrace long gettimeofday_fallback(struct timeval *_tv,
notrace int __vdso_gettimeofday(struct timeval *tv, struct timezone *tz)
{
struct timespec ts;
- struct vdso_data *vdata;
int ret;
- vdata = __get_datapage();
+ const struct vdso_data *vd = __get_datapage();
- ret = do_realtime(&ts, vdata);
+ ret = do_realtime(vd, &ts);
if (ret)
return gettimeofday_fallback(tv, tz);
@@ -265,8 +262,8 @@ notrace int __vdso_gettimeofday(struct timeval *tv, struct timezone *tz)
tv->tv_usec = ts.tv_nsec / 1000;
}
if (tz) {
- tz->tz_minuteswest = vdata->tz_minuteswest;
- tz->tz_dsttime = vdata->tz_dsttime;
+ tz->tz_minuteswest = vd->tz_minuteswest;
+ tz->tz_dsttime = vd->tz_dsttime;
}
return ret;
--
2.18.0.rc1.244.gcf134e6275-goog
^ permalink raw reply related
* RESEND [PATCH v5 01/12] arm: vdso: rename vdso_datapage variables
From: Mark Salyzyn @ 2018-06-18 15:05 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20180618150613.10322-1-salyzyn@android.com>
Take an effort to recode the arm64 vdso code from assembler to C
previously submitted by Andrew Pinski <apinski@cavium.com>, rework
it for use in both arm and arm64, overlapping any optimizations
for each architecture. But instead of landing it in arm64, land the
result into lib/vdso and unify both implementations to simplify
future maintenance.
Rename seq_count to tb_seq_count. Rename tk_is_cntvct to use_syscall.
Rename cs_mult to cs_mono_mult. All to align with the variables in the
arm64 vdso datapage. Rework vdso_read_begin() and vdso_read_retry()
functions to reflect modern access patterns for tb_seq_count field.
Update copyright message to reflect the start of the contributions in
this series.
Signed-off-by: Mark Salyzyn <salyzyn@android.com>
Cc: James Morse <james.morse@arm.com>
Cc: Russell King <linux@armlinux.org.uk>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dmitry Safonov <dsafonov@virtuozzo.com>
Cc: John Stultz <john.stultz@linaro.org>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Laura Abbott <labbott@redhat.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Andy Gross <andy.gross@linaro.org>
Cc: Kevin Brodsky <kevin.brodsky@arm.com>
Cc: Andrew Pinski <apinski@cavium.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-kernel at vger.kernel.org
Cc: linux-arm-kernel at lists.infradead.org
Cc: Jeremy Linton <Jeremy.Linton@arm.com>
v2:
- split first CL into 1 of 7 pieces
v4:
- update commit message to reflect reasoning
v5:
- rebase
---
arch/arm/include/asm/vdso_datapage.h | 6 +--
arch/arm/kernel/vdso.c | 17 ++++----
arch/arm/vdso/vgettimeofday.c | 61 +++++++++++++++-------------
3 files changed, 45 insertions(+), 39 deletions(-)
diff --git a/arch/arm/include/asm/vdso_datapage.h b/arch/arm/include/asm/vdso_datapage.h
index 9be259442fca..fa3e1856244d 100644
--- a/arch/arm/include/asm/vdso_datapage.h
+++ b/arch/arm/include/asm/vdso_datapage.h
@@ -29,8 +29,8 @@
* 32 bytes.
*/
struct vdso_data {
- u32 seq_count; /* sequence count - odd during updates */
- u16 tk_is_cntvct; /* fall back to syscall if false */
+ u32 tb_seq_count; /* sequence count - odd during updates */
+ u16 use_syscall; /* fall back to syscall if true */
u16 cs_shift; /* clocksource shift */
u32 xtime_coarse_sec; /* coarse time */
u32 xtime_coarse_nsec;
@@ -38,7 +38,7 @@ struct vdso_data {
u32 wtm_clock_sec; /* wall to monotonic offset */
u32 wtm_clock_nsec;
u32 xtime_clock_sec; /* CLOCK_REALTIME - seconds */
- u32 cs_mult; /* clocksource multiplier */
+ u32 cs_mono_mult; /* clocksource multiplier */
u64 cs_cycle_last; /* last cycle value */
u64 cs_mask; /* clocksource mask */
diff --git a/arch/arm/kernel/vdso.c b/arch/arm/kernel/vdso.c
index f4dd7f9663c1..c2c57f6b8c60 100644
--- a/arch/arm/kernel/vdso.c
+++ b/arch/arm/kernel/vdso.c
@@ -276,14 +276,14 @@ void arm_install_vdso(struct mm_struct *mm, unsigned long addr)
static void vdso_write_begin(struct vdso_data *vdata)
{
- ++vdso_data->seq_count;
+ ++vdso_data->tb_seq_count;
smp_wmb(); /* Pairs with smp_rmb in vdso_read_retry */
}
static void vdso_write_end(struct vdso_data *vdata)
{
smp_wmb(); /* Pairs with smp_rmb in vdso_read_begin */
- ++vdso_data->seq_count;
+ ++vdso_data->tb_seq_count;
}
static bool tk_is_cntvct(const struct timekeeper *tk)
@@ -307,10 +307,10 @@ static bool tk_is_cntvct(const struct timekeeper *tk)
* counter again, making it even, indicating to userspace that the
* update is finished.
*
- * Userspace is expected to sample seq_count before reading any other
- * fields from the data page. If seq_count is odd, userspace is
+ * Userspace is expected to sample tb_seq_count before reading any other
+ * fields from the data page. If tb_seq_count is odd, userspace is
* expected to wait until it becomes even. After copying data from
- * the page, userspace must sample seq_count again; if it has changed
+ * the page, userspace must sample tb_seq_count again; if it has changed
* from its previous value, userspace must retry the whole sequence.
*
* Calls to update_vsyscall are serialized by the timekeeping core.
@@ -328,18 +328,19 @@ void update_vsyscall(struct timekeeper *tk)
vdso_write_begin(vdso_data);
- vdso_data->tk_is_cntvct = tk_is_cntvct(tk);
+ vdso_data->use_syscall = !tk_is_cntvct(tk);
vdso_data->xtime_coarse_sec = tk->xtime_sec;
vdso_data->xtime_coarse_nsec = (u32)(tk->tkr_mono.xtime_nsec >>
tk->tkr_mono.shift);
vdso_data->wtm_clock_sec = wtm->tv_sec;
vdso_data->wtm_clock_nsec = wtm->tv_nsec;
- if (vdso_data->tk_is_cntvct) {
+ if (!vdso_data->use_syscall) {
vdso_data->cs_cycle_last = tk->tkr_mono.cycle_last;
vdso_data->xtime_clock_sec = tk->xtime_sec;
vdso_data->xtime_clock_snsec = tk->tkr_mono.xtime_nsec;
- vdso_data->cs_mult = tk->tkr_mono.mult;
+ vdso_data->cs_mono_mult = tk->tkr_mono.mult;
+ /* tkr_mono.shift == tkr_raw.shift */
vdso_data->cs_shift = tk->tkr_mono.shift;
vdso_data->cs_mask = tk->tkr_mono.mask;
}
diff --git a/arch/arm/vdso/vgettimeofday.c b/arch/arm/vdso/vgettimeofday.c
index a9dd619c6c29..8cf13af1323c 100644
--- a/arch/arm/vdso/vgettimeofday.c
+++ b/arch/arm/vdso/vgettimeofday.c
@@ -1,18 +1,25 @@
/*
- * Copyright 2015 Mentor Graphics Corporation.
+ * Userspace implementations of gettimeofday() and friends.
*
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; version 2 of the
- * License.
+ * Copyright (C) 2017 Cavium, Inc.
+ * Copyright (C) 2015 Mentor Graphics Corporation
+ * Copyright (C) 2012 ARM Limited
*
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * General Public License for more details.
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ * Author: Will Deacon <will.deacon@arm.com>
+ * Rewriten from arch64 version into C by: Andrew Pinski <apinski@cavium.com>
+ * Reworked and rebased over arm version by: Mark Salyzyn <salyzyn@android.com>
*/
#include <linux/compiler.h>
@@ -31,32 +38,30 @@
extern struct vdso_data *__get_datapage(void);
-static notrace u32 __vdso_read_begin(const struct vdso_data *vdata)
-{
- u32 seq;
-repeat:
- seq = READ_ONCE(vdata->seq_count);
- if (seq & 1) {
- cpu_relax();
- goto repeat;
- }
- return seq;
-}
-
static notrace u32 vdso_read_begin(const struct vdso_data *vdata)
{
u32 seq;
- seq = __vdso_read_begin(vdata);
+ do {
+ seq = READ_ONCE(vdata->tb_seq_count);
+
+ if ((seq & 1) == 0)
+ break;
- smp_rmb(); /* Pairs with smp_wmb in vdso_write_end */
+ cpu_relax();
+ } while (true);
+
+ smp_rmb(); /* Pairs with second smp_wmb in update_vsyscall */
return seq;
}
static notrace int vdso_read_retry(const struct vdso_data *vdata, u32 start)
{
- smp_rmb(); /* Pairs with smp_wmb in vdso_write_begin */
- return vdata->seq_count != start;
+ u32 seq;
+
+ smp_rmb(); /* Pairs with first smp_wmb in update_vsyscall */
+ seq = READ_ONCE(vdata->tb_seq_count);
+ return seq != start;
}
static notrace long clock_gettime_fallback(clockid_t _clkid,
@@ -127,7 +132,7 @@ static notrace u64 get_ns(struct vdso_data *vdata)
cycle_delta = (cycle_now - vdata->cs_cycle_last) & vdata->cs_mask;
- nsec = (cycle_delta * vdata->cs_mult) + vdata->xtime_clock_snsec;
+ nsec = (cycle_delta * vdata->cs_mono_mult) + vdata->xtime_clock_snsec;
nsec >>= vdata->cs_shift;
return nsec;
@@ -141,7 +146,7 @@ static notrace int do_realtime(struct timespec *ts, struct vdso_data *vdata)
do {
seq = vdso_read_begin(vdata);
- if (!vdata->tk_is_cntvct)
+ if (vdata->use_syscall)
return -1;
ts->tv_sec = vdata->xtime_clock_sec;
@@ -164,7 +169,7 @@ static notrace int do_monotonic(struct timespec *ts, struct vdso_data *vdata)
do {
seq = vdso_read_begin(vdata);
- if (!vdata->tk_is_cntvct)
+ if (vdata->use_syscall)
return -1;
ts->tv_sec = vdata->xtime_clock_sec;
--
2.18.0.rc1.244.gcf134e6275-goog
^ permalink raw reply related
* RESEND arm+arm64+aarch32 vdso rewrite
From: Mark Salyzyn @ 2018-06-18 15:05 UTC (permalink / raw)
To: linux-arm-kernel
The following 23 patches are rebased and resent, and represent a
rewrite of the arm and arm64 vDSO into C, adding support for arch32
(32-bit user space hosted 64-bit kernels) and into a common library
that other (arm, or non-arm) architectures may utilize.
[PATCH v5 01/12] arm: vdso: rename vdso_datapage variables
[PATCH v5 02/12] arm: vdso: add include file defining __get_datapage()
[PATCH v5 03/12] arm: vdso: inline assembler operations to compiler.h
[PATCH v5 04/12] arm: vdso: do calculations outside reader loops
[PATCH v6 05/12] arm: vdso: Add support for CLOCK_MONOTONIC_RAW
[PATCH v5 06/12] arm: vdso: add support for clock_getres
[PATCH v5 07/12] arm: vdso: disable profiling
[PATCH v5 08/12] arm: vdso: Add ARCH_CLOCK_FIXED_MASK
[PATCH v5 09/12] arm: vdso: move vgettimeofday.c to lib/vdso/
[PATCH v5 10/12] arm64: vdso: replace gettimeofday.S with global vgettimeofday.C
[PATCH v6 11/12] lib: vdso: Add support for CLOCK_BOOTTIME
[PATCH v5 12/12] lib: vdso: do not expose gettimeofday, if no arch supported timer
[PATCH] lib: vdso: add support for time
[PATCH v2 1/3] arm64: compat: Split the sigreturn trampolines and kuser helpers (C sources)
[PATCH v2 2/3] arm64: compat: Split the sigreturn trampolines and kuser helpers (assembler sources)
[PATCH v2 3/3] arm64: compat: Add CONFIG_KUSER_HELPERS
[PATCH] arm64: compat: Expose offset to registers in sigframes
[PATCH 1/6] arm64: compat: Use vDSO sigreturn trampolines if available
[PATCH 2/6] arm64: elf: Set AT_SYSINFO_EHDR in compat processes
[PATCH 3/6] arm64: Refactor vDSO init/setup
[PATCH v2 4/6] arm64: compat: Add a 32-bit vDSO
[PATCH 5/6] arm64: compat: 32-bit vDSO setup
[PATCH 6/6] arm64: Wire up and expose the new compat vDSO
This patch series' above has been applied to the latest Pixel phones
and resulted in a 0.4% battery improvement.
Signed-off-by: Mark Salyzyn <salyzyn@android.com>
Cc: James Morse <james.morse@arm.com>
Cc: Russell King <linux@armlinux.org.uk>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dmitry Safonov <dsafonov@virtuozzo.com>
Cc: John Stultz <john.stultz@linaro.org>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Laura Abbott <labbott@redhat.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Andy Gross <andy.gross@linaro.org>
Cc: Kevin Brodsky <kevin.brodsky@arm.com>
Cc: Andrew Pinski <apinski@cavium.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-kernel at vger.kernel.org
Cc: linux-arm-kernel at lists.infradead.org
Cc: Jeremy Linton <Jeremy.Linton@arm.com>
^ permalink raw reply
* [PATCH v4 19/19] ARM: dts: sun8i-a33: Add Video Engine and reserved memory nodes
From: Paul Kocialkowski @ 2018-06-18 14:58 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20180618145843.14631-1-paul.kocialkowski@bootlin.com>
This adds nodes for the Video Engine and the associated reserved memory
for the A33. Up to 96 MiB of memory are dedicated to the CMA pool.
The VPU can only map the first 256 MiB of DRAM, so the reserved memory
pool has to be located in that area. Following Allwinner's decision in
downstream software, the last 96 MiB of the first 256 MiB of RAM are
reserved for this purpose.
Signed-off-by: Paul Kocialkowski <paul.kocialkowski@bootlin.com>
diff --git a/arch/arm/boot/dts/sun8i-a33.dtsi b/arch/arm/boot/dts/sun8i-a33.dtsi
index 97a976b3b1ef..c5c108a11285 100644
--- a/arch/arm/boot/dts/sun8i-a33.dtsi
+++ b/arch/arm/boot/dts/sun8i-a33.dtsi
@@ -181,6 +181,21 @@
reg = <0x40000000 0x80000000>;
};
+ reserved-memory {
+ #address-cells = <1>;
+ #size-cells = <1>;
+ ranges;
+
+ /* Address must be kept in the lower 256 MiBs of DRAM for VE. */
+ cma_pool: cma at 4a000000 {
+ compatible = "shared-dma-pool";
+ size = <0x6000000>;
+ alloc-ranges = <0x4a000000 0x6000000>;
+ reusable;
+ linux,cma-default;
+ };
+ };
+
sound: sound {
compatible = "simple-audio-card";
simple-audio-card,name = "sun8i-a33-audio";
@@ -262,6 +277,20 @@
};
};
+ vpu: video-codec at 01c0e000 {
+ compatible = "allwinner,sun8i-a33-video-engine";
+ reg = <0x01c0e000 0x1000>;
+
+ clocks = <&ccu CLK_BUS_VE>, <&ccu CLK_VE>,
+ <&ccu CLK_DRAM_VE>;
+ clock-names = "ahb", "mod", "ram";
+
+ resets = <&ccu RST_BUS_VE>;
+
+ interrupts = <GIC_SPI 58 IRQ_TYPE_LEVEL_HIGH>;
+ allwinner,sram = <&ve_sram 1>;
+ };
+
crypto: crypto-engine at 1c15000 {
compatible = "allwinner,sun4i-a10-crypto";
reg = <0x01c15000 0x1000>;
--
2.17.0
^ permalink raw reply related
* [PATCH v4 18/19] ARM: dts: sun7i-a20: Add Video Engine and reserved memory nodes
From: Paul Kocialkowski @ 2018-06-18 14:58 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20180618145843.14631-1-paul.kocialkowski@bootlin.com>
This adds nodes for the Video Engine and the associated reserved memory
for the A20. Up to 96 MiB of memory are dedicated to the CMA pool.
The VPU can only map the first 256 MiB of DRAM, so the reserved memory
pool has to be located in that area. Following Allwinner's decision in
downstream software, the last 96 MiB of the first 256 MiB of RAM are
reserved for this purpose.
Signed-off-by: Paul Kocialkowski <paul.kocialkowski@bootlin.com>
diff --git a/arch/arm/boot/dts/sun7i-a20.dtsi b/arch/arm/boot/dts/sun7i-a20.dtsi
index 9afd82f9ec79..479272f02686 100644
--- a/arch/arm/boot/dts/sun7i-a20.dtsi
+++ b/arch/arm/boot/dts/sun7i-a20.dtsi
@@ -161,6 +161,21 @@
reg = <0x40000000 0x80000000>;
};
+ reserved-memory {
+ #address-cells = <1>;
+ #size-cells = <1>;
+ ranges;
+
+ /* Address must be kept in the lower 256 MiBs of DRAM for VE. */
+ cma_pool: cma at 4a000000 {
+ compatible = "shared-dma-pool";
+ size = <0x6000000>;
+ alloc-ranges = <0x4a000000 0x6000000>;
+ reusable;
+ linux,cma-default;
+ };
+ };
+
timer {
compatible = "arm,armv7-timer";
interrupts = <GIC_PPI 13 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>,
@@ -464,6 +479,19 @@
};
};
+ vpu: video-codec at 1c0e000 {
+ compatible = "allwinner,sun7i-a20-video-engine";
+ reg = <0x01c0e000 0x1000>;
+
+ clocks = <&ccu CLK_AHB_VE>, <&ccu CLK_VE>,
+ <&ccu CLK_DRAM_VE>;
+ clock-names = "ahb", "mod", "ram";
+
+ resets = <&ccu RST_VE>;
+ interrupts = <GIC_SPI 53 IRQ_TYPE_LEVEL_HIGH>;
+ allwinner,sram = <&ve_sram 1>;
+ };
+
mmc0: mmc at 1c0f000 {
compatible = "allwinner,sun7i-a20-mmc";
reg = <0x01c0f000 0x1000>;
--
2.17.0
^ permalink raw reply related
* [PATCH v4 17/19] ARM: dts: sun5i: Add Video Engine and reserved memory nodes
From: Paul Kocialkowski @ 2018-06-18 14:58 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20180618145843.14631-1-paul.kocialkowski@bootlin.com>
This adds nodes for the Video Engine and the associated reserved memory
for sun5i-based platforms. Up to 96 MiB of memory are dedicated to the
CMA pool.
The VPU can only map the first 256 MiB of DRAM, so the reserved memory
pool has to be located in that area. Following Allwinner's decision in
downstream software, the last 96 MiB of the first 256 MiB of RAM are
reserved for this purpose.
Signed-off-by: Paul Kocialkowski <paul.kocialkowski@bootlin.com>
diff --git a/arch/arm/boot/dts/sun5i.dtsi b/arch/arm/boot/dts/sun5i.dtsi
index 6fa9e28edc59..3138f791b6b2 100644
--- a/arch/arm/boot/dts/sun5i.dtsi
+++ b/arch/arm/boot/dts/sun5i.dtsi
@@ -108,6 +108,21 @@
};
};
+ reserved-memory {
+ #address-cells = <1>;
+ #size-cells = <1>;
+ ranges;
+
+ /* Address must be kept in the lower 256 MiBs of DRAM for VE. */
+ cma_pool: cma at 4a000000 {
+ compatible = "shared-dma-pool";
+ size = <0x6000000>;
+ alloc-ranges = <0x4a000000 0x6000000>;
+ reusable;
+ linux,cma-default;
+ };
+ };
+
soc at 1c00000 {
compatible = "simple-bus";
#address-cells = <1>;
@@ -292,6 +307,19 @@
};
};
+ vpu: video-codec at 1c0e000 {
+ compatible = "allwinner,sun5i-a13-video-engine";
+ reg = <0x01c0e000 0x1000>;
+
+ clocks = <&ccu CLK_AHB_VE>, <&ccu CLK_VE>,
+ <&ccu CLK_DRAM_VE>;
+ clock-names = "ahb", "mod", "ram";
+
+ resets = <&ccu RST_VE>;
+ interrupts = <53>;
+ allwinner,sram = <&ve_sram 1>;
+ };
+
mmc0: mmc at 1c0f000 {
compatible = "allwinner,sun5i-a13-mmc";
reg = <0x01c0f000 0x1000>;
--
2.17.0
^ permalink raw reply related
* [PATCH v4 16/19] media: platform: Add Sunxi-Cedrus VPU decoder driver
From: Paul Kocialkowski @ 2018-06-18 14:58 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20180618145843.14631-1-paul.kocialkowski@bootlin.com>
This introduces the Sunxi-Cedrus VPU driver that supports the VPU found
in Allwinner SoCs, also known as Video Engine. It is implemented through
a v4l2 m2m decoder device and a media device (used for media requests).
So far, it only supports MPEG2 decoding.
Since this VPU is stateless, synchronization with media requests is
required in order to ensure consistency between frame headers that
contain metadata about the frame to process and the raw slice data that
is used to generate the frame.
This driver was made possible thanks to the long-standing effort
carried out by the linux-sunxi community in the interest of reverse
engineering, documenting and implementing support for Allwinner VPU.
Signed-off-by: Paul Kocialkowski <paul.kocialkowski@bootlin.com>
create mode 100644 drivers/media/platform/sunxi/Kconfig
create mode 100644 drivers/media/platform/sunxi/Makefile
create mode 100644 drivers/media/platform/sunxi/cedrus/Kconfig
create mode 100644 drivers/media/platform/sunxi/cedrus/Makefile
create mode 100644 drivers/media/platform/sunxi/cedrus/cedrus.c
create mode 100644 drivers/media/platform/sunxi/cedrus/cedrus.h
create mode 100644 drivers/media/platform/sunxi/cedrus/cedrus_dec.c
create mode 100644 drivers/media/platform/sunxi/cedrus/cedrus_dec.h
create mode 100644 drivers/media/platform/sunxi/cedrus/cedrus_hw.c
create mode 100644 drivers/media/platform/sunxi/cedrus/cedrus_hw.h
create mode 100644 drivers/media/platform/sunxi/cedrus/cedrus_mpeg2.c
create mode 100644 drivers/media/platform/sunxi/cedrus/cedrus_mpeg2.h
create mode 100644 drivers/media/platform/sunxi/cedrus/cedrus_regs.h
create mode 100644 drivers/media/platform/sunxi/cedrus/cedrus_video.c
create mode 100644 drivers/media/platform/sunxi/cedrus/cedrus_video.h
diff --git a/MAINTAINERS b/MAINTAINERS
index 9c125f705f78..b47dee397475 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -656,6 +656,13 @@ L: linux-crypto at vger.kernel.org
S: Maintained
F: drivers/crypto/sunxi-ss/
+ALLWINNER VPU DRIVER
+M: Maxime Ripard <maxime.ripard@bootlin.com>
+M: Paul Kocialkowski <paul.kocialkowski@bootlin.com>
+L: linux-media at vger.kernel.org
+S: Maintained
+F: drivers/media/platform/sunxi/cedrus/
+
ALPHA PORT
M: Richard Henderson <rth@twiddle.net>
M: Ivan Kokshaysky <ink@jurassic.park.msu.ru>
diff --git a/drivers/media/platform/Kconfig b/drivers/media/platform/Kconfig
index c7a1cf8a1b01..26da88bcd87b 100644
--- a/drivers/media/platform/Kconfig
+++ b/drivers/media/platform/Kconfig
@@ -31,6 +31,8 @@ source "drivers/media/platform/davinci/Kconfig"
source "drivers/media/platform/omap/Kconfig"
+source "drivers/media/platform/sunxi/Kconfig"
+
config VIDEO_SH_VOU
tristate "SuperH VOU video output driver"
depends on MEDIA_CAMERA_SUPPORT
diff --git a/drivers/media/platform/Makefile b/drivers/media/platform/Makefile
index 932515df4477..01ac00adcbc4 100644
--- a/drivers/media/platform/Makefile
+++ b/drivers/media/platform/Makefile
@@ -69,6 +69,7 @@ obj-$(CONFIG_VIDEO_ROCKCHIP_RGA) += rockchip/rga/
obj-y += omap/
obj-$(CONFIG_VIDEO_AM437X_VPFE) += am437x/
+obj-$(CONFIG_VIDEO_SUNXI) += sunxi/
obj-$(CONFIG_VIDEO_XILINX) += xilinx/
diff --git a/drivers/media/platform/sunxi/Kconfig b/drivers/media/platform/sunxi/Kconfig
new file mode 100644
index 000000000000..a639b0949826
--- /dev/null
+++ b/drivers/media/platform/sunxi/Kconfig
@@ -0,0 +1,15 @@
+config VIDEO_SUNXI
+ bool "Allwinner sunXi family Video Devices"
+ depends on ARCH_SUNXI
+ help
+ If you have an Allwinner SoC based on the sunXi family, say Y.
+
+ Note that this option doesn't include new drivers in the
+ kernel: saying N will just cause Kconfig to skip all the
+ questions about Allwinner media devices.
+
+if VIDEO_SUNXI
+
+source "drivers/media/platform/sunxi/cedrus/Kconfig"
+
+endif
diff --git a/drivers/media/platform/sunxi/Makefile b/drivers/media/platform/sunxi/Makefile
new file mode 100644
index 000000000000..cee2846c3ecf
--- /dev/null
+++ b/drivers/media/platform/sunxi/Makefile
@@ -0,0 +1 @@
+obj-$(CONFIG_VIDEO_SUNXI_CEDRUS) += cedrus/
diff --git a/drivers/media/platform/sunxi/cedrus/Kconfig b/drivers/media/platform/sunxi/cedrus/Kconfig
new file mode 100644
index 000000000000..870a4b64a45c
--- /dev/null
+++ b/drivers/media/platform/sunxi/cedrus/Kconfig
@@ -0,0 +1,13 @@
+config VIDEO_SUNXI_CEDRUS
+ tristate "Allwinner Cedrus VPU driver"
+ depends on VIDEO_DEV && VIDEO_V4L2 && MEDIA_CONTROLLER
+ depends on HAS_DMA
+ select VIDEOBUF2_DMA_CONTIG
+ select MEDIA_REQUEST_API
+ select V4L2_MEM2MEM_DEV
+ help
+ Support for the VPU found in Allwinner SoCs, also known as the Cedar
+ video engine.
+
+ To compile this driver as a module, choose M here: the module
+ will be called cedrus.
diff --git a/drivers/media/platform/sunxi/cedrus/Makefile b/drivers/media/platform/sunxi/cedrus/Makefile
new file mode 100644
index 000000000000..632a0be90ed7
--- /dev/null
+++ b/drivers/media/platform/sunxi/cedrus/Makefile
@@ -0,0 +1,3 @@
+obj-$(CONFIG_VIDEO_SUNXI_CEDRUS) += cedrus.o
+
+cedrus-y = cedrus.o cedrus_video.o cedrus_hw.o cedrus_dec.o cedrus_mpeg2.o
diff --git a/drivers/media/platform/sunxi/cedrus/cedrus.c b/drivers/media/platform/sunxi/cedrus/cedrus.c
new file mode 100644
index 000000000000..1718db1b549b
--- /dev/null
+++ b/drivers/media/platform/sunxi/cedrus/cedrus.c
@@ -0,0 +1,327 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Sunxi-Cedrus VPU driver
+ *
+ * Copyright (C) 2018 Paul Kocialkowski <paul.kocialkowski@bootlin.com>
+ * Copyright (C) 2016 Florent Revest <florent.revest@free-electrons.com>
+ *
+ * Based on the vim2m driver, that is:
+ *
+ * Copyright (c) 2009-2010 Samsung Electronics Co., Ltd.
+ * Pawel Osciak, <pawel@osciak.com>
+ * Marek Szyprowski, <m.szyprowski@samsung.com>
+ */
+
+#include <linux/platform_device.h>
+#include <linux/module.h>
+#include <linux/of.h>
+
+#include <media/videobuf2-dma-contig.h>
+#include <media/v4l2-device.h>
+#include <media/v4l2-ioctl.h>
+#include <media/v4l2-ctrls.h>
+#include <media/v4l2-mem2mem.h>
+
+#include "cedrus.h"
+#include "cedrus_video.h"
+#include "cedrus_dec.h"
+#include "cedrus_hw.h"
+
+static int cedrus_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+ struct cedrus_ctx *ctx =
+ container_of(ctrl->handler, struct cedrus_ctx, hdl);
+ struct cedrus_dev *dev = ctx->dev;
+
+ switch (ctrl->id) {
+ case V4L2_CID_MPEG_VIDEO_MPEG2_SLICE_HEADER:
+ /* This is kept in memory and used directly. */
+ break;
+ default:
+ v4l2_err(&dev->v4l2_dev, "Invalid control to set\n");
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static const struct v4l2_ctrl_ops cedrus_ctrl_ops = {
+ .s_ctrl = cedrus_s_ctrl,
+};
+
+static const struct cedrus_control controls[] = {
+ [CEDRUS_CTRL_DEC_MPEG2_SLICE_HEADER] = {
+ .id = V4L2_CID_MPEG_VIDEO_MPEG2_SLICE_HEADER,
+ .elem_size = sizeof(struct v4l2_ctrl_mpeg2_slice_header),
+ },
+};
+
+static int cedrus_init_ctrls(struct cedrus_dev *dev, struct cedrus_ctx *ctx)
+{
+ struct v4l2_ctrl_handler *hdl = &ctx->hdl;
+ unsigned int num_ctrls = ARRAY_SIZE(controls);
+ unsigned int i;
+
+ v4l2_ctrl_handler_init(hdl, num_ctrls);
+ if (hdl->error) {
+ v4l2_err(&dev->v4l2_dev,
+ "Failed to initialize control handler\n");
+ return hdl->error;
+ }
+
+ for (i = 0; i < num_ctrls; i++) {
+ struct v4l2_ctrl_config cfg = { 0 };
+
+ cfg.ops = &cedrus_ctrl_ops;
+ cfg.elem_size = controls[i].elem_size;
+ cfg.id = controls[i].id;
+
+ ctx->ctrls[i] = v4l2_ctrl_new_custom(hdl, &cfg, NULL);
+ if (hdl->error) {
+ v4l2_err(&dev->v4l2_dev,
+ "Failed to create new custom control\n");
+
+ v4l2_ctrl_handler_free(hdl);
+ return hdl->error;
+ }
+ }
+
+ ctx->fh.ctrl_handler = hdl;
+ v4l2_ctrl_handler_setup(hdl);
+
+ return 0;
+}
+
+static int cedrus_open(struct file *file)
+{
+ struct cedrus_dev *dev = video_drvdata(file);
+ struct cedrus_ctx *ctx = NULL;
+ int ret;
+
+ if (mutex_lock_interruptible(&dev->dev_mutex))
+ return -ERESTARTSYS;
+
+ ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
+ if (!ctx) {
+ mutex_unlock(&dev->dev_mutex);
+ return -ENOMEM;
+ }
+
+ INIT_WORK(&ctx->run_work, cedrus_device_work);
+
+ INIT_LIST_HEAD(&ctx->src_list);
+ INIT_LIST_HEAD(&ctx->dst_list);
+
+ v4l2_fh_init(&ctx->fh, video_devdata(file));
+ file->private_data = &ctx->fh;
+ ctx->dev = dev;
+
+ ret = cedrus_init_ctrls(dev, ctx);
+ if (ret)
+ goto err_free;
+
+ ctx->fh.m2m_ctx = v4l2_m2m_ctx_init(dev->m2m_dev, ctx,
+ &cedrus_queue_init);
+ if (IS_ERR(ctx->fh.m2m_ctx)) {
+ ret = PTR_ERR(ctx->fh.m2m_ctx);
+ goto err_ctrls;
+ }
+
+ v4l2_fh_add(&ctx->fh);
+
+ mutex_unlock(&dev->dev_mutex);
+
+ return 0;
+
+err_ctrls:
+ v4l2_ctrl_handler_free(&ctx->hdl);
+err_free:
+ kfree(ctx);
+ mutex_unlock(&dev->dev_mutex);
+
+ return ret;
+}
+
+static int cedrus_release(struct file *file)
+{
+ struct cedrus_dev *dev = video_drvdata(file);
+ struct cedrus_ctx *ctx = container_of(file->private_data,
+ struct cedrus_ctx, fh);
+
+ mutex_lock(&dev->dev_mutex);
+
+ v4l2_fh_del(&ctx->fh);
+ v4l2_m2m_ctx_release(ctx->fh.m2m_ctx);
+
+ v4l2_ctrl_handler_free(&ctx->hdl);
+
+ v4l2_fh_exit(&ctx->fh);
+ v4l2_fh_exit(&ctx->fh);
+
+ kfree(ctx);
+
+ mutex_unlock(&dev->dev_mutex);
+
+ return 0;
+}
+
+static const struct v4l2_file_operations cedrus_fops = {
+ .owner = THIS_MODULE,
+ .open = cedrus_open,
+ .release = cedrus_release,
+ .poll = v4l2_m2m_fop_poll,
+ .unlocked_ioctl = video_ioctl2,
+ .mmap = v4l2_m2m_fop_mmap,
+};
+
+static const struct video_device cedrus_video_device = {
+ .name = CEDRUS_NAME,
+ .vfl_dir = VFL_DIR_M2M,
+ .fops = &cedrus_fops,
+ .ioctl_ops = &cedrus_ioctl_ops,
+ .minor = -1,
+ .release = video_device_release_empty,
+};
+
+static const struct v4l2_m2m_ops cedrus_m2m_ops = {
+ .device_run = cedrus_device_run,
+ .job_abort = cedrus_job_abort,
+};
+
+static const struct media_device_ops cedrus_m2m_media_ops = {
+ .req_validate = vb2_request_validate,
+ .req_queue = vb2_m2m_request_queue,
+};
+
+static int cedrus_probe(struct platform_device *pdev)
+{
+ struct cedrus_dev *dev;
+ struct video_device *vfd;
+ int ret;
+
+ dev = devm_kzalloc(&pdev->dev, sizeof(*dev), GFP_KERNEL);
+ if (!dev)
+ return -ENOMEM;
+
+ dev->dev = &pdev->dev;
+ dev->pdev = pdev;
+
+ ret = cedrus_hw_probe(dev);
+ if (ret) {
+ dev_err(&pdev->dev, "Failed to probe hardware\n");
+ return ret;
+ }
+
+ mutex_init(&dev->dev_mutex);
+ spin_lock_init(&dev->irq_lock);
+
+ dev->vfd = cedrus_video_device;
+ vfd = &dev->vfd;
+ vfd->lock = &dev->dev_mutex;
+ vfd->v4l2_dev = &dev->v4l2_dev;
+
+ dev->mdev.dev = &pdev->dev;
+ strlcpy(dev->mdev.model, CEDRUS_NAME, sizeof(dev->mdev.model));
+
+ media_device_init(&dev->mdev);
+ dev->mdev.ops = &cedrus_m2m_media_ops;
+ dev->v4l2_dev.mdev = &dev->mdev;
+ dev->pad[0].flags = MEDIA_PAD_FL_SINK;
+ dev->pad[1].flags = MEDIA_PAD_FL_SOURCE;
+
+ ret = media_entity_pads_init(&vfd->entity, 2, dev->pad);
+ if (ret) {
+ dev_err(&pdev->dev, "Failed to initialize media entity pads\n");
+ return ret;
+ }
+
+ ret = v4l2_device_register(&pdev->dev, &dev->v4l2_dev);
+ if (ret) {
+ dev_err(&pdev->dev, "Failed to register V4L2 device\n");
+ return ret;
+ }
+
+ ret = video_register_device(vfd, VFL_TYPE_GRABBER, 0);
+ if (ret) {
+ v4l2_err(&dev->v4l2_dev, "Failed to register video device\n");
+ goto err_v4l2;
+ }
+
+ video_set_drvdata(vfd, dev);
+ snprintf(vfd->name, sizeof(vfd->name), "%s", cedrus_video_device.name);
+
+ v4l2_info(&dev->v4l2_dev,
+ "Device registered as /dev/video%d\n", vfd->num);
+
+ platform_set_drvdata(pdev, dev);
+
+ dev->m2m_dev = v4l2_m2m_init(&cedrus_m2m_ops);
+ if (IS_ERR(dev->m2m_dev)) {
+ v4l2_err(&dev->v4l2_dev,
+ "Failed to initialize V4L2 M2M device\n");
+ ret = PTR_ERR(dev->m2m_dev);
+ goto err_video;
+ }
+
+ ret = media_device_register(&dev->mdev);
+ if (ret) {
+ v4l2_err(&dev->v4l2_dev, "Failed to register media device\n");
+ goto err_m2m;
+ }
+
+ return 0;
+
+err_m2m:
+ v4l2_m2m_release(dev->m2m_dev);
+err_video:
+ video_unregister_device(&dev->vfd);
+err_v4l2:
+ v4l2_device_unregister(&dev->v4l2_dev);
+
+ return ret;
+}
+
+static int cedrus_remove(struct platform_device *pdev)
+{
+ struct cedrus_dev *dev = platform_get_drvdata(pdev);
+
+ v4l2_info(&dev->v4l2_dev, "Removing " CEDRUS_NAME);
+
+ if (media_devnode_is_registered(dev->mdev.devnode)) {
+ media_device_unregister(&dev->mdev);
+ media_device_cleanup(&dev->mdev);
+ }
+
+ v4l2_m2m_release(dev->m2m_dev);
+ video_unregister_device(&dev->vfd);
+ v4l2_device_unregister(&dev->v4l2_dev);
+ cedrus_hw_remove(dev);
+
+ return 0;
+}
+
+#ifdef CONFIG_OF
+static const struct of_device_id of_cedrus_match[] = {
+ { .compatible = "allwinner,sun4i-a10-video-engine" },
+ { .compatible = "allwinner,sun5i-a13-video-engine" },
+ { .compatible = "allwinner,sun7i-a20-video-engine" },
+ { .compatible = "allwinner,sun8i-a33-video-engine" },
+ { /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, of_cedrus_match);
+#endif
+
+static struct platform_driver cedrus_driver = {
+ .probe = cedrus_probe,
+ .remove = cedrus_remove,
+ .driver = {
+ .name = CEDRUS_NAME,
+ .owner = THIS_MODULE,
+ .of_match_table = of_match_ptr(of_cedrus_match),
+ },
+};
+module_platform_driver(cedrus_driver);
+
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR("Florent Revest <florent.revest@free-electrons.com>");
+MODULE_DESCRIPTION("Sunxi-Cedrus VPU driver");
diff --git a/drivers/media/platform/sunxi/cedrus/cedrus.h b/drivers/media/platform/sunxi/cedrus/cedrus.h
new file mode 100644
index 000000000000..106bda44a3c3
--- /dev/null
+++ b/drivers/media/platform/sunxi/cedrus/cedrus.h
@@ -0,0 +1,117 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Sunxi-Cedrus VPU driver
+ *
+ * Copyright (C) 2018 Paul Kocialkowski <paul.kocialkowski@bootlin.com>
+ * Copyright (C) 2016 Florent Revest <florent.revest@free-electrons.com>
+ *
+ * Based on the vim2m driver, that is:
+ *
+ * Copyright (c) 2009-2010 Samsung Electronics Co., Ltd.
+ * Pawel Osciak, <pawel@osciak.com>
+ * Marek Szyprowski, <m.szyprowski@samsung.com>
+ */
+
+#ifndef _CEDRUS_H_
+#define _CEDRUS_H_
+
+#include <linux/platform_device.h>
+
+#include <media/videobuf2-v4l2.h>
+#include <media/v4l2-device.h>
+#include <media/v4l2-ctrls.h>
+
+#define CEDRUS_NAME "cedrus"
+
+enum cedrus_control_id {
+ CEDRUS_CTRL_DEC_MPEG2_SLICE_HEADER = 0,
+ CEDRUS_CTRL_MAX,
+};
+
+struct cedrus_control {
+ u32 id;
+ u32 elem_size;
+};
+
+struct cedrus_fmt {
+ u32 fourcc;
+ int depth;
+ u32 types;
+ unsigned int num_planes;
+};
+
+struct cedrus_mpeg2_run {
+ const struct v4l2_ctrl_mpeg2_slice_header *hdr;
+};
+
+struct cedrus_run {
+ struct vb2_v4l2_buffer *src;
+ struct vb2_v4l2_buffer *dst;
+
+ union {
+ struct cedrus_mpeg2_run mpeg2;
+ };
+};
+
+struct cedrus_ctx {
+ struct v4l2_fh fh;
+ struct cedrus_dev *dev;
+
+ struct cedrus_fmt *vpu_src_fmt;
+ struct v4l2_pix_format_mplane src_fmt;
+ struct cedrus_fmt *vpu_dst_fmt;
+ struct v4l2_pix_format_mplane dst_fmt;
+
+ struct v4l2_ctrl_handler hdl;
+ struct v4l2_ctrl *ctrls[CEDRUS_CTRL_MAX];
+
+ struct vb2_buffer *dst_bufs[VIDEO_MAX_FRAME];
+
+ int job_abort;
+
+ struct work_struct try_schedule_work;
+ struct work_struct run_work;
+ struct list_head src_list;
+ struct list_head dst_list;
+};
+
+struct cedrus_buffer {
+ struct vb2_v4l2_buffer vb;
+ enum vb2_buffer_state state;
+ struct list_head list;
+};
+
+struct cedrus_dev {
+ struct v4l2_device v4l2_dev;
+ struct video_device vfd;
+ struct media_device mdev;
+ struct media_pad pad[2];
+ struct platform_device *pdev;
+ struct device *dev;
+ struct v4l2_m2m_dev *m2m_dev;
+
+ /* Device file mutex */
+ struct mutex dev_mutex;
+ /* Interrupt spinlock */
+ spinlock_t irq_lock;
+
+ void __iomem *base;
+
+ struct clk *mod_clk;
+ struct clk *ahb_clk;
+ struct clk *ram_clk;
+
+ struct reset_control *rstc;
+};
+
+static inline void cedrus_write(struct cedrus_dev *dev, u32 reg, u32 val)
+{
+ writel(val, dev->base + reg);
+}
+
+static inline u32 cedrus_read(struct cedrus_dev *dev, u32 reg)
+{
+ return readl(dev->base + reg);
+}
+
+#endif
diff --git a/drivers/media/platform/sunxi/cedrus/cedrus_dec.c b/drivers/media/platform/sunxi/cedrus/cedrus_dec.c
new file mode 100644
index 000000000000..bd9727ae9f63
--- /dev/null
+++ b/drivers/media/platform/sunxi/cedrus/cedrus_dec.c
@@ -0,0 +1,170 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Sunxi-Cedrus VPU driver
+ *
+ * Copyright (C) 2018 Paul Kocialkowski <paul.kocialkowski@bootlin.com>
+ * Copyright (C) 2016 Florent Revest <florent.revest@free-electrons.com>
+ *
+ * Based on the vim2m driver, that is:
+ *
+ * Copyright (c) 2009-2010 Samsung Electronics Co., Ltd.
+ * Pawel Osciak, <pawel@osciak.com>
+ * Marek Szyprowski, <m.szyprowski@samsung.com>
+ */
+
+#include <media/videobuf2-dma-contig.h>
+#include <media/v4l2-device.h>
+#include <media/v4l2-ioctl.h>
+#include <media/v4l2-ctrls.h>
+#include <media/v4l2-event.h>
+#include <media/v4l2-mem2mem.h>
+
+#include "cedrus.h"
+#include "cedrus_mpeg2.h"
+#include "cedrus_dec.h"
+#include "cedrus_hw.h"
+
+static inline void *get_ctrl_ptr(struct cedrus_ctx *ctx,
+ enum cedrus_control_id id)
+{
+ struct v4l2_ctrl *ctrl = ctx->ctrls[id];
+
+ return ctrl->p_cur.p;
+}
+
+void cedrus_device_work(struct work_struct *work)
+{
+ struct cedrus_ctx *ctx = container_of(work,
+ struct cedrus_ctx, run_work);
+ struct cedrus_dev *dev = ctx->dev;
+ struct cedrus_buffer *buffer_entry;
+ struct vb2_v4l2_buffer *src_buf, *dst_buf;
+ unsigned long flags;
+
+ spin_lock_irqsave(&ctx->dev->irq_lock, flags);
+
+ if (list_empty(&ctx->src_list) || list_empty(&ctx->dst_list)) {
+ v4l2_err(&dev->v4l2_dev,
+ "Empty source and/or destination buffer lists\n");
+ spin_unlock_irqrestore(&ctx->dev->irq_lock, flags);
+ return;
+ }
+
+ buffer_entry = list_last_entry(&ctx->src_list, struct cedrus_buffer, list);
+ list_del(ctx->src_list.prev);
+
+ src_buf = &buffer_entry->vb;
+ v4l2_m2m_buf_done(src_buf, buffer_entry->state);
+
+ buffer_entry = list_last_entry(&ctx->dst_list, struct cedrus_buffer, list);
+ list_del(ctx->dst_list.prev);
+
+ dst_buf = &buffer_entry->vb;
+ v4l2_m2m_buf_done(dst_buf, buffer_entry->state);
+
+ spin_unlock_irqrestore(&ctx->dev->irq_lock, flags);
+
+ v4l2_m2m_job_finish(ctx->dev->m2m_dev, ctx->fh.m2m_ctx);
+}
+
+void cedrus_device_run(void *priv)
+{
+ struct cedrus_ctx *ctx = priv;
+ struct cedrus_dev *dev = ctx->dev;
+ struct cedrus_run run = { 0 };
+ struct media_request *src_req;
+ unsigned long flags;
+
+ run.src = v4l2_m2m_next_src_buf(ctx->fh.m2m_ctx);
+ if (!run.src) {
+ v4l2_err(&dev->v4l2_dev, "No source buffer to prepare\n");
+ return;
+ }
+
+ run.dst = v4l2_m2m_next_dst_buf(ctx->fh.m2m_ctx);
+ if (!run.dst) {
+ v4l2_err(&dev->v4l2_dev, "No destination buffer to prepare\n");
+ return;
+ }
+
+ /* Apply request(s) controls if needed. */
+ src_req = run.src->vb2_buf.req_obj.req;
+
+ if (src_req)
+ v4l2_ctrl_request_setup(src_req, &ctx->hdl);
+
+ ctx->job_abort = 0;
+
+ spin_lock_irqsave(&ctx->dev->irq_lock, flags);
+
+ switch (ctx->vpu_src_fmt->fourcc) {
+ case V4L2_PIX_FMT_MPEG2_SLICE:
+ if (!ctx->ctrls[CEDRUS_CTRL_DEC_MPEG2_SLICE_HEADER]) {
+ v4l2_err(&dev->v4l2_dev,
+ "Invalid MPEG2 frame header control\n");
+ ctx->job_abort = 1;
+ goto unlock_complete;
+ }
+
+ run.mpeg2.hdr = get_ctrl_ptr(ctx, CEDRUS_CTRL_DEC_MPEG2_SLICE_HEADER);
+ cedrus_mpeg2_setup(ctx, &run);
+ break;
+
+ default:
+ ctx->job_abort = 1;
+ }
+
+unlock_complete:
+ spin_unlock_irqrestore(&ctx->dev->irq_lock, flags);
+
+ /* Complete request(s) controls if needed. */
+
+ if (src_req)
+ v4l2_ctrl_request_complete(src_req, &ctx->hdl);
+
+ spin_lock_irqsave(&ctx->dev->irq_lock, flags);
+
+ if (!ctx->job_abort) {
+ if (ctx->vpu_src_fmt->fourcc == V4L2_PIX_FMT_MPEG2_SLICE)
+ cedrus_mpeg2_trigger(ctx);
+ } else {
+ v4l2_m2m_buf_done(run.src, VB2_BUF_STATE_ERROR);
+ v4l2_m2m_buf_done(run.dst, VB2_BUF_STATE_ERROR);
+ }
+
+ spin_unlock_irqrestore(&ctx->dev->irq_lock, flags);
+
+ if (ctx->job_abort)
+ v4l2_m2m_job_finish(ctx->dev->m2m_dev, ctx->fh.m2m_ctx);
+}
+
+void cedrus_job_abort(void *priv)
+{
+ struct cedrus_ctx *ctx = priv;
+ struct vb2_v4l2_buffer *src_buf, *dst_buf;
+ unsigned long flags;
+
+ ctx->job_abort = 1;
+
+ /*
+ * V4L2 M2M and request API cleanup is done here while hardware state
+ * cleanup is done in the interrupt context. Doing all the cleanup in
+ * the interrupt context is a bit risky, since the job_abort call might
+ * originate from the release hook, where interrupts have already been
+ * disabled.
+ */
+
+ spin_lock_irqsave(&ctx->dev->irq_lock, flags);
+
+ src_buf = v4l2_m2m_src_buf_remove(ctx->fh.m2m_ctx);
+ if (src_buf)
+ v4l2_m2m_buf_done(src_buf, VB2_BUF_STATE_ERROR);
+
+ dst_buf = v4l2_m2m_dst_buf_remove(ctx->fh.m2m_ctx);
+ if (dst_buf)
+ v4l2_m2m_buf_done(dst_buf, VB2_BUF_STATE_ERROR);
+
+ spin_unlock_irqrestore(&ctx->dev->irq_lock, flags);
+
+ v4l2_m2m_job_finish(ctx->dev->m2m_dev, ctx->fh.m2m_ctx);
+}
diff --git a/drivers/media/platform/sunxi/cedrus/cedrus_dec.h b/drivers/media/platform/sunxi/cedrus/cedrus_dec.h
new file mode 100644
index 000000000000..b38812136504
--- /dev/null
+++ b/drivers/media/platform/sunxi/cedrus/cedrus_dec.h
@@ -0,0 +1,27 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Sunxi-Cedrus VPU driver
+ *
+ * Copyright (C) 2018 Paul Kocialkowski <paul.kocialkowski@bootlin.com>
+ * Copyright (C) 2016 Florent Revest <florent.revest@free-electrons.com>
+ *
+ * Based on the vim2m driver, that is:
+ *
+ * Copyright (c) 2009-2010 Samsung Electronics Co., Ltd.
+ * Pawel Osciak, <pawel@osciak.com>
+ * Marek Szyprowski, <m.szyprowski@samsung.com>
+ */
+
+#ifndef _CEDRUS_DEC_H_
+#define _CEDRUS_DEC_H_
+
+extern const struct v4l2_ioctl_ops cedrus_ioctl_ops;
+
+void cedrus_device_work(struct work_struct *work);
+void cedrus_device_run(void *priv);
+void cedrus_job_abort(void *priv);
+
+int cedrus_queue_init(void *priv, struct vb2_queue *src_vq,
+ struct vb2_queue *dst_vq);
+
+#endif
diff --git a/drivers/media/platform/sunxi/cedrus/cedrus_hw.c b/drivers/media/platform/sunxi/cedrus/cedrus_hw.c
new file mode 100644
index 000000000000..9ee1380f4e30
--- /dev/null
+++ b/drivers/media/platform/sunxi/cedrus/cedrus_hw.c
@@ -0,0 +1,262 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Sunxi-Cedrus VPU driver
+ *
+ * Copyright (C) 2018 Paul Kocialkowski <paul.kocialkowski@bootlin.com>
+ * Copyright (C) 2016 Florent Revest <florent.revest@free-electrons.com>
+ *
+ * Based on the vim2m driver, that is:
+ *
+ * Copyright (c) 2009-2010 Samsung Electronics Co., Ltd.
+ * Pawel Osciak, <pawel@osciak.com>
+ * Marek Szyprowski, <m.szyprowski@samsung.com>
+ */
+
+#include <linux/platform_device.h>
+#include <linux/of_reserved_mem.h>
+#include <linux/dma-mapping.h>
+#include <linux/interrupt.h>
+#include <linux/clk.h>
+#include <linux/regmap.h>
+#include <linux/reset.h>
+#include <linux/soc/sunxi/sunxi_sram.h>
+
+#include <media/videobuf2-core.h>
+#include <media/v4l2-mem2mem.h>
+
+#include "cedrus.h"
+#include "cedrus_hw.h"
+#include "cedrus_regs.h"
+
+int cedrus_engine_enable(struct cedrus_dev *dev, enum cedrus_engine engine)
+{
+ u32 reg = 0;
+
+ /*
+ * FIXME: This is only valid on 32-bits DDR's, we should test
+ * it on the A13/A33.
+ */
+ reg |= VE_CTRL_REC_WR_MODE_2MB;
+
+ reg |= VE_CTRL_CACHE_BUS_BW_128;
+
+ switch (engine) {
+ case CEDRUS_ENGINE_MPEG:
+ reg |= VE_CTRL_DEC_MODE_MPEG;
+ break;
+
+ default:
+ return -EINVAL;
+ }
+
+ cedrus_write(dev, VE_CTRL, reg);
+
+ return 0;
+}
+
+void cedrus_engine_disable(struct cedrus_dev *dev)
+{
+ cedrus_write(dev, VE_CTRL, VE_CTRL_DEC_MODE_DISABLED);
+}
+
+static irqreturn_t cedrus_ve_irq(int irq, void *data)
+{
+ struct cedrus_dev *dev = data;
+ struct cedrus_ctx *ctx;
+ struct cedrus_buffer *src_buffer, *dst_buffer;
+ struct vb2_v4l2_buffer *src_vb, *dst_vb;
+ unsigned long flags;
+ unsigned int value, status;
+
+ spin_lock_irqsave(&dev->irq_lock, flags);
+
+ /* Disable MPEG interrupts and stop the MPEG engine. */
+ value = cedrus_read(dev, VE_MPEG_CTRL);
+ cedrus_write(dev, VE_MPEG_CTRL, value & (~0xf));
+
+ status = cedrus_read(dev, VE_MPEG_STATUS);
+ cedrus_write(dev, VE_MPEG_STATUS, 0x0000c00f);
+ cedrus_engine_disable(dev);
+
+ ctx = v4l2_m2m_get_curr_priv(dev->m2m_dev);
+ if (!ctx) {
+ v4l2_err(&dev->v4l2_dev,
+ "Instance released before the end of transaction\n");
+ spin_unlock_irqrestore(&dev->irq_lock, flags);
+
+ return IRQ_HANDLED;
+ }
+
+ src_vb = v4l2_m2m_src_buf_remove(ctx->fh.m2m_ctx);
+ dst_vb = v4l2_m2m_dst_buf_remove(ctx->fh.m2m_ctx);
+
+ if (!src_vb || !dst_vb) {
+ v4l2_err(&dev->v4l2_dev,
+ "Missing source and/or destination buffers\n");
+ spin_unlock_irqrestore(&dev->irq_lock, flags);
+
+ return IRQ_HANDLED;
+ }
+
+ src_buffer = container_of(src_vb, struct cedrus_buffer, vb);
+ dst_buffer = container_of(dst_vb, struct cedrus_buffer, vb);
+
+ /* First bit of MPEG_STATUS indicates success. */
+ if (ctx->job_abort || !(status & 0x01))
+ src_buffer->state = dst_buffer->state = VB2_BUF_STATE_ERROR;
+ else
+ src_buffer->state = dst_buffer->state = VB2_BUF_STATE_DONE;
+
+ list_add_tail(&src_buffer->list, &ctx->src_list);
+ list_add_tail(&dst_buffer->list, &ctx->dst_list);
+
+ spin_unlock_irqrestore(&dev->irq_lock, flags);
+
+ schedule_work(&ctx->run_work);
+
+ return IRQ_HANDLED;
+}
+
+int cedrus_hw_probe(struct cedrus_dev *dev)
+{
+ struct resource *res;
+ int irq_dec;
+ int ret;
+
+ irq_dec = platform_get_irq(dev->pdev, 0);
+ if (irq_dec <= 0) {
+ v4l2_err(&dev->v4l2_dev, "Failed to get IRQ\n");
+ return -ENXIO;
+ }
+ ret = devm_request_irq(dev->dev, irq_dec, cedrus_ve_irq, 0,
+ dev_name(dev->dev), dev);
+ if (ret) {
+ v4l2_err(&dev->v4l2_dev, "Failed to request IRQ\n");
+ return -ENXIO;
+ }
+
+ /*
+ * The VPU is only able to handle bus addresses so we have to subtract
+ * the RAM offset to the physcal addresses.
+ */
+ dev->dev->dma_pfn_offset = PHYS_PFN_OFFSET;
+
+ ret = of_reserved_mem_device_init(dev->dev);
+ if (ret && ret != -ENODEV) {
+ v4l2_err(&dev->v4l2_dev, "Failed to reserved memory\n");
+ return -ENODEV;
+ }
+
+ ret = sunxi_sram_claim(dev->dev);
+ if (ret) {
+ v4l2_err(&dev->v4l2_dev, "Failed to claim SRAM\n");
+ goto err_mem;
+ }
+
+ dev->ahb_clk = devm_clk_get(dev->dev, "ahb");
+ if (IS_ERR(dev->ahb_clk)) {
+ v4l2_err(&dev->v4l2_dev, "Failed to get AHB clock\n");
+
+ ret = PTR_ERR(dev->ahb_clk);
+ goto err_sram;
+ }
+
+ dev->mod_clk = devm_clk_get(dev->dev, "mod");
+ if (IS_ERR(dev->mod_clk)) {
+ v4l2_err(&dev->v4l2_dev, "Failed to get MOD clock\n");
+
+ ret = PTR_ERR(dev->mod_clk);
+ goto err_sram;
+ }
+
+ dev->ram_clk = devm_clk_get(dev->dev, "ram");
+ if (IS_ERR(dev->ram_clk)) {
+ v4l2_err(&dev->v4l2_dev, "Failed to get RAM clock\n");
+
+ ret = PTR_ERR(dev->ram_clk);
+ goto err_sram;
+ }
+
+ dev->rstc = devm_reset_control_get(dev->dev, NULL);
+ if (IS_ERR(dev->rstc)) {
+ v4l2_err(&dev->v4l2_dev, "Failed to get reset control\n");
+
+ ret = PTR_ERR(dev->rstc);
+ goto err_sram;
+ }
+
+ res = platform_get_resource(dev->pdev, IORESOURCE_MEM, 0);
+ dev->base = devm_ioremap_resource(dev->dev, res);
+ if (!dev->base) {
+ v4l2_err(&dev->v4l2_dev, "Failed to map registers\n");
+
+ ret = -EFAULT;
+ goto err_sram;
+ }
+
+ ret = clk_set_rate(dev->mod_clk, CEDRUS_CLOCK_RATE_DEFAULT);
+ if (ret) {
+ v4l2_err(&dev->v4l2_dev, "Failed to set clock rate\n");
+ goto err_sram;
+ }
+
+ ret = clk_prepare_enable(dev->ahb_clk);
+ if (ret) {
+ v4l2_err(&dev->v4l2_dev, "Failed to enable AHB clock\n");
+
+ ret = -EFAULT;
+ goto err_sram;
+ }
+
+ ret = clk_prepare_enable(dev->mod_clk);
+ if (ret) {
+ v4l2_err(&dev->v4l2_dev, "Failed to enable MOD clock\n");
+
+ ret = -EFAULT;
+ goto err_ahb_clk;
+ }
+
+ ret = clk_prepare_enable(dev->ram_clk);
+ if (ret) {
+ v4l2_err(&dev->v4l2_dev, "Failed to enable RAM clock\n");
+
+ ret = -EFAULT;
+ goto err_mod_clk;
+ }
+
+ ret = reset_control_reset(dev->rstc);
+ if (ret) {
+ v4l2_err(&dev->v4l2_dev, "Failed to apply reset\n");
+
+ ret = -EFAULT;
+ goto err_ram_clk;
+ }
+
+ return 0;
+
+err_ram_clk:
+ clk_disable_unprepare(dev->ram_clk);
+err_mod_clk:
+ clk_disable_unprepare(dev->mod_clk);
+err_ahb_clk:
+ clk_disable_unprepare(dev->ahb_clk);
+err_sram:
+ sunxi_sram_release(dev->dev);
+err_mem:
+ of_reserved_mem_device_release(dev->dev);
+
+ return ret;
+}
+
+void cedrus_hw_remove(struct cedrus_dev *dev)
+{
+ reset_control_assert(dev->rstc);
+
+ clk_disable_unprepare(dev->ram_clk);
+ clk_disable_unprepare(dev->mod_clk);
+ clk_disable_unprepare(dev->ahb_clk);
+
+ sunxi_sram_release(dev->dev);
+
+ of_reserved_mem_device_release(dev->dev);
+}
diff --git a/drivers/media/platform/sunxi/cedrus/cedrus_hw.h b/drivers/media/platform/sunxi/cedrus/cedrus_hw.h
new file mode 100644
index 000000000000..ead4f5089881
--- /dev/null
+++ b/drivers/media/platform/sunxi/cedrus/cedrus_hw.h
@@ -0,0 +1,30 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Sunxi-Cedrus VPU driver
+ *
+ * Copyright (C) 2018 Paul Kocialkowski <paul.kocialkowski@bootlin.com>
+ * Copyright (C) 2016 Florent Revest <florent.revest@free-electrons.com>
+ *
+ * Based on the vim2m driver, that is:
+ *
+ * Copyright (c) 2009-2010 Samsung Electronics Co., Ltd.
+ * Pawel Osciak, <pawel@osciak.com>
+ * Marek Szyprowski, <m.szyprowski@samsung.com>
+ */
+
+#ifndef _CEDRUS_HW_H_
+#define _CEDRUS_HW_H_
+
+#define CEDRUS_CLOCK_RATE_DEFAULT 320000000
+
+enum cedrus_engine {
+ CEDRUS_ENGINE_MPEG,
+};
+
+int cedrus_engine_enable(struct cedrus_dev *dev, enum cedrus_engine engine);
+void cedrus_engine_disable(struct cedrus_dev *dev);
+
+int cedrus_hw_probe(struct cedrus_dev *dev);
+void cedrus_hw_remove(struct cedrus_dev *dev);
+
+#endif
diff --git a/drivers/media/platform/sunxi/cedrus/cedrus_mpeg2.c b/drivers/media/platform/sunxi/cedrus/cedrus_mpeg2.c
new file mode 100644
index 000000000000..0d6dcaec73b3
--- /dev/null
+++ b/drivers/media/platform/sunxi/cedrus/cedrus_mpeg2.c
@@ -0,0 +1,146 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Sunxi-Cedrus VPU driver
+ *
+ * Copyright (C) 2018 Paul Kocialkowski <paul.kocialkowski@bootlin.com>
+ * Copyright (C) 2016 Florent Revest <florent.revest@free-electrons.com>
+ *
+ * Based on the vim2m driver, that is:
+ *
+ * Copyright (c) 2009-2010 Samsung Electronics Co., Ltd.
+ * Pawel Osciak, <pawel@osciak.com>
+ * Marek Szyprowski, <m.szyprowski@samsung.com>
+ */
+
+#include <media/videobuf2-dma-contig.h>
+
+#include "cedrus.h"
+#include "cedrus_hw.h"
+#include "cedrus_regs.h"
+
+static const u8 mpeg_default_intra_quant[64] = {
+ 8, 16, 16, 19, 16, 19, 22, 22,
+ 22, 22, 22, 22, 26, 24, 26, 27,
+ 27, 27, 26, 26, 26, 26, 27, 27,
+ 27, 29, 29, 29, 34, 34, 34, 29,
+ 29, 29, 27, 27, 29, 29, 32, 32,
+ 34, 34, 37, 38, 37, 35, 35, 34,
+ 35, 38, 38, 40, 40, 40, 48, 48,
+ 46, 46, 56, 56, 58, 69, 69, 83
+};
+
+#define m_iq(i) (((64 + i) << 8) | mpeg_default_intra_quant[i])
+
+static const u8 mpeg_default_non_intra_quant[64] = {
+ 16, 16, 16, 16, 16, 16, 16, 16,
+ 16, 16, 16, 16, 16, 16, 16, 16,
+ 16, 16, 16, 16, 16, 16, 16, 16,
+ 16, 16, 16, 16, 16, 16, 16, 16,
+ 16, 16, 16, 16, 16, 16, 16, 16,
+ 16, 16, 16, 16, 16, 16, 16, 16,
+ 16, 16, 16, 16, 16, 16, 16, 16,
+ 16, 16, 16, 16, 16, 16, 16, 16
+};
+
+#define m_niq(i) ((i << 8) | mpeg_default_non_intra_quant[i])
+
+void cedrus_mpeg2_setup(struct cedrus_ctx *ctx, struct cedrus_run *run)
+{
+ struct cedrus_dev *dev = ctx->dev;
+ const struct v4l2_ctrl_mpeg2_slice_header *frame_hdr = run->mpeg2.hdr;
+
+ u16 width = DIV_ROUND_UP(frame_hdr->width, 16);
+ u16 height = DIV_ROUND_UP(frame_hdr->height, 16);
+
+ u32 pic_header = 0;
+ u32 vld_len = frame_hdr->slice_len - frame_hdr->slice_pos;
+ int i;
+
+ struct vb2_buffer *fwd_vb2_buf, *bwd_vb2_buf;
+ dma_addr_t src_buf_addr, dst_luma_addr, dst_chroma_addr;
+ dma_addr_t fwd_luma = 0, fwd_chroma = 0, bwd_luma = 0, bwd_chroma = 0;
+
+ fwd_vb2_buf = ctx->dst_bufs[frame_hdr->forward_ref_index];
+ if (fwd_vb2_buf) {
+ fwd_luma = vb2_dma_contig_plane_dma_addr(fwd_vb2_buf, 0);
+ fwd_chroma = vb2_dma_contig_plane_dma_addr(fwd_vb2_buf, 1);
+ }
+
+ bwd_vb2_buf = ctx->dst_bufs[frame_hdr->backward_ref_index];
+ if (bwd_vb2_buf) {
+ bwd_luma = vb2_dma_contig_plane_dma_addr(bwd_vb2_buf, 0);
+ bwd_chroma = vb2_dma_contig_plane_dma_addr(bwd_vb2_buf, 1);
+ }
+
+ /* Activate MPEG engine. */
+ cedrus_engine_enable(dev, CEDRUS_ENGINE_MPEG);
+
+ /* Set quantization matrices. */
+ for (i = 0; i < 64; i++) {
+ cedrus_write(dev, VE_MPEG_IQ_MIN_INPUT, m_iq(i));
+ cedrus_write(dev, VE_MPEG_IQ_MIN_INPUT, m_niq(i));
+ }
+
+ /* Set frame dimensions. */
+ cedrus_write(dev, VE_MPEG_SIZE, width << 8 | height);
+ cedrus_write(dev, VE_MPEG_FRAME_SIZE, width << 20 | height << 4);
+
+ /* Set MPEG picture header. */
+ pic_header |= (frame_hdr->picture_coding_type & 0xf) << 28;
+ pic_header |= (frame_hdr->f_code[0][0] & 0xf) << 24;
+ pic_header |= (frame_hdr->f_code[0][1] & 0xf) << 20;
+ pic_header |= (frame_hdr->f_code[1][0] & 0xf) << 16;
+ pic_header |= (frame_hdr->f_code[1][1] & 0xf) << 12;
+ pic_header |= (frame_hdr->intra_dc_precision & 0x3) << 10;
+ pic_header |= (frame_hdr->picture_structure & 0x3) << 8;
+ pic_header |= (frame_hdr->top_field_first & 0x1) << 7;
+ pic_header |= (frame_hdr->frame_pred_frame_dct & 0x1) << 6;
+ pic_header |= (frame_hdr->concealment_motion_vectors & 0x1) << 5;
+ pic_header |= (frame_hdr->q_scale_type & 0x1) << 4;
+ pic_header |= (frame_hdr->intra_vlc_format & 0x1) << 3;
+ pic_header |= (frame_hdr->alternate_scan & 0x1) << 2;
+ cedrus_write(dev, VE_MPEG_PIC_HDR, pic_header);
+
+ /* Enable interrupt and an unknown control flag. */
+ cedrus_write(dev, VE_MPEG_CTRL, VE_MPEG_CTRL_MPEG2);
+
+ /* Macroblock address. */
+ cedrus_write(dev, VE_MPEG_MBA, 0);
+
+ /* Clear previous errors. */
+ cedrus_write(dev, VE_MPEG_ERROR, 0);
+
+ /* Clear correct macroblocks register. */
+ cedrus_write(dev, VE_MPEG_CTR_MB, 0);
+
+ /* Forward and backward prediction reference buffers. */
+ cedrus_write(dev, VE_MPEG_FWD_LUMA, fwd_luma);
+ cedrus_write(dev, VE_MPEG_FWD_CHROMA, fwd_chroma);
+ cedrus_write(dev, VE_MPEG_BACK_LUMA, bwd_luma);
+ cedrus_write(dev, VE_MPEG_BACK_CHROMA, bwd_chroma);
+
+ /* Destination luma and chroma buffers. */
+ dst_luma_addr = vb2_dma_contig_plane_dma_addr(&run->dst->vb2_buf, 0);
+ dst_chroma_addr = vb2_dma_contig_plane_dma_addr(&run->dst->vb2_buf, 1);
+ cedrus_write(dev, VE_MPEG_REC_LUMA, dst_luma_addr);
+ cedrus_write(dev, VE_MPEG_REC_CHROMA, dst_chroma_addr);
+ cedrus_write(dev, VE_MPEG_ROT_LUMA, dst_luma_addr);
+ cedrus_write(dev, VE_MPEG_ROT_CHROMA, dst_chroma_addr);
+
+ /* Source offset and length in bits. */
+ cedrus_write(dev, VE_MPEG_VLD_OFFSET, frame_hdr->slice_pos);
+ cedrus_write(dev, VE_MPEG_VLD_LEN, vld_len);
+
+ /* Source beginning and end addresses. */
+ src_buf_addr = vb2_dma_contig_plane_dma_addr(&run->src->vb2_buf, 0);
+ cedrus_write(dev, VE_MPEG_VLD_ADDR, VE_MPEG_VLD_ADDR_VAL(src_buf_addr));
+ cedrus_write(dev, VE_MPEG_VLD_END, src_buf_addr + VBV_SIZE - 1);
+}
+
+void cedrus_mpeg2_trigger(struct cedrus_ctx *ctx)
+{
+ struct cedrus_dev *dev = ctx->dev;
+
+ /* Trigger MPEG engine. */
+ cedrus_write(dev, VE_MPEG_TRIGGER, VE_TRIG_MPEG2);
+}
diff --git a/drivers/media/platform/sunxi/cedrus/cedrus_mpeg2.h b/drivers/media/platform/sunxi/cedrus/cedrus_mpeg2.h
new file mode 100644
index 000000000000..fd864fab6986
--- /dev/null
+++ b/drivers/media/platform/sunxi/cedrus/cedrus_mpeg2.h
@@ -0,0 +1,24 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Sunxi-Cedrus VPU driver
+ *
+ * Copyright (C) 2018 Paul Kocialkowski <paul.kocialkowski@bootlin.com>
+ * Copyright (C) 2016 Florent Revest <florent.revest@free-electrons.com>
+ *
+ * Based on the vim2m driver, that is:
+ *
+ * Copyright (c) 2009-2010 Samsung Electronics Co., Ltd.
+ * Pawel Osciak, <pawel@osciak.com>
+ * Marek Szyprowski, <m.szyprowski@samsung.com>
+ */
+
+#ifndef _CEDRUS_MPEG2_H_
+#define _CEDRUS_MPEG2_H_
+
+struct cedrus_ctx;
+struct cedrus_run;
+
+void cedrus_mpeg2_setup(struct cedrus_ctx *ctx, struct cedrus_run *run);
+void cedrus_mpeg2_trigger(struct cedrus_ctx *ctx);
+
+#endif
diff --git a/drivers/media/platform/sunxi/cedrus/cedrus_regs.h b/drivers/media/platform/sunxi/cedrus/cedrus_regs.h
new file mode 100644
index 000000000000..442befcdb6ea
--- /dev/null
+++ b/drivers/media/platform/sunxi/cedrus/cedrus_regs.h
@@ -0,0 +1,167 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Sunxi-Cedrus VPU driver
+ *
+ * Copyright (C) 2018 Paul Kocialkowski <paul.kocialkowski@bootlin.com>
+ * Copyright (C) 2016 Florent Revest <florent.revest@free-electrons.com>
+ *
+ * Based on the vim2m driver, that is:
+ *
+ * Copyright (c) 2009-2010 Samsung Electronics Co., Ltd.
+ * Pawel Osciak, <pawel@osciak.com>
+ * Marek Szyprowski, <m.szyprowski@samsung.com>
+ */
+
+#ifndef _CEDRUS_REGS_H_
+#define _CEDRUS_REGS_H_
+
+/*
+ * For more information, consult http://linux-sunxi.org/VE_Register_guide
+ */
+
+/* VE_MPEG_CTRL:
+ * The bit 3 (0x8) is used to enable IRQs
+ * The other bits are unknown but needed
+ */
+#define VE_MPEG_CTRL_MPEG2 0x800001b8
+#define VE_MPEG_CTRL_MPEG4 (0x80084118 | BIT(7))
+#define VE_MPEG_CTRL_MPEG4_P (VE_MPEG_CTRL_MPEG4 | BIT(12))
+
+/* VE_MPEG_VLD_ADDR:
+ * The bits 27 to 4 are used for the address
+ * The bits 31 to 28 (0x7) are used to select the MPEG or JPEG engine
+ */
+#define VE_MPEG_VLD_ADDR_VAL(x) ((x & 0x0ffffff0) | (x >> 28) | (0x7 << 28))
+
+/* VE_MPEG_TRIGGER:
+ * The first three bits are used to trigger the engine
+ * The bits 24 to 26 are used to select the input format (1 for MPEG1, 2 for
+ * MPEG2, 4 for MPEG4)
+ * The bit 21 (0x8) is used to disable bitstream error handling
+ *
+ * In MPEG4 the w*h value is somehow used for an offset, unknown but needed
+ */
+#define VE_TRIG_MPEG1 0x8100000f
+#define VE_TRIG_MPEG2 0x8200000f
+#define VE_TRIG_MPEG4(w, h) (0x8400000d | ((w * h) << 8))
+
+/* VE_MPEG_SDROT_CTRL:
+ * The bit 8 at zero is used to disable x downscaling
+ * The bit 10 at 0 is used to disable y downscaling
+ * The other bits are unknown but needed
+ */
+#define VE_NO_SDROT_CTRL 0x40620000
+
+/* Decent size fo video buffering verifier */
+#define VBV_SIZE (1024 * 1024)
+
+/* Registers addresses */
+#define VE_CTRL 0x000
+/*
+ * The datasheet states that this should be set to 2MB on a 32bits
+ * DDR-3.
+ */
+#define VE_CTRL_REC_WR_MODE_2MB (1 << 20)
+#define VE_CTRL_REC_WR_MODE_1MB (0 << 20)
+
+#define VE_CTRL_CACHE_BUS_BW_128 (3 << 16)
+#define VE_CTRL_CACHE_BUS_BW_256 (2 << 16)
+
+#define VE_CTRL_DEC_MODE_DISABLED (7 << 0)
+#define VE_CTRL_DEC_MODE_H265 (4 << 0)
+#define VE_CTRL_DEC_MODE_H264 (1 << 0)
+#define VE_CTRL_DEC_MODE_MPEG (0 << 0)
+
+#define VE_VERSION 0x0f0
+
+#define VE_MPEG_PIC_HDR 0x100
+#define VE_MPEG_VOP_HDR 0x104
+#define VE_MPEG_SIZE 0x108
+#define VE_MPEG_FRAME_SIZE 0x10c
+#define VE_MPEG_MBA 0x110
+#define VE_MPEG_CTRL 0x114
+#define VE_MPEG_TRIGGER 0x118
+#define VE_MPEG_STATUS 0x11c
+#define VE_MPEG_TRBTRD_FIELD 0x120
+#define VE_MPEG_TRBTRD_FRAME 0x124
+#define VE_MPEG_VLD_ADDR 0x128
+#define VE_MPEG_VLD_OFFSET 0x12c
+#define VE_MPEG_VLD_LEN 0x130
+#define VE_MPEG_VLD_END 0x134
+#define VE_MPEG_MBH_ADDR 0x138
+#define VE_MPEG_DCAC_ADDR 0x13c
+#define VE_MPEG_NCF_ADDR 0x144
+#define VE_MPEG_REC_LUMA 0x148
+#define VE_MPEG_REC_CHROMA 0x14c
+#define VE_MPEG_FWD_LUMA 0x150
+#define VE_MPEG_FWD_CHROMA 0x154
+#define VE_MPEG_BACK_LUMA 0x158
+#define VE_MPEG_BACK_CHROMA 0x15c
+#define VE_MPEG_IQ_MIN_INPUT 0x180
+#define VE_MPEG_QP_INPUT 0x184
+#define VE_MPEG_JPEG_SIZE 0x1b8
+#define VE_MPEG_JPEG_RES_INT 0x1c0
+#define VE_MPEG_ERROR 0x1c4
+#define VE_MPEG_CTR_MB 0x1c8
+#define VE_MPEG_ROT_LUMA 0x1cc
+#define VE_MPEG_ROT_CHROMA 0x1d0
+#define VE_MPEG_SDROT_CTRL 0x1d4
+#define VE_MPEG_RAM_WRITE_PTR 0x1e0
+#define VE_MPEG_RAM_WRITE_DATA 0x1e4
+
+#define VE_H264_FRAME_SIZE 0x200
+#define VE_H264_PIC_HDR 0x204
+#define VE_H264_SLICE_HDR 0x208
+#define VE_H264_SLICE_HDR2 0x20c
+#define VE_H264_PRED_WEIGHT 0x210
+#define VE_H264_QP_PARAM 0x21c
+#define VE_H264_CTRL 0x220
+#define VE_H264_TRIGGER 0x224
+#define VE_H264_STATUS 0x228
+#define VE_H264_CUR_MB_NUM 0x22c
+#define VE_H264_VLD_ADDR 0x230
+#define VE_H264_VLD_OFFSET 0x234
+#define VE_H264_VLD_LEN 0x238
+#define VE_H264_VLD_END 0x23c
+#define VE_H264_SDROT_CTRL 0x240
+#define VE_H264_OUTPUT_FRAME_IDX 0x24c
+#define VE_H264_EXTRA_BUFFER1 0x250
+#define VE_H264_EXTRA_BUFFER2 0x254
+#define VE_H264_BASIC_BITS 0x2dc
+#define VE_H264_RAM_WRITE_PTR 0x2e0
+#define VE_H264_RAM_WRITE_DATA 0x2e4
+
+#define VE_SRAM_H264_PRED_WEIGHT_TABLE 0x000
+#define VE_SRAM_H264_FRAMEBUFFER_LIST 0x400
+#define VE_SRAM_H264_REF_LIST0 0x640
+#define VE_SRAM_H264_REF_LIST1 0x664
+#define VE_SRAM_H264_SCALING_LISTS 0x800
+
+#define VE_ISP_INPUT_SIZE 0xa00
+#define VE_ISP_INPUT_STRIDE 0xa04
+#define VE_ISP_CTRL 0xa08
+#define VE_ISP_INPUT_LUMA 0xa78
+#define VE_ISP_INPUT_CHROMA 0xa7c
+
+#define VE_AVC_PARAM 0xb04
+#define VE_AVC_QP 0xb08
+#define VE_AVC_MOTION_EST 0xb10
+#define VE_AVC_CTRL 0xb14
+#define VE_AVC_TRIGGER 0xb18
+#define VE_AVC_STATUS 0xb1c
+#define VE_AVC_BASIC_BITS 0xb20
+#define VE_AVC_UNK_BUF 0xb60
+#define VE_AVC_VLE_ADDR 0xb80
+#define VE_AVC_VLE_END 0xb84
+#define VE_AVC_VLE_OFFSET 0xb88
+#define VE_AVC_VLE_MAX 0xb8c
+#define VE_AVC_VLE_LENGTH 0xb90
+#define VE_AVC_REF_LUMA 0xba0
+#define VE_AVC_REF_CHROMA 0xba4
+#define VE_AVC_REC_LUMA 0xbb0
+#define VE_AVC_REC_CHROMA 0xbb4
+#define VE_AVC_REF_SLUMA 0xbb8
+#define VE_AVC_REC_SLUMA 0xbbc
+#define VE_AVC_MB_INFO 0xbc0
+
+#endif
diff --git a/drivers/media/platform/sunxi/cedrus/cedrus_video.c b/drivers/media/platform/sunxi/cedrus/cedrus_video.c
new file mode 100644
index 000000000000..ad7da8d36966
--- /dev/null
+++ b/drivers/media/platform/sunxi/cedrus/cedrus_video.c
@@ -0,0 +1,502 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Sunxi-Cedrus VPU driver
+ *
+ * Copyright (C) 2018 Paul Kocialkowski <paul.kocialkowski@bootlin.com>
+ * Copyright (C) 2016 Florent Revest <florent.revest@free-electrons.com>
+ *
+ * Based on the vim2m driver, that is:
+ *
+ * Copyright (c) 2009-2010 Samsung Electronics Co., Ltd.
+ * Pawel Osciak, <pawel@osciak.com>
+ * Marek Szyprowski, <m.szyprowski@samsung.com>
+ */
+
+#include <media/videobuf2-dma-contig.h>
+#include <media/v4l2-device.h>
+#include <media/v4l2-ioctl.h>
+#include <media/v4l2-ctrls.h>
+#include <media/v4l2-event.h>
+#include <media/v4l2-mem2mem.h>
+
+#include "cedrus.h"
+#include "cedrus_mpeg2.h"
+#include "cedrus_dec.h"
+#include "cedrus_hw.h"
+
+/* Flags that indicate a format can be used for capture/output. */
+#define CEDRUS_CAPTURE BIT(0)
+#define CEDRUS_OUTPUT BIT(1)
+
+#define CEDRUS_MIN_WIDTH 16U
+#define CEDRUS_MIN_HEIGHT 16U
+#define CEDRUS_MAX_WIDTH 3840U
+#define CEDRUS_MAX_HEIGHT 2160U
+
+static struct cedrus_fmt formats[] = {
+ {
+ .fourcc = V4L2_PIX_FMT_MB32_NV12,
+ .types = CEDRUS_CAPTURE,
+ .depth = 2,
+ .num_planes = 2,
+ },
+ {
+ .fourcc = V4L2_PIX_FMT_MPEG2_SLICE,
+ .types = CEDRUS_OUTPUT,
+ .num_planes = 1,
+ },
+};
+
+#define NUM_FORMATS ARRAY_SIZE(formats)
+
+static struct cedrus_fmt *find_format(struct v4l2_format *f)
+{
+ struct cedrus_fmt *fmt;
+ unsigned int k;
+
+ for (k = 0; k < NUM_FORMATS; k++) {
+ fmt = &formats[k];
+ if (fmt->fourcc == f->fmt.pix_mp.pixelformat)
+ break;
+ }
+
+ if (k == NUM_FORMATS)
+ return NULL;
+
+ return &formats[k];
+}
+
+static inline struct cedrus_ctx *file2ctx(struct file *file)
+{
+ return container_of(file->private_data, struct cedrus_ctx, fh);
+}
+
+static int vidioc_querycap(struct file *file, void *priv,
+ struct v4l2_capability *cap)
+{
+ strncpy(cap->driver, CEDRUS_NAME, sizeof(cap->driver) - 1);
+ strncpy(cap->card, CEDRUS_NAME, sizeof(cap->card) - 1);
+ snprintf(cap->bus_info, sizeof(cap->bus_info),
+ "platform:%s", CEDRUS_NAME);
+ cap->device_caps = V4L2_CAP_VIDEO_M2M_MPLANE | V4L2_CAP_STREAMING;
+ cap->capabilities = cap->device_caps | V4L2_CAP_DEVICE_CAPS;
+ return 0;
+}
+
+static int enum_fmt(struct v4l2_fmtdesc *f, u32 type)
+{
+ struct cedrus_fmt *fmt;
+ int i, num = 0;
+
+ for (i = 0; i < NUM_FORMATS; ++i) {
+ if (formats[i].types & type) {
+ /* index-th format of type type found ? */
+ if (num == f->index)
+ break;
+ /*
+ * Correct type but haven't reached our index yet,
+ * just increment per-type index
+ */
+ ++num;
+ }
+ }
+
+ if (i < NUM_FORMATS) {
+ fmt = &formats[i];
+ f->pixelformat = fmt->fourcc;
+ return 0;
+ }
+
+ return -EINVAL;
+}
+
+static int vidioc_enum_fmt_vid_cap(struct file *file, void *priv,
+ struct v4l2_fmtdesc *f)
+{
+ return enum_fmt(f, CEDRUS_CAPTURE);
+}
+
+static int vidioc_enum_fmt_vid_out(struct file *file, void *priv,
+ struct v4l2_fmtdesc *f)
+{
+ return enum_fmt(f, CEDRUS_OUTPUT);
+}
+
+static int vidioc_g_fmt(struct cedrus_ctx *ctx, struct v4l2_format *f)
+{
+ struct cedrus_dev *dev = ctx->dev;
+
+ switch (f->type) {
+ case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE:
+ f->fmt.pix_mp = ctx->dst_fmt;
+ break;
+ case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE:
+ f->fmt.pix_mp = ctx->src_fmt;
+ break;
+ default:
+ v4l2_err(&dev->v4l2_dev,
+ "Invalid buffer type for getting format\n");
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static int vidioc_g_fmt_vid_out(struct file *file, void *priv,
+ struct v4l2_format *f)
+{
+ return vidioc_g_fmt(file2ctx(file), f);
+}
+
+static int vidioc_g_fmt_vid_cap(struct file *file, void *priv,
+ struct v4l2_format *f)
+{
+ return vidioc_g_fmt(file2ctx(file), f);
+}
+
+static int vidioc_try_fmt(struct v4l2_format *f, struct cedrus_fmt *fmt)
+{
+ int i;
+ __u32 bpl;
+
+ f->fmt.pix_mp.field = V4L2_FIELD_NONE;
+ f->fmt.pix_mp.num_planes = fmt->num_planes;
+
+ switch (f->type) {
+ case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE:
+ if (f->fmt.pix_mp.plane_fmt[0].sizeimage == 0)
+ return -EINVAL;
+
+ f->fmt.pix_mp.plane_fmt[0].bytesperline = 0;
+ break;
+ case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE:
+ /* Limit to hardware min/max. */
+ f->fmt.pix_mp.width = clamp(f->fmt.pix_mp.width,
+ CEDRUS_MIN_WIDTH, CEDRUS_MAX_WIDTH);
+ f->fmt.pix_mp.height = clamp(f->fmt.pix_mp.height,
+ CEDRUS_MIN_HEIGHT,
+ CEDRUS_MAX_HEIGHT);
+
+ for (i = 0; i < f->fmt.pix_mp.num_planes; ++i) {
+ bpl = (f->fmt.pix_mp.width * fmt->depth) >> 3;
+ f->fmt.pix_mp.plane_fmt[i].bytesperline = bpl;
+ f->fmt.pix_mp.plane_fmt[i].sizeimage =
+ f->fmt.pix_mp.height * bpl;
+ }
+ break;
+ }
+ return 0;
+}
+
+static int vidioc_try_fmt_vid_cap(struct file *file, void *priv,
+ struct v4l2_format *f)
+{
+ struct cedrus_fmt *fmt;
+ struct cedrus_ctx *ctx = file2ctx(file);
+ struct cedrus_dev *dev = ctx->dev;
+
+ fmt = find_format(f);
+ if (!fmt) {
+ f->fmt.pix_mp.pixelformat = formats[0].fourcc;
+ fmt = find_format(f);
+ }
+ if (!(fmt->types & CEDRUS_CAPTURE)) {
+ v4l2_err(&dev->v4l2_dev, "Invalid destination format: %08x\n",
+ f->fmt.pix_mp.pixelformat);
+ return -EINVAL;
+ }
+
+ return vidioc_try_fmt(f, fmt);
+}
+
+static int vidioc_try_fmt_vid_out(struct file *file, void *priv,
+ struct v4l2_format *f)
+{
+ struct cedrus_fmt *fmt;
+ struct cedrus_ctx *ctx = file2ctx(file);
+ struct cedrus_dev *dev = ctx->dev;
+
+ fmt = find_format(f);
+ if (!fmt) {
+ f->fmt.pix_mp.pixelformat = formats[0].fourcc;
+ fmt = find_format(f);
+ }
+ if (!(fmt->types & CEDRUS_OUTPUT)) {
+ v4l2_err(&dev->v4l2_dev, "Invalid source format: %08x\n",
+ f->fmt.pix_mp.pixelformat);
+ return -EINVAL;
+ }
+
+ return vidioc_try_fmt(f, fmt);
+}
+
+static int vidioc_s_fmt(struct cedrus_ctx *ctx, struct v4l2_format *f)
+{
+ struct cedrus_dev *dev = ctx->dev;
+ struct v4l2_pix_format_mplane *pix_fmt_mp = &f->fmt.pix_mp;
+ struct cedrus_fmt *fmt;
+ int i, ret = 0;
+
+ switch (f->type) {
+ case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE:
+ ctx->vpu_src_fmt = find_format(f);
+ ctx->src_fmt = *pix_fmt_mp;
+ break;
+ case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE:
+ fmt = find_format(f);
+ ctx->vpu_dst_fmt = fmt;
+
+ for (i = 0; i < fmt->num_planes; ++i) {
+ pix_fmt_mp->plane_fmt[i].bytesperline =
+ pix_fmt_mp->width * fmt->depth;
+ pix_fmt_mp->plane_fmt[i].sizeimage =
+ pix_fmt_mp->plane_fmt[i].bytesperline
+ * pix_fmt_mp->height;
+ }
+ ctx->dst_fmt = *pix_fmt_mp;
+ break;
+ default:
+ v4l2_err(&dev->v4l2_dev,
+ "Invalid buffer type for setting format\n");
+ return -EINVAL;
+ }
+
+ return ret;
+}
+
+static int vidioc_s_fmt_vid_cap(struct file *file, void *priv,
+ struct v4l2_format *f)
+{
+ struct cedrus_ctx *ctx = file2ctx(file);
+ int ret;
+
+ ret = vidioc_try_fmt_vid_cap(file, priv, f);
+ if (ret)
+ return ret;
+
+ return vidioc_s_fmt(ctx, f);
+}
+
+static int vidioc_s_fmt_vid_out(struct file *file, void *priv,
+ struct v4l2_format *f)
+{
+ struct cedrus_ctx *ctx = file2ctx(file);
+ int ret;
+
+ ret = vidioc_try_fmt_vid_out(file, priv, f);
+ if (ret)
+ return ret;
+
+ return vidioc_s_fmt(ctx, f);
+}
+
+const struct v4l2_ioctl_ops cedrus_ioctl_ops = {
+ .vidioc_querycap = vidioc_querycap,
+
+ .vidioc_enum_fmt_vid_cap = vidioc_enum_fmt_vid_cap,
+ .vidioc_g_fmt_vid_cap_mplane = vidioc_g_fmt_vid_cap,
+ .vidioc_try_fmt_vid_cap_mplane = vidioc_try_fmt_vid_cap,
+ .vidioc_s_fmt_vid_cap_mplane = vidioc_s_fmt_vid_cap,
+
+ .vidioc_enum_fmt_vid_out_mplane = vidioc_enum_fmt_vid_out,
+ .vidioc_g_fmt_vid_out_mplane = vidioc_g_fmt_vid_out,
+ .vidioc_try_fmt_vid_out_mplane = vidioc_try_fmt_vid_out,
+ .vidioc_s_fmt_vid_out_mplane = vidioc_s_fmt_vid_out,
+
+ .vidioc_reqbufs = v4l2_m2m_ioctl_reqbufs,
+ .vidioc_querybuf = v4l2_m2m_ioctl_querybuf,
+ .vidioc_qbuf = v4l2_m2m_ioctl_qbuf,
+ .vidioc_dqbuf = v4l2_m2m_ioctl_dqbuf,
+ .vidioc_prepare_buf = v4l2_m2m_ioctl_prepare_buf,
+ .vidioc_create_bufs = v4l2_m2m_ioctl_create_bufs,
+ .vidioc_expbuf = v4l2_m2m_ioctl_expbuf,
+
+ .vidioc_streamon = v4l2_m2m_ioctl_streamon,
+ .vidioc_streamoff = v4l2_m2m_ioctl_streamoff,
+
+ .vidioc_subscribe_event = v4l2_ctrl_subscribe_event,
+ .vidioc_unsubscribe_event = v4l2_event_unsubscribe,
+};
+
+static int cedrus_queue_setup(struct vb2_queue *vq, unsigned int *nbufs,
+ unsigned int *nplanes, unsigned int sizes[],
+ struct device *alloc_devs[])
+{
+ struct cedrus_ctx *ctx = vb2_get_drv_priv(vq);
+ struct cedrus_dev *dev = ctx->dev;
+
+ switch (vq->type) {
+ case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE:
+ *nplanes = ctx->vpu_src_fmt->num_planes;
+
+ sizes[0] = ctx->src_fmt.plane_fmt[0].sizeimage;
+ break;
+
+ case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE:
+ *nplanes = ctx->vpu_dst_fmt->num_planes;
+
+ sizes[0] = round_up(ctx->dst_fmt.plane_fmt[0].sizeimage, 8);
+ sizes[1] = sizes[0];
+ break;
+
+ default:
+ v4l2_err(&dev->v4l2_dev,
+ "Invalid buffer type for queue setup\n");
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static int cedrus_buf_init(struct vb2_buffer *vb)
+{
+ struct vb2_queue *vq = vb->vb2_queue;
+ struct cedrus_ctx *ctx = container_of(vq->drv_priv,
+ struct cedrus_ctx, fh);
+
+ if (vq->type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE)
+ ctx->dst_bufs[vb->index] = vb;
+
+ return 0;
+}
+
+static void cedrus_buf_cleanup(struct vb2_buffer *vb)
+{
+ struct vb2_queue *vq = vb->vb2_queue;
+ struct cedrus_ctx *ctx = container_of(vq->drv_priv,
+ struct cedrus_ctx, fh);
+
+ if (vq->type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE)
+ ctx->dst_bufs[vb->index] = NULL;
+}
+
+static int cedrus_buf_prepare(struct vb2_buffer *vb)
+{
+ struct cedrus_ctx *ctx = vb2_get_drv_priv(vb->vb2_queue);
+ struct cedrus_dev *dev = ctx->dev;
+ struct vb2_queue *vq = vb->vb2_queue;
+ int i;
+
+ switch (vq->type) {
+ case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE:
+ if (vb2_plane_size(vb, 0)
+ < ctx->src_fmt.plane_fmt[0].sizeimage) {
+ v4l2_err(&dev->v4l2_dev,
+ "Buffer plane size too small for output\n");
+ return -EINVAL;
+ }
+ break;
+
+ case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE:
+ for (i = 0; i < ctx->vpu_dst_fmt->num_planes; ++i) {
+ if (vb2_plane_size(vb, i)
+ < ctx->dst_fmt.plane_fmt[i].sizeimage) {
+ v4l2_err(&dev->v4l2_dev,
+ "Buffer plane %d size too small for capture\n",
+ i);
+ break;
+ }
+ }
+
+ if (i != ctx->vpu_dst_fmt->num_planes)
+ return -EINVAL;
+ break;
+
+ default:
+ v4l2_err(&dev->v4l2_dev,
+ "Invalid buffer type for buffer preparation\n");
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static void cedrus_stop_streaming(struct vb2_queue *q)
+{
+ struct cedrus_ctx *ctx = vb2_get_drv_priv(q);
+ struct vb2_v4l2_buffer *vbuf;
+ unsigned long flags;
+
+ flush_scheduled_work();
+ for (;;) {
+ spin_lock_irqsave(&ctx->dev->irq_lock, flags);
+
+ if (V4L2_TYPE_IS_OUTPUT(q->type))
+ vbuf = v4l2_m2m_src_buf_remove(ctx->fh.m2m_ctx);
+ else
+ vbuf = v4l2_m2m_dst_buf_remove(ctx->fh.m2m_ctx);
+
+ spin_unlock_irqrestore(&ctx->dev->irq_lock, flags);
+
+ if (vbuf == NULL)
+ return;
+
+ v4l2_ctrl_request_complete(vbuf->vb2_buf.req_obj.req,
+ &ctx->hdl);
+ v4l2_m2m_buf_done(vbuf, VB2_BUF_STATE_ERROR);
+ }
+}
+
+static void cedrus_buf_queue(struct vb2_buffer *vb)
+{
+ struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
+ struct cedrus_ctx *ctx = vb2_get_drv_priv(vb->vb2_queue);
+
+ v4l2_m2m_buf_queue(ctx->fh.m2m_ctx, vbuf);
+}
+
+static void cedrus_buf_request_complete(struct vb2_buffer *vb)
+{
+ struct cedrus_ctx *ctx = vb2_get_drv_priv(vb->vb2_queue);
+
+ v4l2_ctrl_request_complete(vb->req_obj.req, &ctx->hdl);
+}
+
+static struct vb2_ops cedrus_qops = {
+ .queue_setup = cedrus_queue_setup,
+ .buf_prepare = cedrus_buf_prepare,
+ .buf_init = cedrus_buf_init,
+ .buf_cleanup = cedrus_buf_cleanup,
+ .buf_queue = cedrus_buf_queue,
+ .buf_request_complete = cedrus_buf_request_complete,
+ .stop_streaming = cedrus_stop_streaming,
+ .wait_prepare = vb2_ops_wait_prepare,
+ .wait_finish = vb2_ops_wait_finish,
+};
+
+int cedrus_queue_init(void *priv, struct vb2_queue *src_vq,
+ struct vb2_queue *dst_vq)
+{
+ struct cedrus_ctx *ctx = priv;
+ int ret;
+
+ src_vq->type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE;
+ src_vq->io_modes = VB2_MMAP | VB2_DMABUF;
+ src_vq->drv_priv = ctx;
+ src_vq->buf_struct_size = sizeof(struct cedrus_buffer);
+ src_vq->allow_zero_bytesused = 1;
+ src_vq->min_buffers_needed = 1;
+ src_vq->ops = &cedrus_qops;
+ src_vq->mem_ops = &vb2_dma_contig_memops;
+ src_vq->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_COPY;
+ src_vq->lock = &ctx->dev->dev_mutex;
+ src_vq->dev = ctx->dev->dev;
+
+ ret = vb2_queue_init(src_vq);
+ if (ret)
+ return ret;
+
+ dst_vq->type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE;
+ dst_vq->io_modes = VB2_MMAP | VB2_DMABUF;
+ dst_vq->drv_priv = ctx;
+ dst_vq->buf_struct_size = sizeof(struct cedrus_buffer);
+ dst_vq->allow_zero_bytesused = 1;
+ dst_vq->min_buffers_needed = 1;
+ dst_vq->ops = &cedrus_qops;
+ dst_vq->mem_ops = &vb2_dma_contig_memops;
+ dst_vq->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_COPY;
+ dst_vq->lock = &ctx->dev->dev_mutex;
+ dst_vq->dev = ctx->dev->dev;
+
+ return vb2_queue_init(dst_vq);
+}
diff --git a/drivers/media/platform/sunxi/cedrus/cedrus_video.h b/drivers/media/platform/sunxi/cedrus/cedrus_video.h
new file mode 100644
index 000000000000..ed7cea8a6d8f
--- /dev/null
+++ b/drivers/media/platform/sunxi/cedrus/cedrus_video.h
@@ -0,0 +1,23 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Sunxi-Cedrus VPU driver
+ *
+ * Copyright (C) 2018 Paul Kocialkowski <paul.kocialkowski@bootlin.com>
+ * Copyright (C) 2016 Florent Revest <florent.revest@free-electrons.com>
+ *
+ * Based on the vim2m driver, that is:
+ *
+ * Copyright (c) 2009-2010 Samsung Electronics Co., Ltd.
+ * Pawel Osciak, <pawel@osciak.com>
+ * Marek Szyprowski, <m.szyprowski@samsung.com>
+ */
+
+#ifndef _CEDRUS_VIDEO_H_
+#define _CEDRUS_VIDEO_H_
+
+extern const struct v4l2_ioctl_ops cedrus_ioctl_ops;
+
+int cedrus_queue_init(void *priv, struct vb2_queue *src_vq,
+ struct vb2_queue *dst_vq);
+
+#endif
--
2.17.0
^ permalink raw reply related
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox