* [kernel-hardening][RFC PATCH v2 01/12] x86: add PAX_REFCOUNT support
2015-12-17 14:57 [kernel-hardening][RFC PATCH v2 00/12] Add PAX_REFCOUNT overflow protection David Windsor
@ 2015-12-17 14:57 ` David Windsor
2015-12-17 17:16 ` Greg KH
2015-12-17 14:57 ` [kernel-hardening][RFC PATCH v2 02/12] percpu_ref: decrease per-CPU refcount bias David Windsor
` (11 subsequent siblings)
12 siblings, 1 reply; 32+ messages in thread
From: David Windsor @ 2015-12-17 14:57 UTC (permalink / raw)
To: kernel-hardening; +Cc: David Windsor
Add overflow protection to atomic_t and create a new type atomic_unchecked_t
for users of atomic_t not using the type for reference counting.
Signed-off-by: David Windsor <dave@progbits.org>
---
arch/x86/include/asm/atomic.h | 263 ++++++++++++++++++++++++++++++++++---
arch/x86/include/asm/atomic64_32.h | 100 ++++++++++++++
arch/x86/include/asm/atomic64_64.h | 164 +++++++++++++++++++++--
arch/x86/include/asm/bitops.h | 6 +-
arch/x86/include/asm/cmpxchg.h | 39 ++++++
arch/x86/include/asm/hw_irq.h | 4 +-
arch/x86/include/asm/local.h | 106 +++++++++++++--
arch/x86/include/asm/preempt.h | 2 +-
arch/x86/include/asm/rmwcc.h | 84 ++++++++++--
arch/x86/include/asm/rwsem.h | 60 ++++++++-
arch/x86/kernel/apic/apic.c | 2 +-
arch/x86/kernel/apic/io_apic.c | 4 +-
arch/x86/kernel/cpu/mcheck/mce.c | 12 +-
arch/x86/kernel/i8259.c | 2 +-
arch/x86/kernel/irq.c | 8 +-
arch/x86/kernel/kgdb.c | 6 +-
arch/x86/kernel/pvclock.c | 8 +-
arch/x86/kernel/tboot.c | 8 +-
arch/x86/kernel/traps.c | 6 +
arch/x86/lib/atomic64_386_32.S | 164 +++++++++++++++++++++++
arch/x86/lib/atomic64_cx8_32.S | 94 ++++++++++++-
arch/x86/mm/mmio-mod.c | 4 +-
fs/exec.c | 24 ++++
include/asm-generic/atomic-long.h | 210 +++++++++++++++++++++++++++++
include/asm-generic/atomic64.h | 12 ++
include/asm-generic/local.h | 13 ++
include/linux/sched.h | 13 ++
include/linux/types.h | 16 +++
28 files changed, 1346 insertions(+), 88 deletions(-)
diff --git a/arch/x86/include/asm/atomic.h b/arch/x86/include/asm/atomic.h
index e916895..0a77aa0 100644
--- a/arch/x86/include/asm/atomic.h
+++ b/arch/x86/include/asm/atomic.h
@@ -28,6 +28,17 @@ static __always_inline int atomic_read(const atomic_t *v)
}
/**
+ * atomic_read_unchecked - read atomic variable
+ * @v: pointer of type atomic_unchecked_t
+ *
+ * Atomically reads the value of @v.
+ */
+static __always_inline int atomic_read_unchecked(const atomic_unchecked_t *v)
+{
+ return ACCESS_ONCE((v)->counter);
+}
+
+/**
* atomic_set - set atomic variable
* @v: pointer of type atomic_t
* @i: required value
@@ -40,6 +51,18 @@ static __always_inline void atomic_set(atomic_t *v, int i)
}
/**
+ * atomic_set_unchecked - set atomic variable
+ * @v: pointer of type atomic_unchecked_t
+ * @i: required value
+ *
+ * Atomically sets the value of @v to @i.
+ */
+static __always_inline void atomic_set_unchecked(atomic_unchecked_t *v, int i)
+{
+ v->counter = i;
+}
+
+/**
* atomic_add - add integer to atomic variable
* @i: integer value to add
* @v: pointer of type atomic_t
@@ -48,7 +71,29 @@ static __always_inline void atomic_set(atomic_t *v, int i)
*/
static __always_inline void atomic_add(int i, atomic_t *v)
{
- asm volatile(LOCK_PREFIX "addl %1,%0"
+ asm volatile(LOCK_PREFIX "addl %1,%0\n"
+
+#ifdef CONFIG_PAX_REFCOUNT
+ "jno 0f\n"
+ LOCK_PREFIX "subl %1,%0\n"
+ "int $4\n0:\n"
+ _ASM_EXTABLE(0b, 0b)
+#endif
+
+ : "+m" (v->counter)
+ : "ir" (i));
+}
+
+/**
+ * atomic_add_unchecked - add integer to atomic variable
+ * @i: integer value to add
+ * @v: pointer of type atomic_unchecked_t
+ *
+ * Atomically adds @i to @v.
+ */
+static __always_inline void atomic_add_unchecked(int i, atomic_unchecked_t *v)
+{
+ asm volatile(LOCK_PREFIX "addl %1,%0\n"
: "+m" (v->counter)
: "ir" (i));
}
@@ -62,7 +107,29 @@ static __always_inline void atomic_add(int i, atomic_t *v)
*/
static __always_inline void atomic_sub(int i, atomic_t *v)
{
- asm volatile(LOCK_PREFIX "subl %1,%0"
+ asm volatile(LOCK_PREFIX "subl %1,%0\n"
+
+#ifdef CONFIG_PAX_REFCOUNT
+ "jno 0f\n"
+ LOCK_PREFIX "addl %1,%0\n"
+ "int $4\n0:\n"
+ _ASM_EXTABLE(0b, 0b)
+#endif
+
+ : "+m" (v->counter)
+ : "ir" (i));
+}
+
+/**
+ * atomic_sub_unchecked - subtract integer from atomic variable
+ * @i: integer value to subtract
+ * @v: pointer of type atomic_unchecked_t
+ *
+ * Atomically subtracts @i from @v.
+ */
+static __always_inline void atomic_sub_unchecked(int i, atomic_unchecked_t *v)
+{
+ asm volatile(LOCK_PREFIX "subl %1,%0\n"
: "+m" (v->counter)
: "ir" (i));
}
@@ -78,7 +145,7 @@ static __always_inline void atomic_sub(int i, atomic_t *v)
*/
static __always_inline int atomic_sub_and_test(int i, atomic_t *v)
{
- GEN_BINARY_RMWcc(LOCK_PREFIX "subl", v->counter, "er", i, "%0", "e");
+ GEN_BINARY_RMWcc(LOCK_PREFIX "subl", LOCK_PREFIX "addl", v->counter, "er", i, "%0", "e");
}
/**
@@ -89,7 +156,27 @@ static __always_inline int atomic_sub_and_test(int i, atomic_t *v)
*/
static __always_inline void atomic_inc(atomic_t *v)
{
- asm volatile(LOCK_PREFIX "incl %0"
+ asm volatile(LOCK_PREFIX "incl %0\n"
+
+#ifdef CONFIG_PAX_REFCOUNT
+ "jno 0f\n"
+ LOCK_PREFIX "decl %0\n"
+ "int $4\n0:\n"
+ _ASM_EXTABLE(0b, 0b)
+#endif
+
+ : "+m" (v->counter));
+}
+
+/**
+ * atomic_inc_unchecked - increment atomic variable
+ * @v: pointer of type atomic_unchecked_t
+ *
+ * Atomically increments @v by 1.
+ */
+static __always_inline void atomic_inc_unchecked(atomic_unchecked_t *v)
+{
+ asm volatile(LOCK_PREFIX "incl %0\n"
: "+m" (v->counter));
}
@@ -101,7 +188,27 @@ static __always_inline void atomic_inc(atomic_t *v)
*/
static __always_inline void atomic_dec(atomic_t *v)
{
- asm volatile(LOCK_PREFIX "decl %0"
+ asm volatile(LOCK_PREFIX "decl %0\n"
+
+#ifdef CONFIG_PAX_REFCOUNT
+ "jno 0f\n"
+ LOCK_PREFIX "incl %0\n"
+ "int $4\n0:\n"
+ _ASM_EXTABLE(0b, 0b)
+#endif
+
+ : "+m" (v->counter));
+}
+
+/**
+ * atomic_dec_unchecked - decrement atomic variable
+ * @v: pointer of type atomic_unchecked_t
+ *
+ * Atomically decrements @v by 1.
+ */
+static __always_inline void atomic_dec_unchecked(atomic_unchecked_t *v)
+{
+ asm volatile(LOCK_PREFIX "decl %0\n"
: "+m" (v->counter));
}
@@ -115,7 +222,7 @@ static __always_inline void atomic_dec(atomic_t *v)
*/
static __always_inline int atomic_dec_and_test(atomic_t *v)
{
- GEN_UNARY_RMWcc(LOCK_PREFIX "decl", v->counter, "%0", "e");
+ GEN_UNARY_RMWcc(LOCK_PREFIX "decl", LOCK_PREFIX "incl", v->counter, "%0", "e");
}
/**
@@ -128,7 +235,20 @@ static __always_inline int atomic_dec_and_test(atomic_t *v)
*/
static __always_inline int atomic_inc_and_test(atomic_t *v)
{
- GEN_UNARY_RMWcc(LOCK_PREFIX "incl", v->counter, "%0", "e");
+ GEN_UNARY_RMWcc(LOCK_PREFIX "incl", LOCK_PREFIX "decl", v->counter, "%0", "e");
+}
+
+/**
+ * atomic_inc_and_test_unchecked - increment and test
+ * @v: pointer of type atomic_unchecked_t
+ *
+ * Atomically increments @v by 1
+ * and returns true if the result is zero, or false for all
+ * other cases.
+ */
+static __always_inline int atomic_inc_and_test_unchecked(atomic_unchecked_t *v)
+{
+ GEN_UNARY_RMWcc_unchecked(LOCK_PREFIX "incl", v->counter, "%0", "e");
}
/**
@@ -142,7 +262,7 @@ static __always_inline int atomic_inc_and_test(atomic_t *v)
*/
static __always_inline int atomic_add_negative(int i, atomic_t *v)
{
- GEN_BINARY_RMWcc(LOCK_PREFIX "addl", v->counter, "er", i, "%0", "s");
+ GEN_BINARY_RMWcc(LOCK_PREFIX "addl", LOCK_PREFIX "subl", v->counter, "er", i, "%0", "s");
}
/**
@@ -154,6 +274,18 @@ static __always_inline int atomic_add_negative(int i, atomic_t *v)
*/
static __always_inline int atomic_add_return(int i, atomic_t *v)
{
+ return i + xadd_check_overflow(&v->counter, i);
+}
+
+/**
+ * atomic_add_return_unchecked - add integer and return
+ * @i: integer value to add
+ * @v: pointer of type atomi_uncheckedc_t
+ *
+ * Atomically adds @i to @v and returns @i + @v
+ */
+static __always_inline int atomic_add_return_unchecked(int i, atomic_unchecked_t *v)
+{
return i + xadd(&v->counter, i);
}
@@ -170,6 +302,10 @@ static __always_inline int atomic_sub_return(int i, atomic_t *v)
}
#define atomic_inc_return(v) (atomic_add_return(1, v))
+static __always_inline int atomic_inc_return_unchecked(atomic_unchecked_t *v)
+{
+ return atomic_add_return_unchecked(1, v);
+}
#define atomic_dec_return(v) (atomic_sub_return(1, v))
static __always_inline int atomic_cmpxchg(atomic_t *v, int old, int new)
@@ -177,11 +313,21 @@ static __always_inline int atomic_cmpxchg(atomic_t *v, int old, int new)
return cmpxchg(&v->counter, old, new);
}
+static __always_inline int atomic_cmpxchg_unchecked(atomic_unchecked_t *v, int old, int new)
+{
+ return cmpxchg(&v->counter, old, new);
+}
+
static inline int atomic_xchg(atomic_t *v, int new)
{
return xchg(&v->counter, new);
}
+static inline int atomic_xchg_unchecked(atomic_unchecked_t *v, int new)
+{
+ return xchg(&v->counter, new);
+}
+
/**
* __atomic_add_unless - add unless the number is already a given value
* @v: pointer of type atomic_t
@@ -193,12 +339,25 @@ static inline int atomic_xchg(atomic_t *v, int new)
*/
static __always_inline int __atomic_add_unless(atomic_t *v, int a, int u)
{
- int c, old;
+ int c, old, new;
c = atomic_read(v);
for (;;) {
- if (unlikely(c == (u)))
+ if (unlikely(c == u))
break;
- old = atomic_cmpxchg((v), c, c + (a));
+
+ asm volatile("addl %2,%0\n"
+
+#ifdef CONFIG_PAX_REFCOUNT
+ "jno 0f\n"
+ "subl %2,%0\n"
+ "int $4\n0:\n"
+ _ASM_EXTABLE(0b, 0b)
+#endif
+
+ : "=r" (new)
+ : "0" (c), "ir" (a));
+
+ old = atomic_cmpxchg(v, c, new);
if (likely(old == c))
break;
c = old;
@@ -207,6 +366,49 @@ static __always_inline int __atomic_add_unless(atomic_t *v, int a, int u)
}
/**
+ * atomic_inc_not_zero_hint - increment if not null
+ * @v: pointer of type atomic_t
+ * @hint: probable value of the atomic before the increment
+ *
+ * This version of atomic_inc_not_zero() gives a hint of probable
+ * value of the atomic. This helps processor to not read the memory
+ * before doing the atomic read/modify/write cycle, lowering
+ * number of bus transactions on some arches.
+ *
+ * Returns: 0 if increment was not done, 1 otherwise.
+ */
+#define atomic_inc_not_zero_hint atomic_inc_not_zero_hint
+static inline int atomic_inc_not_zero_hint(atomic_t *v, int hint)
+{
+ int val, c = hint, new;
+
+ /* sanity test, should be removed by compiler if hint is a constant */
+ if (!hint)
+ return __atomic_add_unless(v, 1, 0);
+
+ do {
+ asm volatile("incl %0\n"
+
+#ifdef CONFIG_PAX_REFCOUNT
+ "jno 0f\n"
+ "decl %0\n"
+ "int $4\n0:\n"
+ _ASM_EXTABLE(0b, 0b)
+#endif
+
+ : "=r" (new)
+ : "0" (c));
+
+ val = atomic_cmpxchg(v, c, new);
+ if (val == c)
+ return 1;
+ c = val;
+ } while (c);
+
+ return 0;
+}
+
+/**
* atomic_inc_short - increment of a short integer
* @v: pointer to type int
*
@@ -220,14 +422,37 @@ static __always_inline short int atomic_inc_short(short int *v)
}
/* These are x86-specific, used by some header files */
-#define atomic_clear_mask(mask, addr) \
- asm volatile(LOCK_PREFIX "andl %0,%1" \
- : : "r" (~(mask)), "m" (*(addr)) : "memory")
-
-#define atomic_set_mask(mask, addr) \
- asm volatile(LOCK_PREFIX "orl %0,%1" \
- : : "r" ((unsigned)(mask)), "m" (*(addr)) \
- : "memory")
+static inline void atomic_clear_mask(unsigned int mask, atomic_t *v)
+{
+ asm volatile(LOCK_PREFIX "andl %1,%0"
+ : "+m" (v->counter)
+ : "r" (~(mask))
+ : "memory");
+}
+
+static inline void atomic_clear_mask_unchecked(unsigned int mask, atomic_unchecked_t *v)
+{
+ asm volatile(LOCK_PREFIX "andl %1,%0"
+ : "+m" (v->counter)
+ : "r" (~(mask))
+ : "memory");
+}
+
+static inline void atomic_set_mask(unsigned int mask, atomic_t *v)
+{
+ asm volatile(LOCK_PREFIX "orl %1,%0"
+ : "+m" (v->counter)
+ : "r" (mask)
+ : "memory");
+}
+
+static inline void atomic_set_mask_unchecked(unsigned int mask, atomic_unchecked_t *v)
+{
+ asm volatile(LOCK_PREFIX "orl %1,%0"
+ : "+m" (v->counter)
+ : "r" (mask)
+ : "memory");
+}
#ifdef CONFIG_X86_32
# include <asm/atomic64_32.h>
diff --git a/arch/x86/include/asm/atomic64_32.h b/arch/x86/include/asm/atomic64_32.h
index b154de7..a4bc513 100644
--- a/arch/x86/include/asm/atomic64_32.h
+++ b/arch/x86/include/asm/atomic64_32.h
@@ -12,6 +12,14 @@ typedef struct {
u64 __aligned(8) counter;
} atomic64_t;
+#ifdef CONFIG_PAX_REFCOUNT
+typedef struct {
+ u64 __aligned(8) counter;
+} atomic64_unchecked_t;
+#else
+typedef atomic64_t atomic64_unchecked_t;
+#endif
+
#define ATOMIC64_INIT(val) { (val) }
#define __ATOMIC64_DECL(sym) void atomic64_##sym(atomic64_t *, ...)
@@ -37,21 +45,31 @@ typedef struct {
ATOMIC64_DECL_ONE(sym##_386)
ATOMIC64_DECL_ONE(add_386);
+ATOMIC64_DECL_ONE(add_unchecked_386);
ATOMIC64_DECL_ONE(sub_386);
+ATOMIC64_DECL_ONE(sub_unchecked_386);
ATOMIC64_DECL_ONE(inc_386);
+ATOMIC64_DECL_ONE(inc_unchecked_386);
ATOMIC64_DECL_ONE(dec_386);
+ATOMIC64_DECL_ONE(dec_unchecked_386);
#endif
#define alternative_atomic64(f, out, in...) \
__alternative_atomic64(f, f, ASM_OUTPUT2(out), ## in)
ATOMIC64_DECL(read);
+ATOMIC64_DECL(read_unchecked);
ATOMIC64_DECL(set);
+ATOMIC64_DECL(set_unchecked);
ATOMIC64_DECL(xchg);
ATOMIC64_DECL(add_return);
+ATOMIC64_DECL(add_return_unchecked);
ATOMIC64_DECL(sub_return);
+ATOMIC64_DECL(sub_return_unchecked);
ATOMIC64_DECL(inc_return);
+ATOMIC64_DECL(inc_return_unchecked);
ATOMIC64_DECL(dec_return);
+ATOMIC64_DECL(dec_return_unchecked);
ATOMIC64_DECL(dec_if_positive);
ATOMIC64_DECL(inc_not_zero);
ATOMIC64_DECL(add_unless);
@@ -77,6 +95,21 @@ static inline long long atomic64_cmpxchg(atomic64_t *v, long long o, long long n
}
/**
+ * atomic64_cmpxchg_unchecked - cmpxchg atomic64 variable
+ * @p: pointer to type atomic64_unchecked_t
+ * @o: expected value
+ * @n: new value
+ *
+ * Atomically sets @v to @n if it was equal to @o and returns
+ * the old value.
+ */
+
+static inline long long atomic64_cmpxchg_unchecked(atomic64_unchecked_t *v, long long o, long long n)
+{
+ return cmpxchg64(&v->counter, o, n);
+}
+
+/**
* atomic64_xchg - xchg atomic64 variable
* @v: pointer to type atomic64_t
* @n: value to assign
@@ -112,6 +145,22 @@ static inline void atomic64_set(atomic64_t *v, long long i)
}
/**
+ * atomic64_set_unchecked - set atomic64 variable
+ * @v: pointer to type atomic64_unchecked_t
+ * @n: value to assign
+ *
+ * Atomically sets the value of @v to @n.
+ */
+static inline void atomic64_set_unchecked(atomic64_unchecked_t *v, long long i)
+{
+ unsigned high = (unsigned)(i >> 32);
+ unsigned low = (unsigned)i;
+ alternative_atomic64(set, /* no output */,
+ "S" (v), "b" (low), "c" (high)
+ : "eax", "edx", "memory");
+}
+
+/**
* atomic64_read - read atomic64 variable
* @v: pointer to type atomic64_t
*
@@ -125,6 +174,19 @@ static inline long long atomic64_read(const atomic64_t *v)
}
/**
+ * atomic64_read_unchecked - read atomic64 variable
+ * @v: pointer to type atomic64_unchecked_t
+ *
+ * Atomically reads the value of @v and returns it.
+ */
+static inline long long atomic64_read_unchecked(const atomic64_unchecked_t *v)
+{
+ long long r;
+ alternative_atomic64(read, "=&A" (r), "c" (v) : "memory");
+ return r;
+ }
+
+/**
* atomic64_add_return - add and return
* @i: integer value to add
* @v: pointer to type atomic64_t
@@ -139,6 +201,21 @@ static inline long long atomic64_add_return(long long i, atomic64_t *v)
return i;
}
+/**
+ * atomic64_add_return_unchecked - add and return
+ * @i: integer value to add
+ * @v: pointer to type atomic64_unchecked_t
+ *
+ * Atomically adds @i to @v and returns @i + *@v
+ */
+static inline long long atomic64_add_return_unchecked(long long i, atomic64_unchecked_t *v)
+{
+ alternative_atomic64(add_return_unchecked,
+ ASM_OUTPUT2("+A" (i), "+c" (v)),
+ ASM_NO_INPUT_CLOBBER("memory"));
+ return i;
+}
+
/*
* Other variants with different arithmetic operators:
*/
@@ -158,6 +235,14 @@ static inline long long atomic64_inc_return(atomic64_t *v)
return a;
}
+static inline long long atomic64_inc_return_unchecked(atomic64_unchecked_t *v)
+{
+ long long a;
+ alternative_atomic64(inc_return_unchecked, "=&A" (a),
+ "S" (v) : "memory", "ecx");
+ return a;
+}
+
static inline long long atomic64_dec_return(atomic64_t *v)
{
long long a;
@@ -182,6 +267,21 @@ static inline long long atomic64_add(long long i, atomic64_t *v)
}
/**
+ * atomic64_add_unchecked - add integer to atomic64 variable
+ * @i: integer value to add
+ * @v: pointer to type atomic64_unchecked_t
+ *
+ * Atomically adds @i to @v.
+ */
+static inline long long atomic64_add_unchecked(long long i, atomic64_unchecked_t *v)
+{
+ __alternative_atomic64(add_unchecked, add_return_unchecked,
+ ASM_OUTPUT2("+A" (i), "+c" (v)),
+ ASM_NO_INPUT_CLOBBER("memory"));
+ return i;
+}
+
+/**
* atomic64_sub - subtract the atomic64 variable
* @i: integer value to subtract
* @v: pointer to type atomic64_t
diff --git a/arch/x86/include/asm/atomic64_64.h b/arch/x86/include/asm/atomic64_64.h
index b965f9e..fc9be92 100644
--- a/arch/x86/include/asm/atomic64_64.h
+++ b/arch/x86/include/asm/atomic64_64.h
@@ -22,6 +22,18 @@ static inline long atomic64_read(const atomic64_t *v)
}
/**
+ * atomic64_read_unchecked - read atomic64 variable
+ * @v: pointer of type atomic64_unchecked_t
+ *
+ * Atomically reads the value of @v.
+ * Doesn't imply a read memory barrier.
+ */
+static inline long atomic64_read_unchecked(const atomic64_unchecked_t *v)
+{
+ return ACCESS_ONCE((v)->counter);
+}
+
+/**
* atomic64_set - set atomic64 variable
* @v: pointer to type atomic64_t
* @i: required value
@@ -34,6 +46,18 @@ static inline void atomic64_set(atomic64_t *v, long i)
}
/**
+ * atomic64_set_unchecked - set atomic64 variable
+ * @v: pointer to type atomic64_unchecked_t
+ * @i: required value
+ *
+ * Atomically sets the value of @v to @i.
+ */
+static inline void atomic64_set_unchecked(atomic64_unchecked_t *v, long i)
+{
+ v->counter = i;
+}
+
+/**
* atomic64_add - add integer to atomic64 variable
* @i: integer value to add
* @v: pointer to type atomic64_t
@@ -42,6 +66,28 @@ static inline void atomic64_set(atomic64_t *v, long i)
*/
static __always_inline void atomic64_add(long i, atomic64_t *v)
{
+ asm volatile(LOCK_PREFIX "addq %1,%0\n"
+
+#ifdef CONFIG_PAX_REFCOUNT
+ "jno 0f\n"
+ LOCK_PREFIX "subq %1,%0\n"
+ "int $4\n0:\n"
+ _ASM_EXTABLE(0b, 0b)
+#endif
+
+ : "=m" (v->counter)
+ : "er" (i), "m" (v->counter));
+}
+
+/**
+ * atomic64_add_unchecked - add integer to atomic64 variable
+ * @i: integer value to add
+ * @v: pointer to type atomic64_unchecked_t
+ *
+ * Atomically adds @i to @v.
+ */
+static __always_inline void atomic64_add_unchecked(long i, atomic64_unchecked_t *v)
+{
asm volatile(LOCK_PREFIX "addq %1,%0"
: "=m" (v->counter)
: "er" (i), "m" (v->counter));
@@ -56,7 +102,29 @@ static __always_inline void atomic64_add(long i, atomic64_t *v)
*/
static inline void atomic64_sub(long i, atomic64_t *v)
{
- asm volatile(LOCK_PREFIX "subq %1,%0"
+ asm volatile(LOCK_PREFIX "subq %1,%0\n"
+
+#ifdef CONFIG_PAX_REFCOUNT
+ "jno 0f\n"
+ LOCK_PREFIX "addq %1,%0\n"
+ "int $4\n0:\n"
+ _ASM_EXTABLE(0b, 0b)
+#endif
+
+ : "=m" (v->counter)
+ : "er" (i), "m" (v->counter));
+}
+
+/**
+ * atomic64_sub_unchecked - subtract the atomic64 variable
+ * @i: integer value to subtract
+ * @v: pointer to type atomic64_unchecked_t
+ *
+ * Atomically subtracts @i from @v.
+ */
+static inline void atomic64_sub_unchecked(long i, atomic64_unchecked_t *v)
+{
+ asm volatile(LOCK_PREFIX "subq %1,%0\n"
: "=m" (v->counter)
: "er" (i), "m" (v->counter));
}
@@ -72,7 +140,7 @@ static inline void atomic64_sub(long i, atomic64_t *v)
*/
static inline int atomic64_sub_and_test(long i, atomic64_t *v)
{
- GEN_BINARY_RMWcc(LOCK_PREFIX "subq", v->counter, "er", i, "%0", "e");
+ GEN_BINARY_RMWcc(LOCK_PREFIX "subq", LOCK_PREFIX "addq", v->counter, "er", i, "%0", "e");
}
/**
@@ -83,6 +151,27 @@ static inline int atomic64_sub_and_test(long i, atomic64_t *v)
*/
static __always_inline void atomic64_inc(atomic64_t *v)
{
+ asm volatile(LOCK_PREFIX "incq %0\n"
+
+#ifdef CONFIG_PAX_REFCOUNT
+ "jno 0f\n"
+ LOCK_PREFIX "decq %0\n"
+ "int $4\n0:\n"
+ _ASM_EXTABLE(0b, 0b)
+#endif
+
+ : "=m" (v->counter)
+ : "m" (v->counter));
+}
+
+/**
+ * atomic64_inc_unchecked - increment atomic64 variable
+ * @v: pointer to type atomic64_unchecked_t
+ *
+ * Atomically increments @v by 1.
+ */
+static __always_inline void atomic64_inc_unchecked(atomic64_unchecked_t *v)
+{
asm volatile(LOCK_PREFIX "incq %0"
: "=m" (v->counter)
: "m" (v->counter));
@@ -96,7 +185,28 @@ static __always_inline void atomic64_inc(atomic64_t *v)
*/
static __always_inline void atomic64_dec(atomic64_t *v)
{
- asm volatile(LOCK_PREFIX "decq %0"
+ asm volatile(LOCK_PREFIX "decq %0\n"
+
+#ifdef CONFIG_PAX_REFCOUNT
+ "jno 0f\n"
+ LOCK_PREFIX "incq %0\n"
+ "int $4\n0:\n"
+ _ASM_EXTABLE(0b, 0b)
+#endif
+
+ : "=m" (v->counter)
+ : "m" (v->counter));
+}
+
+/**
+ * atomic64_dec_unchecked - decrement atomic64 variable
+ * @v: pointer to type atomic64_t
+ *
+ * Atomically decrements @v by 1.
+ */
+static __always_inline void atomic64_dec_unchecked(atomic64_unchecked_t *v)
+{
+ asm volatile(LOCK_PREFIX "decq %0\n"
: "=m" (v->counter)
: "m" (v->counter));
}
@@ -111,7 +221,7 @@ static __always_inline void atomic64_dec(atomic64_t *v)
*/
static inline int atomic64_dec_and_test(atomic64_t *v)
{
- GEN_UNARY_RMWcc(LOCK_PREFIX "decq", v->counter, "%0", "e");
+ GEN_UNARY_RMWcc(LOCK_PREFIX "decq", LOCK_PREFIX "incq", v->counter, "%0", "e");
}
/**
@@ -124,7 +234,7 @@ static inline int atomic64_dec_and_test(atomic64_t *v)
*/
static inline int atomic64_inc_and_test(atomic64_t *v)
{
- GEN_UNARY_RMWcc(LOCK_PREFIX "incq", v->counter, "%0", "e");
+ GEN_UNARY_RMWcc(LOCK_PREFIX "incq", LOCK_PREFIX "decq", v->counter, "%0", "e");
}
/**
@@ -138,7 +248,7 @@ static inline int atomic64_inc_and_test(atomic64_t *v)
*/
static inline int atomic64_add_negative(long i, atomic64_t *v)
{
- GEN_BINARY_RMWcc(LOCK_PREFIX "addq", v->counter, "er", i, "%0", "s");
+ GEN_BINARY_RMWcc(LOCK_PREFIX "addq", LOCK_PREFIX "subq", v->counter, "er", i, "%0", "s");
}
/**
@@ -150,6 +260,18 @@ static inline int atomic64_add_negative(long i, atomic64_t *v)
*/
static __always_inline long atomic64_add_return(long i, atomic64_t *v)
{
+ return i + xadd_check_overflow(&v->counter, i);
+}
+
+/**
+ * atomic64_add_return_unchecked - add and return
+ * @i: integer value to add
+ * @v: pointer to type atomic64_unchecked_t
+ *
+ * Atomically adds @i to @v and returns @i + @v
+ */
+static __always_inline long atomic64_add_return_unchecked(long i, atomic64_unchecked_t *v)
+{
return i + xadd(&v->counter, i);
}
@@ -159,6 +281,10 @@ static inline long atomic64_sub_return(long i, atomic64_t *v)
}
#define atomic64_inc_return(v) (atomic64_add_return(1, (v)))
+static inline long atomic64_inc_return_unchecked(atomic64_unchecked_t *v)
+{
+ return atomic64_add_return_unchecked(1, v);
+}
#define atomic64_dec_return(v) (atomic64_sub_return(1, (v)))
static inline long atomic64_cmpxchg(atomic64_t *v, long old, long new)
@@ -166,6 +292,11 @@ static inline long atomic64_cmpxchg(atomic64_t *v, long old, long new)
return cmpxchg(&v->counter, old, new);
}
+static inline long atomic64_cmpxchg_unchecked(atomic64_unchecked_t *v, long old, long new)
+{
+ return cmpxchg(&v->counter, old, new);
+}
+
static inline long atomic64_xchg(atomic64_t *v, long new)
{
return xchg(&v->counter, new);
@@ -182,17 +313,30 @@ static inline long atomic64_xchg(atomic64_t *v, long new)
*/
static inline int atomic64_add_unless(atomic64_t *v, long a, long u)
{
- long c, old;
+ long c, old, new;
c = atomic64_read(v);
for (;;) {
- if (unlikely(c == (u)))
+ if (unlikely(c == u))
break;
- old = atomic64_cmpxchg((v), c, c + (a));
+
+ asm volatile("add %2,%0\n"
+
+#ifdef CONFIG_PAX_REFCOUNT
+ "jno 0f\n"
+ "sub %2,%0\n"
+ "int $4\n0:\n"
+ _ASM_EXTABLE(0b, 0b)
+#endif
+
+ : "=r" (new)
+ : "0" (c), "ir" (a));
+
+ old = atomic64_cmpxchg(v, c, new);
if (likely(old == c))
break;
c = old;
}
- return c != (u);
+ return c != u;
}
#define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0)
diff --git a/arch/x86/include/asm/bitops.h b/arch/x86/include/asm/bitops.h
index cfe3b95..69126a1 100644
--- a/arch/x86/include/asm/bitops.h
+++ b/arch/x86/include/asm/bitops.h
@@ -203,7 +203,7 @@ static inline void change_bit(long nr, volatile unsigned long *addr)
*/
static inline int test_and_set_bit(long nr, volatile unsigned long *addr)
{
- GEN_BINARY_RMWcc(LOCK_PREFIX "bts", *addr, "Ir", nr, "%0", "c");
+ GEN_BINARY_RMWcc_unchecked(LOCK_PREFIX "bts", *addr, "Ir", nr, "%0", "c");
}
/**
@@ -249,7 +249,7 @@ static inline int __test_and_set_bit(long nr, volatile unsigned long *addr)
*/
static inline int test_and_clear_bit(long nr, volatile unsigned long *addr)
{
- GEN_BINARY_RMWcc(LOCK_PREFIX "btr", *addr, "Ir", nr, "%0", "c");
+ GEN_BINARY_RMWcc_unchecked(LOCK_PREFIX "btr", *addr, "Ir", nr, "%0", "c");
}
/**
@@ -302,7 +302,7 @@ static inline int __test_and_change_bit(long nr, volatile unsigned long *addr)
*/
static inline int test_and_change_bit(long nr, volatile unsigned long *addr)
{
- GEN_BINARY_RMWcc(LOCK_PREFIX "btc", *addr, "Ir", nr, "%0", "c");
+ GEN_BINARY_RMWcc_unchecked(LOCK_PREFIX "btc", *addr, "Ir", nr, "%0", "c");
}
static __always_inline int constant_test_bit(long nr, const volatile unsigned long *addr)
diff --git a/arch/x86/include/asm/cmpxchg.h b/arch/x86/include/asm/cmpxchg.h
index ad19841..0784041 100644
--- a/arch/x86/include/asm/cmpxchg.h
+++ b/arch/x86/include/asm/cmpxchg.h
@@ -14,8 +14,12 @@ extern void __cmpxchg_wrong_size(void)
__compiletime_error("Bad argument size for cmpxchg");
extern void __xadd_wrong_size(void)
__compiletime_error("Bad argument size for xadd");
+extern void __xadd_check_overflow_wrong_size(void)
+ __compiletime_error("Bad argument size for xadd_check_overflow");
extern void __add_wrong_size(void)
__compiletime_error("Bad argument size for add");
+extern void __add_check_overflow_wrong_size(void)
+ __compiletime_error("Bad argument size for add_check_overflow");
/*
* Constants for operation sizes. On 32-bit, the 64-bit size it set to
@@ -67,6 +71,38 @@ extern void __add_wrong_size(void)
__ret; \
})
+#ifdef CONFIG_PAX_REFCOUNT
+#define __xchg_op_check_overflow(ptr, arg, op, lock) \
+ ({ \
+ __typeof__ (*(ptr)) __ret = (arg); \
+ switch (sizeof(*(ptr))) { \
+ case __X86_CASE_L: \
+ asm volatile (lock #op "l %0, %1\n" \
+ "jno 0f\n" \
+ "mov %0,%1\n" \
+ "int $4\n0:\n" \
+ _ASM_EXTABLE(0b, 0b) \
+ : "+r" (__ret), "+m" (*(ptr)) \
+ : : "memory", "cc"); \
+ break; \
+ case __X86_CASE_Q: \
+ asm volatile (lock #op "q %q0, %1\n" \
+ "jno 0f\n" \
+ "mov %0,%1\n" \
+ "int $4\n0:\n" \
+ _ASM_EXTABLE(0b, 0b) \
+ : "+r" (__ret), "+m" (*(ptr)) \
+ : : "memory", "cc"); \
+ break; \
+ default: \
+ __ ## op ## _check_overflow_wrong_size(); \
+ } \
+ __ret; \
+ })
+#else
+#define __xchg_op_check_overflow(ptr, arg, op, lock) __xchg_op(ptr, arg, op, lock)
+#endif
+
/*
* Note: no "lock" prefix even on SMP: xchg always implies lock anyway.
* Since this is generally used to protect other memory information, we
@@ -165,6 +201,9 @@ extern void __add_wrong_size(void)
#define xadd_sync(ptr, inc) __xadd((ptr), (inc), "lock; ")
#define xadd_local(ptr, inc) __xadd((ptr), (inc), "")
+#define __xadd_check_overflow(ptr, inc, lock) __xchg_op_check_overflow((ptr), (inc), xadd, lock)
+#define xadd_check_overflow(ptr, inc) __xadd_check_overflow((ptr), (inc), LOCK_PREFIX)
+
#define __add(ptr, inc, lock) \
({ \
__typeof__ (*(ptr)) __ret = (inc); \
diff --git a/arch/x86/include/asm/hw_irq.h b/arch/x86/include/asm/hw_irq.h
index 6615032..9c233be 100644
--- a/arch/x86/include/asm/hw_irq.h
+++ b/arch/x86/include/asm/hw_irq.h
@@ -158,8 +158,8 @@ static inline void unlock_vector_lock(void) {}
#endif /* CONFIG_X86_LOCAL_APIC */
/* Statistics */
-extern atomic_t irq_err_count;
-extern atomic_t irq_mis_count;
+extern atomic_unchecked_t irq_err_count;
+extern atomic_unchecked_t irq_mis_count;
extern void elcr_set_level_irq(unsigned int irq);
diff --git a/arch/x86/include/asm/local.h b/arch/x86/include/asm/local.h
index 4ad6560..75c7bdd 100644
--- a/arch/x86/include/asm/local.h
+++ b/arch/x86/include/asm/local.h
@@ -10,33 +10,97 @@ typedef struct {
atomic_long_t a;
} local_t;
+typedef struct {
+ atomic_long_unchecked_t a;
+} local_unchecked_t;
+
#define LOCAL_INIT(i) { ATOMIC_LONG_INIT(i) }
#define local_read(l) atomic_long_read(&(l)->a)
+#define local_read_unchecked(l) atomic_long_read_unchecked(&(l)->a)
#define local_set(l, i) atomic_long_set(&(l)->a, (i))
+#define local_set_unchecked(l, i) atomic_long_set_unchecked(&(l)->a, (i))
static inline void local_inc(local_t *l)
{
- asm volatile(_ASM_INC "%0"
+ asm volatile(_ASM_INC "%0\n"
+
+#ifdef CONFIG_PAX_REFCOUNT
+ "jno 0f\n"
+ _ASM_DEC "%0\n"
+ "int $4\n0:\n"
+ _ASM_EXTABLE(0b, 0b)
+#endif
+
+ : "+m" (l->a.counter));
+}
+
+static inline void local_inc_unchecked(local_unchecked_t *l)
+{
+ asm volatile(_ASM_INC "%0\n"
: "+m" (l->a.counter));
}
static inline void local_dec(local_t *l)
{
- asm volatile(_ASM_DEC "%0"
+ asm volatile(_ASM_DEC "%0\n"
+
+#ifdef CONFIG_PAX_REFCOUNT
+ "jno 0f\n"
+ _ASM_INC "%0\n"
+ "int $4\n0:\n"
+ _ASM_EXTABLE(0b, 0b)
+#endif
+
+ : "+m" (l->a.counter));
+}
+
+static inline void local_dec_unchecked(local_unchecked_t *l)
+{
+ asm volatile(_ASM_DEC "%0\n"
: "+m" (l->a.counter));
}
static inline void local_add(long i, local_t *l)
{
- asm volatile(_ASM_ADD "%1,%0"
+ asm volatile(_ASM_ADD "%1,%0\n"
+
+#ifdef CONFIG_PAX_REFCOUNT
+ "jno 0f\n"
+ _ASM_SUB "%1,%0\n"
+ "int $4\n0:\n"
+ _ASM_EXTABLE(0b, 0b)
+#endif
+
+ : "+m" (l->a.counter)
+ : "ir" (i));
+}
+
+static inline void local_add_unchecked(long i, local_unchecked_t *l)
+{
+ asm volatile(_ASM_ADD "%1,%0\n"
: "+m" (l->a.counter)
: "ir" (i));
}
static inline void local_sub(long i, local_t *l)
{
- asm volatile(_ASM_SUB "%1,%0"
+ asm volatile(_ASM_SUB "%1,%0\n"
+
+#ifdef CONFIG_PAX_REFCOUNT
+ "jno 0f\n"
+ _ASM_ADD "%1,%0\n"
+ "int $4\n0:\n"
+ _ASM_EXTABLE(0b, 0b)
+#endif
+
+ : "+m" (l->a.counter)
+ : "ir" (i));
+}
+
+static inline void local_sub_unchecked(long i, local_unchecked_t *l)
+{
+ asm volatile(_ASM_SUB "%1,%0\n"
: "+m" (l->a.counter)
: "ir" (i));
}
@@ -52,7 +116,7 @@ static inline void local_sub(long i, local_t *l)
*/
static inline int local_sub_and_test(long i, local_t *l)
{
- GEN_BINARY_RMWcc(_ASM_SUB, l->a.counter, "er", i, "%0", "e");
+ GEN_BINARY_RMWcc(_ASM_SUB, _ASM_ADD, l->a.counter, "er", i, "%0", "e");
}
/**
@@ -65,7 +129,7 @@ static inline int local_sub_and_test(long i, local_t *l)
*/
static inline int local_dec_and_test(local_t *l)
{
- GEN_UNARY_RMWcc(_ASM_DEC, l->a.counter, "%0", "e");
+ GEN_UNARY_RMWcc(_ASM_DEC, _ASM_INC, l->a.counter, "%0", "e");
}
/**
@@ -78,7 +142,7 @@ static inline int local_dec_and_test(local_t *l)
*/
static inline int local_inc_and_test(local_t *l)
{
- GEN_UNARY_RMWcc(_ASM_INC, l->a.counter, "%0", "e");
+ GEN_UNARY_RMWcc(_ASM_INC, _ASM_DEC, l->a.counter, "%0", "e");
}
/**
@@ -92,7 +156,7 @@ static inline int local_inc_and_test(local_t *l)
*/
static inline int local_add_negative(long i, local_t *l)
{
- GEN_BINARY_RMWcc(_ASM_ADD, l->a.counter, "er", i, "%0", "s");
+ GEN_BINARY_RMWcc(_ASM_ADD, _ASM_SUB, l->a.counter, "er", i, "%0", "s");
}
/**
@@ -105,6 +169,30 @@ static inline int local_add_negative(long i, local_t *l)
static inline long local_add_return(long i, local_t *l)
{
long __i = i;
+ asm volatile(_ASM_XADD "%0, %1\n"
+
+#ifdef CONFIG_PAX_REFCOUNT
+ "jno 0f\n"
+ _ASM_MOV "%0,%1\n"
+ "int $4\n0:\n"
+ _ASM_EXTABLE(0b, 0b)
+#endif
+
+ : "+r" (i), "+m" (l->a.counter)
+ : : "memory");
+ return i + __i;
+}
+
+/**
+ * local_add_return_unchecked - add and return
+ * @i: integer value to add
+ * @l: pointer to type local_unchecked_t
+ *
+ * Atomically adds @i to @l and returns @i + @l
+ */
+static inline long local_add_return_unchecked(long i, local_unchecked_t *l)
+{
+ long __i = i;
asm volatile(_ASM_XADD "%0, %1;"
: "+r" (i), "+m" (l->a.counter)
: : "memory");
@@ -121,6 +209,8 @@ static inline long local_sub_return(long i, local_t *l)
#define local_cmpxchg(l, o, n) \
(cmpxchg_local(&((l)->a.counter), (o), (n)))
+#define local_cmpxchg_unchecked(l, o, n) \
+ (cmpxchg_local(&((l)->a.counter), (o), (n)))
/* Always has a lock prefix */
#define local_xchg(l, n) (xchg(&((l)->a.counter), (n)))
diff --git a/arch/x86/include/asm/preempt.h b/arch/x86/include/asm/preempt.h
index b12f810..aedcc13 100644
--- a/arch/x86/include/asm/preempt.h
+++ b/arch/x86/include/asm/preempt.h
@@ -84,7 +84,7 @@ static __always_inline void __preempt_count_sub(int val)
*/
static __always_inline bool __preempt_count_dec_and_test(void)
{
- GEN_UNARY_RMWcc("decl", __preempt_count, __percpu_arg(0), "e");
+ GEN_UNARY_RMWcc("decl", "incl", __preempt_count, __percpu_arg(0), "e");
}
/*
diff --git a/arch/x86/include/asm/rmwcc.h b/arch/x86/include/asm/rmwcc.h
index 8f7866a..e442f20 100644
--- a/arch/x86/include/asm/rmwcc.h
+++ b/arch/x86/include/asm/rmwcc.h
@@ -3,7 +3,34 @@
#ifdef CC_HAVE_ASM_GOTO
-#define __GEN_RMWcc(fullop, var, cc, ...) \
+#ifdef CONFIG_PAX_REFCOUNT
+#define __GEN_RMWcc(fullop, fullantiop, var, cc, ...) \
+do { \
+ asm_volatile_goto (fullop \
+ ";jno 0f\n" \
+ fullantiop \
+ ";int $4\n0:\n" \
+ _ASM_EXTABLE(0b, 0b) \
+ ";j" cc " %l[cc_label]" \
+ : : "m" (var), ## __VA_ARGS__ \
+ : "memory" : cc_label); \
+ return 0; \
+cc_label: \
+ return 1; \
+} while (0)
+#else
+#define __GEN_RMWcc(fullop, fullantiop, var, cc, ...) \
+do { \
+ asm_volatile_goto (fullop ";j" cc " %l[cc_label]" \
+ : : "m" (var), ## __VA_ARGS__ \
+ : "memory" : cc_label); \
+ return 0; \
+cc_label: \
+ return 1; \
+} while (0)
+#endif
+
+#define __GEN_RMWcc_unchecked(fullop, var, cc, ...) \
do { \
asm_volatile_goto (fullop "; j" cc " %l[cc_label]" \
: : "m" (var), ## __VA_ARGS__ \
@@ -13,15 +40,46 @@ cc_label: \
return 1; \
} while (0)
-#define GEN_UNARY_RMWcc(op, var, arg0, cc) \
- __GEN_RMWcc(op " " arg0, var, cc)
+#define GEN_UNARY_RMWcc(op, antiop, var, arg0, cc) \
+ __GEN_RMWcc(op " " arg0, antiop " " arg0, var, cc)
+
+#define GEN_UNARY_RMWcc_unchecked(op, var, arg0, cc) \
+ __GEN_RMWcc_unchecked(op " " arg0, var, cc)
+
+#define GEN_BINARY_RMWcc(op, antiop, var, vcon, val, arg0, cc) \
+ __GEN_RMWcc(op " %1, " arg0, antiop " %1, " arg0, var, cc, vcon (val))
-#define GEN_BINARY_RMWcc(op, var, vcon, val, arg0, cc) \
- __GEN_RMWcc(op " %1, " arg0, var, cc, vcon (val))
+#define GEN_BINARY_RMWcc_unchecked(op, var, vcon, val, arg0, cc) \
+ __GEN_RMWcc_unchecked(op " %1, " arg0, var, cc, vcon (val))
#else /* !CC_HAVE_ASM_GOTO */
-#define __GEN_RMWcc(fullop, var, cc, ...) \
+#ifdef CONFIG_PAX_REFCOUNT
+#define __GEN_RMWcc(fullop, fullantiop, var, cc, ...) \
+do { \
+ char c; \
+ asm volatile (fullop \
+ ";jno 0f\n" \
+ fullantiop \
+ ";int $4\n0:\n" \
+ _ASM_EXTABLE(0b, 0b) \
+ "; set" cc " %1" \
+ : "+m" (var), "=qm" (c) \
+ : __VA_ARGS__ : "memory"); \
+ return c != 0; \
+} while (0)
+#else
+#define __GEN_RMWcc(fullop, fullantiop, var, cc, ...) \
+do { \
+ char c; \
+ asm volatile (fullop "; set" cc " %1" \
+ : "+m" (var), "=qm" (c) \
+ : __VA_ARGS__ : "memory"); \
+ return c != 0; \
+} while (0)
+#endif
+
+#define __GEN_RMWcc_unchecked(fullop, var, cc, ...) \
do { \
char c; \
asm volatile (fullop "; set" cc " %1" \
@@ -30,11 +88,17 @@ do { \
return c != 0; \
} while (0)
-#define GEN_UNARY_RMWcc(op, var, arg0, cc) \
- __GEN_RMWcc(op " " arg0, var, cc)
+#define GEN_UNARY_RMWcc(op, antiop, var, arg0, cc) \
+ __GEN_RMWcc(op " " arg0, antiop " " arg0, var, cc)
+
+#define GEN_UNARY_RMWcc_unchecked(op, var, arg0, cc) \
+ __GEN_RMWcc_unchecked(op " " arg0, var, cc)
+
+#define GEN_BINARY_RMWcc(op, antiop, var, vcon, val, arg0, cc) \
+ __GEN_RMWcc(op " %2, " arg0, antiop " %2, " arg0, var, cc, vcon (val))
-#define GEN_BINARY_RMWcc(op, var, vcon, val, arg0, cc) \
- __GEN_RMWcc(op " %2, " arg0, var, cc, vcon (val))
+#define GEN_BINARY_RMWcc_unchecked(op, var, vcon, val, arg0, cc) \
+ __GEN_RMWcc_unchecked(op " %2, " arg0, var, cc, vcon (val))
#endif /* CC_HAVE_ASM_GOTO */
diff --git a/arch/x86/include/asm/rwsem.h b/arch/x86/include/asm/rwsem.h
index cad82c9..2e5c5c1 100644
--- a/arch/x86/include/asm/rwsem.h
+++ b/arch/x86/include/asm/rwsem.h
@@ -64,6 +64,14 @@ static inline void __down_read(struct rw_semaphore *sem)
{
asm volatile("# beginning down_read\n\t"
LOCK_PREFIX _ASM_INC "(%1)\n\t"
+
+#ifdef CONFIG_PAX_REFCOUNT
+ "jno 0f\n"
+ LOCK_PREFIX _ASM_DEC "(%1)\n"
+ "int $4\n0:\n"
+ _ASM_EXTABLE(0b, 0b)
+#endif
+
/* adds 0x00000001 */
" jns 1f\n"
" call call_rwsem_down_read_failed\n"
@@ -85,6 +93,14 @@ static inline int __down_read_trylock(struct rw_semaphore *sem)
"1:\n\t"
" mov %1,%2\n\t"
" add %3,%2\n\t"
+
+#ifdef CONFIG_PAX_REFCOUNT
+ "jno 0f\n"
+ "sub %3,%2\n"
+ "int $4\n0:\n"
+ _ASM_EXTABLE(0b, 0b)
+#endif
+
" jle 2f\n\t"
LOCK_PREFIX " cmpxchg %2,%0\n\t"
" jnz 1b\n\t"
@@ -104,6 +120,14 @@ static inline void __down_write_nested(struct rw_semaphore *sem, int subclass)
long tmp;
asm volatile("# beginning down_write\n\t"
LOCK_PREFIX " xadd %1,(%2)\n\t"
+
+#ifdef CONFIG_PAX_REFCOUNT
+ "jno 0f\n"
+ "mov %1,(%2)\n"
+ "int $4\n0:\n"
+ _ASM_EXTABLE(0b, 0b)
+#endif
+
/* adds 0xffff0001, returns the old value */
" test " __ASM_SEL(%w1,%k1) "," __ASM_SEL(%w1,%k1) "\n\t"
/* was the active mask 0 before? */
@@ -155,6 +179,14 @@ static inline void __up_read(struct rw_semaphore *sem)
long tmp;
asm volatile("# beginning __up_read\n\t"
LOCK_PREFIX " xadd %1,(%2)\n\t"
+
+#ifdef CONFIG_PAX_REFCOUNT
+ "jno 0f\n"
+ "mov %1,(%2)\n"
+ "int $4\n0:\n"
+ _ASM_EXTABLE(0b, 0b)
+#endif
+
/* subtracts 1, returns the old value */
" jns 1f\n\t"
" call call_rwsem_wake\n" /* expects old value in %edx */
@@ -173,6 +205,14 @@ static inline void __up_write(struct rw_semaphore *sem)
long tmp;
asm volatile("# beginning __up_write\n\t"
LOCK_PREFIX " xadd %1,(%2)\n\t"
+
+#ifdef CONFIG_PAX_REFCOUNT
+ "jno 0f\n"
+ "mov %1,(%2)\n"
+ "int $4\n0:\n"
+ _ASM_EXTABLE(0b, 0b)
+#endif
+
/* subtracts 0xffff0001, returns the old value */
" jns 1f\n\t"
" call call_rwsem_wake\n" /* expects old value in %edx */
@@ -190,6 +230,14 @@ static inline void __downgrade_write(struct rw_semaphore *sem)
{
asm volatile("# beginning __downgrade_write\n\t"
LOCK_PREFIX _ASM_ADD "%2,(%1)\n\t"
+
+#ifdef CONFIG_PAX_REFCOUNT
+ "jno 0f\n"
+ LOCK_PREFIX _ASM_SUB "%2,(%1)\n"
+ "int $4\n0:\n"
+ _ASM_EXTABLE(0b, 0b)
+#endif
+
/*
* transitions 0xZZZZ0001 -> 0xYYYY0001 (i386)
* 0xZZZZZZZZ00000001 -> 0xYYYYYYYY00000001 (x86_64)
@@ -208,7 +256,15 @@ static inline void __downgrade_write(struct rw_semaphore *sem)
*/
static inline void rwsem_atomic_add(long delta, struct rw_semaphore *sem)
{
- asm volatile(LOCK_PREFIX _ASM_ADD "%1,%0"
+ asm volatile(LOCK_PREFIX _ASM_ADD "%1,%0\n"
+
+#ifdef CONFIG_PAX_REFCOUNT
+ "jno 0f\n"
+ LOCK_PREFIX _ASM_SUB "%1,%0\n"
+ "int $4\n0:\n"
+ _ASM_EXTABLE(0b, 0b)
+#endif
+
: "+m" (sem->count)
: "er" (delta));
}
@@ -218,7 +274,7 @@ static inline void rwsem_atomic_add(long delta, struct rw_semaphore *sem)
*/
static inline long rwsem_atomic_update(long delta, struct rw_semaphore *sem)
{
- return delta + xadd(&sem->count, delta);
+ return delta + xadd_check_overflow(&sem->count, delta);
}
#endif /* __KERNEL__ */
diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c
index 307a498..60e3759 100644
--- a/arch/x86/kernel/apic/apic.c
+++ b/arch/x86/kernel/apic/apic.c
@@ -1864,7 +1864,7 @@ static inline void __smp_error_interrupt(struct pt_regs *regs)
apic_write(APIC_ESR, 0);
v = apic_read(APIC_ESR);
ack_APIC_irq();
- atomic_inc(&irq_err_count);
+ atomic_inc_unchecked(&irq_err_count);
apic_printk(APIC_DEBUG, KERN_DEBUG "APIC error on CPU%d: %02x",
smp_processor_id(), v);
diff --git a/arch/x86/kernel/apic/io_apic.c b/arch/x86/kernel/apic/io_apic.c
index 11b46d9..473648d 100644
--- a/arch/x86/kernel/apic/io_apic.c
+++ b/arch/x86/kernel/apic/io_apic.c
@@ -1682,7 +1682,7 @@ static unsigned int startup_ioapic_irq(struct irq_data *data)
return was_pending;
}
-atomic_t irq_mis_count;
+atomic_unchecked_t irq_mis_count;
#ifdef CONFIG_GENERIC_PENDING_IRQ
static bool io_apic_level_ack_pending(struct mp_chip_data *data)
@@ -1821,7 +1821,7 @@ static void ioapic_ack_level(struct irq_data *irq_data)
* at the cpu.
*/
if (!(v & (1 << (i & 0x1f)))) {
- atomic_inc(&irq_mis_count);
+ atomic_inc_unchecked(&irq_mis_count);
eoi_ioapic_pin(cfg->vector, irq_data->chip_data);
}
diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c
index df919ff..22f7362 100644
--- a/arch/x86/kernel/cpu/mcheck/mce.c
+++ b/arch/x86/kernel/cpu/mcheck/mce.c
@@ -292,10 +292,10 @@ static void print_mce(struct mce *m)
#define PANIC_TIMEOUT 5 /* 5 seconds */
-static atomic_t mce_panicked;
+static atomic_unchecked_t mce_panicked;
static int fake_panic;
-static atomic_t mce_fake_panicked;
+static atomic_unchecked_t mce_fake_panicked;
/* Panic in progress. Enable interrupts and wait for final IPI */
static void wait_for_panic(void)
@@ -319,7 +319,7 @@ static void mce_panic(const char *msg, struct mce *final, char *exp)
/*
* Make sure only one CPU runs in machine check panic
*/
- if (atomic_inc_return(&mce_panicked) > 1)
+ if (atomic_inc_return_unchecked(&mce_panicked) > 1)
wait_for_panic();
barrier();
@@ -327,7 +327,7 @@ static void mce_panic(const char *msg, struct mce *final, char *exp)
console_verbose();
} else {
/* Don't log too much for fake panic */
- if (atomic_inc_return(&mce_fake_panicked) > 1)
+ if (atomic_inc_return_unchecked(&mce_fake_panicked) > 1)
return;
}
/* First print corrected ones that are still unlogged */
@@ -752,7 +752,7 @@ static int mce_timed_out(u64 *t, const char *msg)
* might have been modified by someone else.
*/
rmb();
- if (atomic_read(&mce_panicked))
+ if (atomic_read_unchecked(&mce_panicked))
wait_for_panic();
if (!mca_cfg.monarch_timeout)
goto out;
@@ -2555,7 +2555,7 @@ struct dentry *mce_get_debugfs_dir(void)
static void mce_reset(void)
{
cpu_missing = 0;
- atomic_set(&mce_fake_panicked, 0);
+ atomic_set_unchecked(&mce_fake_panicked, 0);
atomic_set(&mce_executing, 0);
atomic_set(&mce_callin, 0);
atomic_set(&global_nwo, 0);
diff --git a/arch/x86/kernel/i8259.c b/arch/x86/kernel/i8259.c
index 16cb827..989dbb2 100644
--- a/arch/x86/kernel/i8259.c
+++ b/arch/x86/kernel/i8259.c
@@ -208,7 +208,7 @@ spurious_8259A_irq:
"spurious 8259A interrupt: IRQ%d.\n", irq);
spurious_irq_mask |= irqmask;
}
- atomic_inc(&irq_err_count);
+ atomic_inc_unchecked(&irq_err_count);
/*
* Theoretically we do not have to handle this IRQ,
* but in Linux this does not cause problems and is
diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c
index c7dfe1b..146f63c 100644
--- a/arch/x86/kernel/irq.c
+++ b/arch/x86/kernel/irq.c
@@ -28,7 +28,7 @@ EXPORT_PER_CPU_SYMBOL(irq_stat);
DEFINE_PER_CPU(struct pt_regs *, irq_regs);
EXPORT_PER_CPU_SYMBOL(irq_regs);
-atomic_t irq_err_count;
+atomic_unchecked_t irq_err_count;
/* Function pointer for generic interrupt vector handling */
void (*x86_platform_ipi_callback)(void) = NULL;
@@ -144,9 +144,9 @@ int arch_show_interrupts(struct seq_file *p, int prec)
seq_printf(p, "%10u ", irq_stats(j)->irq_hv_callback_count);
seq_puts(p, " Hypervisor callback interrupts\n");
#endif
- seq_printf(p, "%*s: %10u\n", prec, "ERR", atomic_read(&irq_err_count));
+ seq_printf(p, "%*s: %10u\n", prec, "ERR", atomic_read_unchecked(&irq_err_count));
#if defined(CONFIG_X86_IO_APIC)
- seq_printf(p, "%*s: %10u\n", prec, "MIS", atomic_read(&irq_mis_count));
+ seq_printf(p, "%*s: %10u\n", prec, "MIS", atomic_read_unchecked(&irq_mis_count));
#endif
#ifdef CONFIG_HAVE_KVM
seq_printf(p, "%*s: ", prec, "PIN");
@@ -198,7 +198,7 @@ u64 arch_irq_stat_cpu(unsigned int cpu)
u64 arch_irq_stat(void)
{
- u64 sum = atomic_read(&irq_err_count);
+ u64 sum = atomic_read_unchecked(&irq_err_count);
return sum;
}
diff --git a/arch/x86/kernel/kgdb.c b/arch/x86/kernel/kgdb.c
index d6178d9..cb20db8 100644
--- a/arch/x86/kernel/kgdb.c
+++ b/arch/x86/kernel/kgdb.c
@@ -475,12 +475,12 @@ int kgdb_arch_handle_exception(int e_vector, int signo, int err_code,
case 'k':
/* clear the trace bit */
linux_regs->flags &= ~X86_EFLAGS_TF;
- atomic_set(&kgdb_cpu_doing_single_step, -1);
+ atomic_set_unchecked(&kgdb_cpu_doing_single_step, -1);
/* set the trace bit if we're stepping */
if (remcomInBuffer[0] == 's') {
linux_regs->flags |= X86_EFLAGS_TF;
- atomic_set(&kgdb_cpu_doing_single_step,
+ atomic_set_unchecked(&kgdb_cpu_doing_single_step,
raw_smp_processor_id());
}
@@ -545,7 +545,7 @@ static int __kgdb_notify(struct die_args *args, unsigned long cmd)
switch (cmd) {
case DIE_DEBUG:
- if (atomic_read(&kgdb_cpu_doing_single_step) != -1) {
+ if (atomic_read_unchecked(&kgdb_cpu_doing_single_step) != -1) {
if (user_mode(regs))
return single_step_cont(regs, args);
break;
diff --git a/arch/x86/kernel/pvclock.c b/arch/x86/kernel/pvclock.c
index 2f355d2..e75ed0a 100644
--- a/arch/x86/kernel/pvclock.c
+++ b/arch/x86/kernel/pvclock.c
@@ -51,11 +51,11 @@ void pvclock_touch_watchdogs(void)
reset_hung_task_detector();
}
-static atomic64_t last_value = ATOMIC64_INIT(0);
+static atomic64_unchecked_t last_value = ATOMIC64_INIT(0);
void pvclock_resume(void)
{
- atomic64_set(&last_value, 0);
+ atomic64_set_unchecked(&last_value, 0);
}
u8 pvclock_read_flags(struct pvclock_vcpu_time_info *src)
@@ -105,11 +105,11 @@ cycle_t pvclock_clocksource_read(struct pvclock_vcpu_time_info *src)
* updating at the same time, and one of them could be slightly behind,
* making the assumption that last_value always go forward fail to hold.
*/
- last = atomic64_read(&last_value);
+ last = atomic64_read_unchecked(&last_value);
do {
if (ret < last)
return last;
- last = atomic64_cmpxchg(&last_value, last, ret);
+ last = atomic64_cmpxchg_unchecked(&last_value, last, ret);
} while (unlikely(last != ret));
return ret;
diff --git a/arch/x86/kernel/tboot.c b/arch/x86/kernel/tboot.c
index 91a4496..d1a52ee 100644
--- a/arch/x86/kernel/tboot.c
+++ b/arch/x86/kernel/tboot.c
@@ -310,7 +310,7 @@ static int tboot_extended_sleep(u8 sleep_state, u32 val_a, u32 val_b)
return -ENODEV;
}
-static atomic_t ap_wfs_count;
+static atomic_unchecked_t ap_wfs_count;
static int tboot_wait_for_aps(int num_aps)
{
@@ -334,9 +334,9 @@ static int tboot_cpu_callback(struct notifier_block *nfb, unsigned long action,
{
switch (action) {
case CPU_DYING:
- atomic_inc(&ap_wfs_count);
+ atomic_inc_unchecked(&ap_wfs_count);
if (num_online_cpus() == 1)
- if (tboot_wait_for_aps(atomic_read(&ap_wfs_count)))
+ if (tboot_wait_for_aps(atomic_read_unchecked(&ap_wfs_count)))
return NOTIFY_BAD;
break;
}
@@ -422,7 +422,7 @@ static __init int tboot_late_init(void)
tboot_create_trampoline();
- atomic_set(&ap_wfs_count, 0);
+ atomic_set_unchecked(&ap_wfs_count, 0);
register_hotcpu_notifier(&tboot_cpu_notifier);
#ifdef CONFIG_DEBUG_FS
diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c
index f579192..4328936 100644
--- a/arch/x86/kernel/traps.c
+++ b/arch/x86/kernel/traps.c
@@ -213,6 +213,12 @@ do_trap_no_signal(struct task_struct *tsk, int trapnr, char *str,
tsk->thread.trap_nr = trapnr;
die(str, regs, error_code);
}
+
+#ifdef CONFIG_PAX_REFCOUNT
+ if (trapnr == X86_TRAP_OF)
+ pax_report_refcount_overflow(regs);
+#endif
+
return 0;
}
diff --git a/arch/x86/lib/atomic64_386_32.S b/arch/x86/lib/atomic64_386_32.S
index 9b0ca8f..bb4af41 100644
--- a/arch/x86/lib/atomic64_386_32.S
+++ b/arch/x86/lib/atomic64_386_32.S
@@ -45,6 +45,10 @@ BEGIN(read)
movl (v), %eax
movl 4(v), %edx
RET_ENDP
+BEGIN(read_unchecked)
+ movl (v), %eax
+ movl 4(v), %edx
+RET_ENDP
#undef v
#define v %esi
@@ -52,6 +56,10 @@ BEGIN(set)
movl %ebx, (v)
movl %ecx, 4(v)
RET_ENDP
+BEGIN(set_unchecked)
+ movl %ebx, (v)
+ movl %ecx, 4(v)
+RET_ENDP
#undef v
#define v %esi
@@ -67,6 +75,20 @@ RET_ENDP
BEGIN(add)
addl %eax, (v)
adcl %edx, 4(v)
+
+#ifdef CONFIG_PAX_REFCOUNT
+ jno 0f
+ subl %eax, (v)
+ sbbl %edx, 4(v)
+ int $4
+0:
+ _ASM_EXTABLE(0b, 0b)
+#endif
+
+RET_ENDP
+BEGIN(add_unchecked)
+ addl %eax, (v)
+ adcl %edx, 4(v)
RET_ENDP
#undef v
@@ -74,6 +96,24 @@ RET_ENDP
BEGIN(add_return)
addl (v), %eax
adcl 4(v), %edx
+
+#ifdef CONFIG_PAX_REFCOUNT
+ into
+1234:
+ _ASM_EXTABLE(1234b, 2f)
+#endif
+
+ movl %eax, (v)
+ movl %edx, 4(v)
+
+#ifdef CONFIG_PAX_REFCOUNT
+2:
+#endif
+
+RET_ENDP
+BEGIN(add_return_unchecked)
+ addl (v), %eax
+ adcl 4(v), %edx
movl %eax, (v)
movl %edx, 4(v)
RET_ENDP
@@ -83,6 +123,20 @@ RET_ENDP
BEGIN(sub)
subl %eax, (v)
sbbl %edx, 4(v)
+
+#ifdef CONFIG_PAX_REFCOUNT
+ jno 0f
+ addl %eax, (v)
+ adcl %edx, 4(v)
+ int $4
+0:
+ _ASM_EXTABLE(0b, 0b)
+#endif
+
+RET_ENDP
+BEGIN(sub_unchecked)
+ subl %eax, (v)
+ sbbl %edx, 4(v)
RET_ENDP
#undef v
@@ -93,6 +147,27 @@ BEGIN(sub_return)
sbbl $0, %edx
addl (v), %eax
adcl 4(v), %edx
+
+#ifdef CONFIG_PAX_REFCOUNT
+ into
+1234:
+ _ASM_EXTABLE(1234b, 2f)
+#endif
+
+ movl %eax, (v)
+ movl %edx, 4(v)
+
+#ifdef CONFIG_PAX_REFCOUNT
+2:
+#endif
+
+RET_ENDP
+BEGIN(sub_return_unchecked)
+ negl %edx
+ negl %eax
+ sbbl $0, %edx
+ addl (v), %eax
+ adcl 4(v), %edx
movl %eax, (v)
movl %edx, 4(v)
RET_ENDP
@@ -102,6 +177,20 @@ RET_ENDP
BEGIN(inc)
addl $1, (v)
adcl $0, 4(v)
+
+#ifdef CONFIG_PAX_REFCOUNT
+ jno 0f
+ subl $1, (v)
+ sbbl $0, 4(v)
+ int $4
+0:
+ _ASM_EXTABLE(0b, 0b)
+#endif
+
+RET_ENDP
+BEGIN(inc_unchecked)
+ addl $1, (v)
+ adcl $0, 4(v)
RET_ENDP
#undef v
@@ -111,6 +200,26 @@ BEGIN(inc_return)
movl 4(v), %edx
addl $1, %eax
adcl $0, %edx
+
+#ifdef CONFIG_PAX_REFCOUNT
+ into
+1234:
+ _ASM_EXTABLE(1234b, 2f)
+#endif
+
+ movl %eax, (v)
+ movl %edx, 4(v)
+
+#ifdef CONFIG_PAX_REFCOUNT
+2:
+#endif
+
+RET_ENDP
+BEGIN(inc_return_unchecked)
+ movl (v), %eax
+ movl 4(v), %edx
+ addl $1, %eax
+ adcl $0, %edx
movl %eax, (v)
movl %edx, 4(v)
RET_ENDP
@@ -120,6 +229,20 @@ RET_ENDP
BEGIN(dec)
subl $1, (v)
sbbl $0, 4(v)
+
+#ifdef CONFIG_PAX_REFCOUNT
+ jno 0f
+ addl $1, (v)
+ adcl $0, 4(v)
+ int $4
+0:
+ _ASM_EXTABLE(0b, 0b)
+#endif
+
+RET_ENDP
+BEGIN(dec_unchecked)
+ subl $1, (v)
+ sbbl $0, 4(v)
RET_ENDP
#undef v
@@ -129,6 +252,26 @@ BEGIN(dec_return)
movl 4(v), %edx
subl $1, %eax
sbbl $0, %edx
+
+#ifdef CONFIG_PAX_REFCOUNT
+ into
+1234:
+ _ASM_EXTABLE(1234b, 2f)
+#endif
+
+ movl %eax, (v)
+ movl %edx, 4(v)
+
+#ifdef CONFIG_PAX_REFCOUNT
+2:
+#endif
+
+RET_ENDP
+BEGIN(dec_return_unchecked)
+ movl (v), %eax
+ movl 4(v), %edx
+ subl $1, %eax
+ sbbl $0, %edx
movl %eax, (v)
movl %edx, 4(v)
RET_ENDP
@@ -140,6 +283,13 @@ BEGIN(add_unless)
adcl %edx, %edi
addl (v), %eax
adcl 4(v), %edx
+
+#ifdef CONFIG_PAX_REFCOUNT
+ into
+1234:
+ _ASM_EXTABLE(1234b, 2f)
+#endif
+
cmpl %eax, %ecx
je 3f
1:
@@ -165,6 +315,13 @@ BEGIN(inc_not_zero)
1:
addl $1, %eax
adcl $0, %edx
+
+#ifdef CONFIG_PAX_REFCOUNT
+ into
+1234:
+ _ASM_EXTABLE(1234b, 2f)
+#endif
+
movl %eax, (v)
movl %edx, 4(v)
movl $1, %eax
@@ -183,6 +340,13 @@ BEGIN(dec_if_positive)
movl 4(v), %edx
subl $1, %eax
sbbl $0, %edx
+
+#ifdef CONFIG_PAX_REFCOUNT
+ into
+1234:
+ _ASM_EXTABLE(1234b, 1f)
+#endif
+
js 1f
movl %eax, (v)
movl %edx, 4(v)
diff --git a/arch/x86/lib/atomic64_cx8_32.S b/arch/x86/lib/atomic64_cx8_32.S
index db3ae854..845f5c3 100644
--- a/arch/x86/lib/atomic64_cx8_32.S
+++ b/arch/x86/lib/atomic64_cx8_32.S
@@ -22,9 +22,16 @@
ENTRY(atomic64_read_cx8)
read64 %ecx
+ pax_force_retaddr
ret
ENDPROC(atomic64_read_cx8)
+ENTRY(atomic64_read_unchecked_cx8)
+ read64 %ecx
+ pax_force_retaddr
+ ret
+ENDPROC(atomic64_read_unchecked_cx8)
+
ENTRY(atomic64_set_cx8)
1:
/* we don't need LOCK_PREFIX since aligned 64-bit writes
@@ -32,20 +39,33 @@ ENTRY(atomic64_set_cx8)
cmpxchg8b (%esi)
jne 1b
+ pax_force_retaddr
ret
ENDPROC(atomic64_set_cx8)
+ENTRY(atomic64_set_unchecked_cx8)
+1:
+/* we don't need LOCK_PREFIX since aligned 64-bit writes
+ * are atomic on 586 and newer */
+ cmpxchg8b (%esi)
+ jne 1b
+
+ pax_force_retaddr
+ ret
+ENDPROC(atomic64_set_unchecked_cx8)
+
ENTRY(atomic64_xchg_cx8)
1:
LOCK_PREFIX
cmpxchg8b (%esi)
jne 1b
+ pax_force_retaddr
ret
ENDPROC(atomic64_xchg_cx8)
-.macro addsub_return func ins insc
-ENTRY(atomic64_\func\()_return_cx8)
+.macro addsub_return func ins insc unchecked=""
+ENTRY(atomic64_\func\()_return\unchecked\()_cx8)
pushl %ebp
pushl %ebx
pushl %esi
@@ -61,6 +81,15 @@ ENTRY(atomic64_\func\()_return_cx8)
movl %edx, %ecx
\ins\()l %esi, %ebx
\insc\()l %edi, %ecx
+
+.ifb \unchecked
+#ifdef CONFIG_PAX_REFCOUNT
+ into
+2:
+ _ASM_EXTABLE(2b, 3f)
+#endif
+.endif
+
LOCK_PREFIX
cmpxchg8b (%ebp)
jne 1b
@@ -68,19 +97,29 @@ ENTRY(atomic64_\func\()_return_cx8)
10:
movl %ebx, %eax
movl %ecx, %edx
+
+.ifb \unchecked
+#ifdef CONFIG_PAX_REFCOUNT
+3:
+#endif
+.endif
+
popl %edi
popl %esi
popl %ebx
popl %ebp
+ pax_force_retaddr
ret
-ENDPROC(atomic64_\func\()_return_cx8)
+ENDPROC(atomic64_\func\()_return\unchecked\()_cx8)
.endm
addsub_return add add adc
addsub_return sub sub sbb
+addsub_return add add adc _unchecked
+addsub_return sub sub sbb _unchecked
-.macro incdec_return func ins insc
-ENTRY(atomic64_\func\()_return_cx8)
+.macro incdec_return func ins insc unchecked=""
+ENTRY(atomic64_\func\()_return\unchecked\()_cx8)
pushl %ebx
read64 %esi
@@ -89,6 +128,15 @@ ENTRY(atomic64_\func\()_return_cx8)
movl %edx, %ecx
\ins\()l $1, %ebx
\insc\()l $0, %ecx
+
+.ifb \unchecked
+#ifdef CONFIG_PAX_REFCOUNT
+ into
+2:
+ _ASM_EXTABLE(2b, 3f)
+#endif
+.endif
+
LOCK_PREFIX
cmpxchg8b (%esi)
jne 1b
@@ -96,13 +144,23 @@ ENTRY(atomic64_\func\()_return_cx8)
10:
movl %ebx, %eax
movl %ecx, %edx
+
+.ifb \unchecked
+#ifdef CONFIG_PAX_REFCOUNT
+3:
+#endif
+.endif
+
popl %ebx
+ pax_force_retaddr
ret
-ENDPROC(atomic64_\func\()_return_cx8)
+ENDPROC(atomic64_\func\()_return\unchecked\()_cx8)
.endm
incdec_return inc add adc
incdec_return dec sub sbb
+incdec_return inc add adc _unchecked
+incdec_return dec sub sbb _unchecked
ENTRY(atomic64_dec_if_positive_cx8)
pushl %ebx
@@ -113,6 +171,13 @@ ENTRY(atomic64_dec_if_positive_cx8)
movl %edx, %ecx
subl $1, %ebx
sbb $0, %ecx
+
+#ifdef CONFIG_PAX_REFCOUNT
+ into
+1234:
+ _ASM_EXTABLE(1234b, 2f)
+#endif
+
js 2f
LOCK_PREFIX
cmpxchg8b (%esi)
@@ -122,6 +187,7 @@ ENTRY(atomic64_dec_if_positive_cx8)
movl %ebx, %eax
movl %ecx, %edx
popl %ebx
+ pax_force_retaddr
ret
ENDPROC(atomic64_dec_if_positive_cx8)
@@ -144,6 +210,13 @@ ENTRY(atomic64_add_unless_cx8)
movl %edx, %ecx
addl %ebp, %ebx
adcl %edi, %ecx
+
+#ifdef CONFIG_PAX_REFCOUNT
+ into
+1234:
+ _ASM_EXTABLE(1234b, 3f)
+#endif
+
LOCK_PREFIX
cmpxchg8b (%esi)
jne 1b
@@ -153,6 +226,7 @@ ENTRY(atomic64_add_unless_cx8)
addl $8, %esp
popl %ebx
popl %ebp
+ pax_force_retaddr
ret
4:
cmpl %edx, 4(%esp)
@@ -173,6 +247,13 @@ ENTRY(atomic64_inc_not_zero_cx8)
xorl %ecx, %ecx
addl $1, %ebx
adcl %edx, %ecx
+
+#ifdef CONFIG_PAX_REFCOUNT
+ into
+1234:
+ _ASM_EXTABLE(1234b, 3f)
+#endif
+
LOCK_PREFIX
cmpxchg8b (%esi)
jne 1b
@@ -180,5 +261,6 @@ ENTRY(atomic64_inc_not_zero_cx8)
movl $1, %eax
3:
popl %ebx
+ pax_force_retaddr
ret
ENDPROC(atomic64_inc_not_zero_cx8)
diff --git a/arch/x86/mm/mmio-mod.c b/arch/x86/mm/mmio-mod.c
index 0057a7a..6fa3eb7 100644
--- a/arch/x86/mm/mmio-mod.c
+++ b/arch/x86/mm/mmio-mod.c
@@ -234,7 +234,7 @@ static void post(struct kmmio_probe *p, unsigned long condition,
static void ioremap_trace_core(resource_size_t offset, unsigned long size,
void __iomem *addr)
{
- static atomic_t next_id;
+ static atomic_unchecked_t next_id;
struct remap_trace *trace = kmalloc(sizeof(*trace), GFP_KERNEL);
/* These are page-unaligned. */
struct mmiotrace_map map = {
@@ -258,7 +258,7 @@ static void ioremap_trace_core(resource_size_t offset, unsigned long size,
.private = trace
},
.phys = offset,
- .id = atomic_inc_return(&next_id)
+ .id = atomic_inc_return_unchecked(&next_id)
};
map.map_id = trace->id;
diff --git a/fs/exec.c b/fs/exec.c
index 1977c2a..f1ac583 100644
--- a/fs/exec.c
+++ b/fs/exec.c
@@ -57,6 +57,11 @@
#include <linux/oom.h>
#include <linux/compat.h>
+#ifdef CONFIG_PAX_REFCOUNT
+#include <linux/kallsyms.h>
+#include <linux/kdebug.h>
+#endif
+
#include <asm/uaccess.h>
#include <asm/mmu_context.h>
#include <asm/tlb.h>
@@ -1742,4 +1747,23 @@ COMPAT_SYSCALL_DEFINE5(execveat, int, fd,
getname_flags(filename, lookup_flags, NULL),
argv, envp, flags);
}
+
+#ifdef CONFIG_PAX_REFCOUNT
+void pax_report_refcount_overflow(struct pt_regs *regs)
+{
+ if (current->signal->curr_ip)
+ printk(KERN_EMERG "PAX: From %pI4: refcount overflow detected in: %s:%d, uid/euid: %u/%u\n",
+ ¤t->signal->curr_ip, current->comm, task_pid_nr(current),
+ from_kuid_munged(&init_user_ns, current_uid()), from_kuid_munged(&init_user_ns, current_euid()));
+ else
+ printk(KERN_EMERG "PAX: refcount overflow detected in: %s:%d, uid/euid: %u/%u\n", current->comm, task_pid_nr(current),
+ from_kuid_munged(&init_user_ns, current_uid()), from_kuid_munged(&init_user_ns, current_euid()));
+ print_symbol(KERN_EMERG "PAX: refcount overflow occured at: %s\n", instruction_pointer(regs));
+ preempt_disable();
+ show_regs(regs);
+ preempt_enable();
+ force_sig_info(SIGKILL, SEND_SIG_FORCED, current);
+}
+#endif
+
#endif
diff --git a/include/asm-generic/atomic-long.h b/include/asm-generic/atomic-long.h
index b7babf0..04ad282 100644
--- a/include/asm-generic/atomic-long.h
+++ b/include/asm-generic/atomic-long.h
@@ -22,6 +22,12 @@
typedef atomic64_t atomic_long_t;
+#ifdef CONFIG_PAX_REFCOUNT
+typedef atomic64_unchecked_t atomic_long_unchecked_t;
+#else
+typedef atomic64_t atomic_long_unchecked_t;
+#endif
+
#define ATOMIC_LONG_INIT(i) ATOMIC64_INIT(i)
static inline long atomic_long_read(atomic_long_t *l)
@@ -31,6 +37,15 @@ static inline long atomic_long_read(atomic_long_t *l)
return (long)atomic64_read(v);
}
+#ifdef CONFIG_PAX_REFCOUNT
+static inline long atomic_long_read_unchecked(atomic_long_unchecked_t *l)
+{
+ atomic64_unchecked_t *v = (atomic64_unchecked_t *)l;
+
+ return (long)atomic64_read_unchecked(v);
+}
+#endif
+
static inline void atomic_long_set(atomic_long_t *l, long i)
{
atomic64_t *v = (atomic64_t *)l;
@@ -38,6 +53,15 @@ static inline void atomic_long_set(atomic_long_t *l, long i)
atomic64_set(v, i);
}
+#ifdef CONFIG_PAX_REFCOUNT
+static inline void atomic_long_set_unchecked(atomic_long_unchecked_t *l, long i)
+{
+ atomic64_unchecked_t *v = (atomic64_unchecked_t *)l;
+
+ atomic64_set_unchecked(v, i);
+}
+#endif
+
static inline void atomic_long_inc(atomic_long_t *l)
{
atomic64_t *v = (atomic64_t *)l;
@@ -45,6 +69,15 @@ static inline void atomic_long_inc(atomic_long_t *l)
atomic64_inc(v);
}
+#ifdef CONFIG_PAX_REFCOUNT
+static inline void atomic_long_inc_unchecked(atomic_long_unchecked_t *l)
+{
+ atomic64_unchecked_t *v = (atomic64_unchecked_t *)l;
+
+ atomic64_inc_unchecked(v);
+}
+#endif
+
static inline void atomic_long_dec(atomic_long_t *l)
{
atomic64_t *v = (atomic64_t *)l;
@@ -52,6 +85,15 @@ static inline void atomic_long_dec(atomic_long_t *l)
atomic64_dec(v);
}
+#ifdef CONFIG_PAX_REFCOUNT
+static inline void atomic_long_dec_unchecked(atomic_long_unchecked_t *l)
+{
+ atomic64_unchecked_t *v = (atomic64_unchecked_t *)l;
+
+ atomic64_dec_unchecked(v);
+}
+#endif
+
static inline void atomic_long_add(long i, atomic_long_t *l)
{
atomic64_t *v = (atomic64_t *)l;
@@ -59,6 +101,15 @@ static inline void atomic_long_add(long i, atomic_long_t *l)
atomic64_add(i, v);
}
+#ifdef CONFIG_PAX_REFCOUNT
+static inline void atomic_long_add_unchecked(long i, atomic_long_unchecked_t *l)
+{
+ atomic64_unchecked_t *v = (atomic64_unchecked_t *)l;
+
+ atomic64_add_unchecked(i, v);
+}
+#endif
+
static inline void atomic_long_sub(long i, atomic_long_t *l)
{
atomic64_t *v = (atomic64_t *)l;
@@ -66,6 +117,15 @@ static inline void atomic_long_sub(long i, atomic_long_t *l)
atomic64_sub(i, v);
}
+#ifdef CONFIG_PAX_REFCOUNT
+static inline void atomic_long_sub_unchecked(long i, atomic_long_unchecked_t *l)
+{
+ atomic64_unchecked_t *v = (atomic64_unchecked_t *)l;
+
+ atomic64_sub_unchecked(i, v);
+}
+#endif
+
static inline int atomic_long_sub_and_test(long i, atomic_long_t *l)
{
atomic64_t *v = (atomic64_t *)l;
@@ -101,6 +161,15 @@ static inline long atomic_long_add_return(long i, atomic_long_t *l)
return (long)atomic64_add_return(i, v);
}
+#ifdef CONFIG_PAX_REFCOUNT
+static inline long atomic_long_add_return_unchecked(long i, atomic_long_unchecked_t *l)
+{
+ atomic64_unchecked_t *v = (atomic64_unchecked_t *)l;
+
+ return (long)atomic64_add_return_unchecked(i, v);
+}
+#endif
+
static inline long atomic_long_sub_return(long i, atomic_long_t *l)
{
atomic64_t *v = (atomic64_t *)l;
@@ -115,6 +184,15 @@ static inline long atomic_long_inc_return(atomic_long_t *l)
return (long)atomic64_inc_return(v);
}
+#ifdef CONFIG_PAX_REFCOUNT
+static inline long atomic_long_inc_return_unchecked(atomic_long_unchecked_t *l)
+{
+ atomic64_unchecked_t *v = (atomic64_unchecked_t *)l;
+
+ return (long)atomic64_inc_return_unchecked(v);
+}
+#endif
+
static inline long atomic_long_dec_return(atomic_long_t *l)
{
atomic64_t *v = (atomic64_t *)l;
@@ -140,6 +218,12 @@ static inline long atomic_long_add_unless(atomic_long_t *l, long a, long u)
typedef atomic_t atomic_long_t;
+#ifdef CONFIG_PAX_REFCOUNT
+typedef atomic_unchecked_t atomic_long_unchecked_t;
+#else
+typedef atomic_t atomic_long_unchecked_t;
+#endif
+
#define ATOMIC_LONG_INIT(i) ATOMIC_INIT(i)
static inline long atomic_long_read(atomic_long_t *l)
{
@@ -148,6 +232,15 @@ static inline long atomic_long_read(atomic_long_t *l)
return (long)atomic_read(v);
}
+#ifdef CONFIG_PAX_REFCOUNT
+static inline long atomic_long_read_unchecked(atomic_long_unchecked_t *l)
+{
+ atomic_unchecked_t *v = (atomic_unchecked_t *)l;
+
+ return (long)atomic_read_unchecked(v);
+}
+#endif
+
static inline void atomic_long_set(atomic_long_t *l, long i)
{
atomic_t *v = (atomic_t *)l;
@@ -155,6 +248,15 @@ static inline void atomic_long_set(atomic_long_t *l, long i)
atomic_set(v, i);
}
+#ifdef CONFIG_PAX_REFCOUNT
+static inline void atomic_long_set_unchecked(atomic_long_unchecked_t *l, long i)
+{
+ atomic_unchecked_t *v = (atomic_unchecked_t *)l;
+
+ atomic_set_unchecked(v, i);
+}
+#endif
+
static inline void atomic_long_inc(atomic_long_t *l)
{
atomic_t *v = (atomic_t *)l;
@@ -162,6 +264,15 @@ static inline void atomic_long_inc(atomic_long_t *l)
atomic_inc(v);
}
+#ifdef CONFIG_PAX_REFCOUNT
+static inline void atomic_long_inc_unchecked(atomic_long_unchecked_t *l)
+{
+ atomic_unchecked_t *v = (atomic_unchecked_t *)l;
+
+ atomic_inc_unchecked(v);
+}
+#endif
+
static inline void atomic_long_dec(atomic_long_t *l)
{
atomic_t *v = (atomic_t *)l;
@@ -169,6 +280,15 @@ static inline void atomic_long_dec(atomic_long_t *l)
atomic_dec(v);
}
+#ifdef CONFIG_PAX_REFCOUNT
+static inline void atomic_long_dec_unchecked(atomic_long_unchecked_t *l)
+{
+ atomic_unchecked_t *v = (atomic_unchecked_t *)l;
+
+ atomic_dec_unchecked(v);
+}
+#endif
+
static inline void atomic_long_add(long i, atomic_long_t *l)
{
atomic_t *v = (atomic_t *)l;
@@ -176,6 +296,15 @@ static inline void atomic_long_add(long i, atomic_long_t *l)
atomic_add(i, v);
}
+#ifdef CONFIG_PAX_REFCOUNT
+static inline void atomic_long_add_unchecked(long i, atomic_long_unchecked_t *l)
+{
+ atomic_unchecked_t *v = (atomic_unchecked_t *)l;
+
+ atomic_add_unchecked(i, v);
+}
+#endif
+
static inline void atomic_long_sub(long i, atomic_long_t *l)
{
atomic_t *v = (atomic_t *)l;
@@ -183,6 +312,15 @@ static inline void atomic_long_sub(long i, atomic_long_t *l)
atomic_sub(i, v);
}
+#ifdef CONFIG_PAX_REFCOUNT
+static inline void atomic_long_sub_unchecked(long i, atomic_long_unchecked_t *l)
+{
+ atomic_unchecked_t *v = (atomic_unchecked_t *)l;
+
+ atomic_sub_unchecked(i, v);
+}
+#endif
+
static inline int atomic_long_sub_and_test(long i, atomic_long_t *l)
{
atomic_t *v = (atomic_t *)l;
@@ -218,6 +356,16 @@ static inline long atomic_long_add_return(long i, atomic_long_t *l)
return (long)atomic_add_return(i, v);
}
+#ifdef CONFIG_PAX_REFCOUNT
+static inline long atomic_long_add_return_unchecked(long i, atomic_long_unchecked_t *l)
+{
+ atomic_unchecked_t *v = (atomic_unchecked_t *)l;
+
+ return (long)atomic_add_return_unchecked(i, v);
+}
+
+#endif
+
static inline long atomic_long_sub_return(long i, atomic_long_t *l)
{
atomic_t *v = (atomic_t *)l;
@@ -232,6 +380,15 @@ static inline long atomic_long_inc_return(atomic_long_t *l)
return (long)atomic_inc_return(v);
}
+#ifdef CONFIG_PAX_REFCOUNT
+static inline long atomic_long_inc_return_unchecked(atomic_long_unchecked_t *l)
+{
+ atomic_unchecked_t *v = (atomic_unchecked_t *)l;
+
+ return (long)atomic_inc_return_unchecked(v);
+}
+#endif
+
static inline long atomic_long_dec_return(atomic_long_t *l)
{
atomic_t *v = (atomic_t *)l;
@@ -255,4 +412,57 @@ static inline long atomic_long_add_unless(atomic_long_t *l, long a, long u)
#endif /* BITS_PER_LONG == 64 */
+#ifdef CONFIG_PAX_REFCOUNT
+static inline void pax_refcount_needs_these_functions(void)
+{
+ atomic_read_unchecked((atomic_unchecked_t *)NULL);
+ atomic_set_unchecked((atomic_unchecked_t *)NULL, 0);
+ atomic_add_unchecked(0, (atomic_unchecked_t *)NULL);
+ atomic_sub_unchecked(0, (atomic_unchecked_t *)NULL);
+ atomic_inc_unchecked((atomic_unchecked_t *)NULL);
+ (void)atomic_inc_and_test_unchecked((atomic_unchecked_t *)NULL);
+ atomic_inc_return_unchecked((atomic_unchecked_t *)NULL);
+ atomic_add_return_unchecked(0, (atomic_unchecked_t *)NULL);
+ atomic_dec_unchecked((atomic_unchecked_t *)NULL);
+ atomic_cmpxchg_unchecked((atomic_unchecked_t *)NULL, 0, 0);
+ (void)atomic_xchg_unchecked((atomic_unchecked_t *)NULL, 0);
+#ifdef CONFIG_X86
+ atomic_clear_mask_unchecked(0, NULL);
+ atomic_set_mask_unchecked(0, NULL);
+#endif
+
+ atomic_long_read_unchecked((atomic_long_unchecked_t *)NULL);
+ atomic_long_set_unchecked((atomic_long_unchecked_t *)NULL, 0);
+ atomic_long_add_unchecked(0, (atomic_long_unchecked_t *)NULL);
+ atomic_long_sub_unchecked(0, (atomic_long_unchecked_t *)NULL);
+ atomic_long_inc_unchecked((atomic_long_unchecked_t *)NULL);
+ atomic_long_add_return_unchecked(0, (atomic_long_unchecked_t *)NULL);
+ atomic_long_inc_return_unchecked((atomic_long_unchecked_t *)NULL);
+ atomic_long_dec_unchecked((atomic_long_unchecked_t *)NULL);
+}
+#else
+#define atomic_read_unchecked(v) atomic_read(v)
+#define atomic_set_unchecked(v, i) atomic_set((v), (i))
+#define atomic_add_unchecked(i, v) atomic_add((i), (v))
+#define atomic_sub_unchecked(i, v) atomic_sub((i), (v))
+#define atomic_inc_unchecked(v) atomic_inc(v)
+#define atomic_inc_and_test_unchecked(v) atomic_inc_and_test(v)
+#define atomic_inc_return_unchecked(v) atomic_inc_return(v)
+#define atomic_add_return_unchecked(i, v) atomic_add_return((i), (v))
+#define atomic_dec_unchecked(v) atomic_dec(v)
+#define atomic_cmpxchg_unchecked(v, o, n) atomic_cmpxchg((v), (o), (n))
+#define atomic_xchg_unchecked(v, i) atomic_xchg((v), (i))
+#define atomic_clear_mask_unchecked(mask, v) atomic_clear_mask((mask), (v))
+#define atomic_set_mask_unchecked(mask, v) atomic_set_mask((mask), (v))
+
+#define atomic_long_read_unchecked(v) atomic_long_read(v)
+#define atomic_long_set_unchecked(v, i) atomic_long_set((v), (i))
+#define atomic_long_add_unchecked(i, v) atomic_long_add((i), (v))
+#define atomic_long_sub_unchecked(i, v) atomic_long_sub((i), (v))
+#define atomic_long_inc_unchecked(v) atomic_long_inc(v)
+#define atomic_long_add_return_unchecked(i, v) atomic_long_add_return((i), (v))
+#define atomic_long_inc_return_unchecked(v) atomic_long_inc_return(v)
+#define atomic_long_dec_unchecked(v) atomic_long_dec(v)
+#endif
+
#endif /* _ASM_GENERIC_ATOMIC_LONG_H */
diff --git a/include/asm-generic/atomic64.h b/include/asm-generic/atomic64.h
index 30ad9c8..c70c170 100644
--- a/include/asm-generic/atomic64.h
+++ b/include/asm-generic/atomic64.h
@@ -16,6 +16,8 @@ typedef struct {
long long counter;
} atomic64_t;
+typedef atomic64_t atomic64_unchecked_t;
+
#define ATOMIC64_INIT(i) { (i) }
extern long long atomic64_read(const atomic64_t *v);
@@ -51,4 +53,14 @@ extern int atomic64_add_unless(atomic64_t *v, long long a, long long u);
#define atomic64_dec_and_test(v) (atomic64_dec_return((v)) == 0)
#define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1LL, 0LL)
+#define atomic64_read_unchecked(v) atomic64_read(v)
+#define atomic64_set_unchecked(v, i) atomic64_set((v), (i))
+#define atomic64_add_unchecked(a, v) atomic64_add((a), (v))
+#define atomic64_add_return_unchecked(a, v) atomic64_add_return((a), (v))
+#define atomic64_sub_unchecked(a, v) atomic64_sub((a), (v))
+#define atomic64_inc_unchecked(v) atomic64_inc(v)
+#define atomic64_inc_return_unchecked(v) atomic64_inc_return(v)
+#define atomic64_dec_unchecked(v) atomic64_dec(v)
+#define atomic64_cmpxchg_unchecked(v, o, n) atomic64_cmpxchg((v), (o), (n))
+
#endif /* _ASM_GENERIC_ATOMIC64_H */
diff --git a/include/asm-generic/local.h b/include/asm-generic/local.h
index 9ceb03b..62b0b8f 100644
--- a/include/asm-generic/local.h
+++ b/include/asm-generic/local.h
@@ -23,24 +23,37 @@ typedef struct
atomic_long_t a;
} local_t;
+typedef struct {
+ atomic_long_unchecked_t a;
+} local_unchecked_t;
+
#define LOCAL_INIT(i) { ATOMIC_LONG_INIT(i) }
#define local_read(l) atomic_long_read(&(l)->a)
+#define local_read_unchecked(l) atomic_long_read_unchecked(&(l)->a)
#define local_set(l,i) atomic_long_set((&(l)->a),(i))
+#define local_set_unchecked(l,i) atomic_long_set_unchecked((&(l)->a),(i))
#define local_inc(l) atomic_long_inc(&(l)->a)
+#define local_inc_unchecked(l) atomic_long_inc_unchecked(&(l)->a)
#define local_dec(l) atomic_long_dec(&(l)->a)
+#define local_dec_unchecked(l) atomic_long_dec_unchecked(&(l)->a)
#define local_add(i,l) atomic_long_add((i),(&(l)->a))
+#define local_add_unchecked(i,l) atomic_long_add_unchecked((i),(&(l)->a))
#define local_sub(i,l) atomic_long_sub((i),(&(l)->a))
+#define local_sub_unchecked(i,l) atomic_long_sub_unchecked((i),(&(l)->a))
#define local_sub_and_test(i, l) atomic_long_sub_and_test((i), (&(l)->a))
#define local_dec_and_test(l) atomic_long_dec_and_test(&(l)->a)
#define local_inc_and_test(l) atomic_long_inc_and_test(&(l)->a)
#define local_add_negative(i, l) atomic_long_add_negative((i), (&(l)->a))
#define local_add_return(i, l) atomic_long_add_return((i), (&(l)->a))
+#define local_add_return_unchecked(i, l) atomic_long_add_return_unchecked((i), (&(l)->a))
#define local_sub_return(i, l) atomic_long_sub_return((i), (&(l)->a))
#define local_inc_return(l) atomic_long_inc_return(&(l)->a)
+#define local_dec_return(l) atomic_long_dec_return(&(l)->a)
#define local_cmpxchg(l, o, n) atomic_long_cmpxchg((&(l)->a), (o), (n))
+#define local_cmpxchg_unchecked(l, o, n) atomic_long_cmpxchg((&(l)->a), (o), (n))
#define local_xchg(l, n) atomic_long_xchg((&(l)->a), (n))
#define local_add_unless(l, _a, u) atomic_long_add_unless((&(l)->a), (_a), (u))
#define local_inc_not_zero(l) atomic_long_inc_not_zero(&(l)->a)
diff --git a/include/linux/sched.h b/include/linux/sched.h
index bfca8aa..20cf887 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -749,6 +749,17 @@ struct signal_struct {
#ifdef CONFIG_TASKSTATS
struct taskstats *stats;
#endif
+
+#ifdef CONFIG_GRKERNSEC
+ u32 curr_ip;
+ u32 saved_ip;
+ u32 gr_saddr;
+ u32 gr_daddr;
+ u16 gr_sport;
+ u16 gr_dport;
+ u8 used_accept:1;
+#endif
+
#ifdef CONFIG_AUDIT
unsigned audit_tty;
unsigned audit_tty_log_passwd;
@@ -1804,6 +1815,8 @@ extern int arch_task_struct_size __read_mostly;
# define arch_task_struct_size (sizeof(struct task_struct))
#endif
+extern void pax_report_refcount_overflow(struct pt_regs *regs);
+
/* Future-safe accessor for struct task_struct's cpus_allowed. */
#define tsk_cpus_allowed(tsk) (&(tsk)->cpus_allowed)
diff --git a/include/linux/types.h b/include/linux/types.h
index 8715287..1be77ee 100644
--- a/include/linux/types.h
+++ b/include/linux/types.h
@@ -176,10 +176,26 @@ typedef struct {
int counter;
} atomic_t;
+#ifdef CONFIG_PAX_REFCOUNT
+typedef struct {
+ int counter;
+} atomic_unchecked_t;
+#else
+typedef atomic_t atomic_unchecked_t;
+#endif
+
#ifdef CONFIG_64BIT
typedef struct {
long counter;
} atomic64_t;
+
+#ifdef CONFIG_PAX_REFCOUNT
+typedef struct {
+ long counter;
+} atomic64_unchecked_t;
+#else
+typedef atomic64_t atomic64_unchecked_t;
+#endif
#endif
struct list_head {
--
2.5.0
^ permalink raw reply related [flat|nested] 32+ messages in thread* Re: [kernel-hardening][RFC PATCH v2 01/12] x86: add PAX_REFCOUNT support
2015-12-17 14:57 ` [kernel-hardening][RFC PATCH v2 01/12] x86: add PAX_REFCOUNT support David Windsor
@ 2015-12-17 17:16 ` Greg KH
2015-12-17 17:55 ` David Windsor
0 siblings, 1 reply; 32+ messages in thread
From: Greg KH @ 2015-12-17 17:16 UTC (permalink / raw)
To: kernel-hardening; +Cc: David Windsor
On Thu, Dec 17, 2015 at 09:57:39AM -0500, David Windsor wrote:
> --- a/include/linux/sched.h
> +++ b/include/linux/sched.h
> @@ -749,6 +749,17 @@ struct signal_struct {
> #ifdef CONFIG_TASKSTATS
> struct taskstats *stats;
> #endif
> +
> +#ifdef CONFIG_GRKERNSEC
> + u32 curr_ip;
> + u32 saved_ip;
> + u32 gr_saddr;
> + u32 gr_daddr;
> + u16 gr_sport;
> + u16 gr_dport;
> + u8 used_accept:1;
> +#endif
> +
Why is this here in this patch?
thanks,
greg k-h
^ permalink raw reply [flat|nested] 32+ messages in thread* Re: [kernel-hardening][RFC PATCH v2 01/12] x86: add PAX_REFCOUNT support
2015-12-17 17:16 ` Greg KH
@ 2015-12-17 17:55 ` David Windsor
2015-12-17 18:11 ` Greg KH
0 siblings, 1 reply; 32+ messages in thread
From: David Windsor @ 2015-12-17 17:55 UTC (permalink / raw)
To: kernel-hardening
On Thu, Dec 17, 2015 at 12:16 PM, Greg KH <gregkh@linuxfoundation.org> wrote:
> On Thu, Dec 17, 2015 at 09:57:39AM -0500, David Windsor wrote:
>> --- a/include/linux/sched.h
>> +++ b/include/linux/sched.h
>> @@ -749,6 +749,17 @@ struct signal_struct {
>> #ifdef CONFIG_TASKSTATS
>> struct taskstats *stats;
>> #endif
>> +
>> +#ifdef CONFIG_GRKERNSEC
>> + u32 curr_ip;
>> + u32 saved_ip;
>> + u32 gr_saddr;
>> + u32 gr_daddr;
>> + u16 gr_sport;
>> + u16 gr_dport;
>> + u8 used_accept:1;
>> +#endif
>> +
>
> Why is this here in this patch?
>
For pax_report_refcount_overflow in fs/exec.c:
1754: if (current->signal->curr_ip)
This is guarded by CONFIG_PAX_REFCOUNT, which, as it stands, should
depend on CONFIG_GRKERNSEC. The Kconfig options likely need to
change, depending on the naming of these features moving forward.
> thanks,
>
> greg k-h
^ permalink raw reply [flat|nested] 32+ messages in thread* Re: [kernel-hardening][RFC PATCH v2 01/12] x86: add PAX_REFCOUNT support
2015-12-17 17:55 ` David Windsor
@ 2015-12-17 18:11 ` Greg KH
2015-12-17 18:23 ` David Windsor
2015-12-18 2:59 ` David Windsor
0 siblings, 2 replies; 32+ messages in thread
From: Greg KH @ 2015-12-17 18:11 UTC (permalink / raw)
To: kernel-hardening
On Thu, Dec 17, 2015 at 12:55:44PM -0500, David Windsor wrote:
> On Thu, Dec 17, 2015 at 12:16 PM, Greg KH <gregkh@linuxfoundation.org> wrote:
> > On Thu, Dec 17, 2015 at 09:57:39AM -0500, David Windsor wrote:
> >> --- a/include/linux/sched.h
> >> +++ b/include/linux/sched.h
> >> @@ -749,6 +749,17 @@ struct signal_struct {
> >> #ifdef CONFIG_TASKSTATS
> >> struct taskstats *stats;
> >> #endif
> >> +
> >> +#ifdef CONFIG_GRKERNSEC
> >> + u32 curr_ip;
> >> + u32 saved_ip;
> >> + u32 gr_saddr;
> >> + u32 gr_daddr;
> >> + u16 gr_sport;
> >> + u16 gr_dport;
> >> + u8 used_accept:1;
> >> +#endif
> >> +
> >
> > Why is this here in this patch?
> >
>
> For pax_report_refcount_overflow in fs/exec.c:
>
> 1754: if (current->signal->curr_ip)
>
> This is guarded by CONFIG_PAX_REFCOUNT, which, as it stands, should
> depend on CONFIG_GRKERNSEC. The Kconfig options likely need to
> change, depending on the naming of these features moving forward.
But that has nothing to do with this patch, and as that config option
isn't present in the kernel at this point in time, this change is not
needed at all, right?
Or am I missing something?
thanks,
greg k-h
^ permalink raw reply [flat|nested] 32+ messages in thread* Re: [kernel-hardening][RFC PATCH v2 01/12] x86: add PAX_REFCOUNT support
2015-12-17 18:11 ` Greg KH
@ 2015-12-17 18:23 ` David Windsor
2015-12-17 19:46 ` Greg KH
2015-12-18 2:59 ` David Windsor
1 sibling, 1 reply; 32+ messages in thread
From: David Windsor @ 2015-12-17 18:23 UTC (permalink / raw)
To: kernel-hardening
Well, pax_report_refcount_overflow is the mechanism for logging
overflows and for killing the offending process. Since that's related
to adding PAX_REFCOUNT protection (not necessarily to just x86,
though), I put it in this patch. I suppose I could have made another
patch for arch-independent changes like this.
You are correct that CONFIG_GRKERNSEC and CONFIG_PAX_REFCOUNT aren't
currently in the kernel, but they are both added in a later patch in
this series (patch 11/12). I should have made those changes part of
this patch, though. I will update the series and resubmit later.
Hopefully this answers your question?
Thanks,
David
On Thu, Dec 17, 2015 at 1:11 PM, Greg KH <greg@kroah.com> wrote:
> On Thu, Dec 17, 2015 at 12:55:44PM -0500, David Windsor wrote:
>> On Thu, Dec 17, 2015 at 12:16 PM, Greg KH <gregkh@linuxfoundation.org> wrote:
>> > On Thu, Dec 17, 2015 at 09:57:39AM -0500, David Windsor wrote:
>> >> --- a/include/linux/sched.h
>> >> +++ b/include/linux/sched.h
>> >> @@ -749,6 +749,17 @@ struct signal_struct {
>> >> #ifdef CONFIG_TASKSTATS
>> >> struct taskstats *stats;
>> >> #endif
>> >> +
>> >> +#ifdef CONFIG_GRKERNSEC
>> >> + u32 curr_ip;
>> >> + u32 saved_ip;
>> >> + u32 gr_saddr;
>> >> + u32 gr_daddr;
>> >> + u16 gr_sport;
>> >> + u16 gr_dport;
>> >> + u8 used_accept:1;
>> >> +#endif
>> >> +
>> >
>> > Why is this here in this patch?
>> >
>>
>> For pax_report_refcount_overflow in fs/exec.c:
>>
>> 1754: if (current->signal->curr_ip)
>>
>> This is guarded by CONFIG_PAX_REFCOUNT, which, as it stands, should
>> depend on CONFIG_GRKERNSEC. The Kconfig options likely need to
>> change, depending on the naming of these features moving forward.
>
> But that has nothing to do with this patch, and as that config option
> isn't present in the kernel at this point in time, this change is not
> needed at all, right?
>
> Or am I missing something?
>
> thanks,
>
> greg k-h
^ permalink raw reply [flat|nested] 32+ messages in thread* Re: [kernel-hardening][RFC PATCH v2 01/12] x86: add PAX_REFCOUNT support
2015-12-17 18:23 ` David Windsor
@ 2015-12-17 19:46 ` Greg KH
0 siblings, 0 replies; 32+ messages in thread
From: Greg KH @ 2015-12-17 19:46 UTC (permalink / raw)
To: kernel-hardening
On Thu, Dec 17, 2015 at 01:23:15PM -0500, David Windsor wrote:
> Well, pax_report_refcount_overflow is the mechanism for logging
> overflows and for killing the offending process. Since that's related
> to adding PAX_REFCOUNT protection (not necessarily to just x86,
> though), I put it in this patch. I suppose I could have made another
> patch for arch-independent changes like this.
Yes please, each patch should only do one specific thing. This one
seemed to do many different things, using kconfig options that were not
present yet, making the code useless :)
> You are correct that CONFIG_GRKERNSEC and CONFIG_PAX_REFCOUNT aren't
> currently in the kernel, but they are both added in a later patch in
> this series (patch 11/12). I should have made those changes part of
> this patch, though. I will update the series and resubmit later.
You can introduce the build option first or last, either is fine, but it
was confusing to see this without anything seemingly relevant to the
new atomic type.
thanks,
greg k-h
^ permalink raw reply [flat|nested] 32+ messages in thread
* Re: [kernel-hardening][RFC PATCH v2 01/12] x86: add PAX_REFCOUNT support
2015-12-17 18:11 ` Greg KH
2015-12-17 18:23 ` David Windsor
@ 2015-12-18 2:59 ` David Windsor
1 sibling, 0 replies; 32+ messages in thread
From: David Windsor @ 2015-12-18 2:59 UTC (permalink / raw)
To: kernel-hardening; +Cc: greg
On Thu, Dec 17, 2015 at 1:11 PM, Greg KH <greg@kroah.com> wrote:
> On Thu, Dec 17, 2015 at 12:55:44PM -0500, David Windsor wrote:
>> On Thu, Dec 17, 2015 at 12:16 PM, Greg KH <gregkh@linuxfoundation.org> wrote:
>> > On Thu, Dec 17, 2015 at 09:57:39AM -0500, David Windsor wrote:
>> >> --- a/include/linux/sched.h
>> >> +++ b/include/linux/sched.h
>> >> @@ -749,6 +749,17 @@ struct signal_struct {
>> >> #ifdef CONFIG_TASKSTATS
>> >> struct taskstats *stats;
>> >> #endif
>> >> +
>> >> +#ifdef CONFIG_GRKERNSEC
>> >> + u32 curr_ip;
>> >> + u32 saved_ip;
>> >> + u32 gr_saddr;
>> >> + u32 gr_daddr;
>> >> + u16 gr_sport;
>> >> + u16 gr_dport;
>> >> + u8 used_accept:1;
>> >> +#endif
>> >> +
>> >
>> > Why is this here in this patch?
>> >
>>
>> For pax_report_refcount_overflow in fs/exec.c:
>>
>> 1754: if (current->signal->curr_ip)
>>
>> This is guarded by CONFIG_PAX_REFCOUNT, which, as it stands, should
>> depend on CONFIG_GRKERNSEC. The Kconfig options likely need to
>> change, depending on the naming of these features moving forward.
>
> But that has nothing to do with this patch, and as that config option
> isn't present in the kernel at this point in time, this change is not
> needed at all, right?
>
> Or am I missing something?
>
Actually, there is a PaX-only version of pax_report_refcount_overflow
[1] that doesn't include instruction pointer tracking, which is a
grsecurity feature (thanks minipli), and the reason why this piece was
pulled into my patch.
I'll update this to use the PaX-only version. If the relevant portion
of grsecurity ends up getting pulled in as well, we'll return to the
version originally posted here.
Thanks,
David
[1] https://github.com/minipli/linux-grsec/blob/v4.3.3-pax/fs/exec.c#L1936-L1947
> thanks,
>
> greg k-h
^ permalink raw reply [flat|nested] 32+ messages in thread
* [kernel-hardening][RFC PATCH v2 02/12] percpu_ref: decrease per-CPU refcount bias
2015-12-17 14:57 [kernel-hardening][RFC PATCH v2 00/12] Add PAX_REFCOUNT overflow protection David Windsor
2015-12-17 14:57 ` [kernel-hardening][RFC PATCH v2 01/12] x86: add PAX_REFCOUNT support David Windsor
@ 2015-12-17 14:57 ` David Windsor
2015-12-17 20:56 ` Kees Cook
2015-12-17 14:57 ` [kernel-hardening][RFC PATCH v2 03/12] kernel: opt out of PAX_REFCOUNT protection David Windsor
` (10 subsequent siblings)
12 siblings, 1 reply; 32+ messages in thread
From: David Windsor @ 2015-12-17 14:57 UTC (permalink / raw)
To: kernel-hardening; +Cc: David Windsor
This change is necessary to prevent overflows from occuring in
percpu_switch_to_atomic_rcu during init on x86.
Signed-off-by: David Windsor <dave@progbits.org>
---
lib/percpu-refcount.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/percpu-refcount.c b/lib/percpu-refcount.c
index 6111bcb..02e816b 100644
--- a/lib/percpu-refcount.c
+++ b/lib/percpu-refcount.c
@@ -31,7 +31,7 @@
* atomic_long_t can't hit 0 before we've added up all the percpu refs.
*/
-#define PERCPU_COUNT_BIAS (1LU << (BITS_PER_LONG - 1))
+#define PERCPU_COUNT_BIAS (1LU << (BITS_PER_LONG - 2))
static DECLARE_WAIT_QUEUE_HEAD(percpu_ref_switch_waitq);
--
2.5.0
^ permalink raw reply related [flat|nested] 32+ messages in thread* Re: [kernel-hardening][RFC PATCH v2 02/12] percpu_ref: decrease per-CPU refcount bias
2015-12-17 14:57 ` [kernel-hardening][RFC PATCH v2 02/12] percpu_ref: decrease per-CPU refcount bias David Windsor
@ 2015-12-17 20:56 ` Kees Cook
0 siblings, 0 replies; 32+ messages in thread
From: Kees Cook @ 2015-12-17 20:56 UTC (permalink / raw)
To: kernel-hardening@lists.openwall.com; +Cc: David Windsor
On Thu, Dec 17, 2015 at 6:57 AM, David Windsor <dave@progbits.org> wrote:
> This change is necessary to prevent overflows from occuring in
> percpu_switch_to_atomic_rcu during init on x86.
I'm curious to find out why this overflows under "normal" operation.
Is there any downside to this change?
-Kees
>
> Signed-off-by: David Windsor <dave@progbits.org>
> ---
> lib/percpu-refcount.c | 2 +-
> 1 file changed, 1 insertion(+), 1 deletion(-)
>
> diff --git a/lib/percpu-refcount.c b/lib/percpu-refcount.c
> index 6111bcb..02e816b 100644
> --- a/lib/percpu-refcount.c
> +++ b/lib/percpu-refcount.c
> @@ -31,7 +31,7 @@
> * atomic_long_t can't hit 0 before we've added up all the percpu refs.
> */
>
> -#define PERCPU_COUNT_BIAS (1LU << (BITS_PER_LONG - 1))
> +#define PERCPU_COUNT_BIAS (1LU << (BITS_PER_LONG - 2))
>
> static DECLARE_WAIT_QUEUE_HEAD(percpu_ref_switch_waitq);
>
> --
> 2.5.0
>
>
--
Kees Cook
Chrome OS & Brillo Security
^ permalink raw reply [flat|nested] 32+ messages in thread
* [kernel-hardening][RFC PATCH v2 03/12] kernel: opt out of PAX_REFCOUNT protection
2015-12-17 14:57 [kernel-hardening][RFC PATCH v2 00/12] Add PAX_REFCOUNT overflow protection David Windsor
2015-12-17 14:57 ` [kernel-hardening][RFC PATCH v2 01/12] x86: add PAX_REFCOUNT support David Windsor
2015-12-17 14:57 ` [kernel-hardening][RFC PATCH v2 02/12] percpu_ref: decrease per-CPU refcount bias David Windsor
@ 2015-12-17 14:57 ` David Windsor
2015-12-17 14:57 ` [kernel-hardening][RFC PATCH v2 04/12] mm: " David Windsor
` (9 subsequent siblings)
12 siblings, 0 replies; 32+ messages in thread
From: David Windsor @ 2015-12-17 14:57 UTC (permalink / raw)
To: kernel-hardening; +Cc: David Windsor
Signed-off-by: David Windsor <dave@progbits.org>
---
include/linux/blktrace_api.h | 2 +-
include/linux/irqdesc.h | 2 +-
include/linux/kgdb.h | 2 +-
include/linux/oprofile.h | 4 +-
include/linux/padata.h | 2 +-
include/linux/perf_event.h | 8 +--
include/linux/sched.h | 2 +-
include/video/udlfb.h | 8 +--
kernel/audit.c | 8 +--
kernel/auditsc.c | 4 +-
kernel/debug/debug_core.c | 16 +++---
kernel/events/core.c | 22 ++++-----
kernel/irq/manage.c | 2 +-
kernel/irq/spurious.c | 2 +-
kernel/locking/lockdep.c | 2 +-
kernel/locking/rtmutex-tester.c | 24 ++++-----
kernel/padata.c | 4 +-
kernel/profile.c | 14 +++---
kernel/rcu/rcutorture.c | 60 +++++++++++-----------
kernel/rcu/tree.c | 64 ++++++++++++------------
kernel/rcu/tree.h | 26 +++++-----
kernel/rcu/tree_plugin.h | 12 ++---
kernel/rcu/tree_trace.c | 22 ++++-----
kernel/sched/auto_group.c | 4 +-
kernel/time/timer_stats.c | 10 ++--
kernel/trace/blktrace.c | 6 +--
kernel/trace/ftrace.c | 4 +-
kernel/trace/ring_buffer.c | 96 ++++++++++++++++++------------------
kernel/trace/trace_clock.c | 4 +-
kernel/trace/trace_functions_graph.c | 4 +-
kernel/trace/trace_mmiotrace.c | 8 +--
31 files changed, 224 insertions(+), 224 deletions(-)
diff --git a/include/linux/blktrace_api.h b/include/linux/blktrace_api.h
index afc1343..9735539 100644
--- a/include/linux/blktrace_api.h
+++ b/include/linux/blktrace_api.h
@@ -25,7 +25,7 @@ struct blk_trace {
struct dentry *dropped_file;
struct dentry *msg_file;
struct list_head running_list;
- atomic_t dropped;
+ atomic_unchecked_t dropped;
};
extern int blk_trace_ioctl(struct block_device *, unsigned, char __user *);
diff --git a/include/linux/irqdesc.h b/include/linux/irqdesc.h
index fcea4e4..cff381d 100644
--- a/include/linux/irqdesc.h
+++ b/include/linux/irqdesc.h
@@ -59,7 +59,7 @@ struct irq_desc {
unsigned int irq_count; /* For detecting broken IRQs */
unsigned long last_unhandled; /* Aging timer for unhandled count */
unsigned int irqs_unhandled;
- atomic_t threads_handled;
+ atomic_unchecked_t threads_handled;
int threads_handled_last;
raw_spinlock_t lock;
struct cpumask *percpu_enabled;
diff --git a/include/linux/kgdb.h b/include/linux/kgdb.h
index e465bb1..9226a23 100644
--- a/include/linux/kgdb.h
+++ b/include/linux/kgdb.h
@@ -52,7 +52,7 @@ extern int kgdb_connected;
extern int kgdb_io_module_registered;
extern atomic_t kgdb_setting_breakpoint;
-extern atomic_t kgdb_cpu_doing_single_step;
+extern atomic_unchecked_t kgdb_cpu_doing_single_step;
extern struct task_struct *kgdb_usethread;
extern struct task_struct *kgdb_contthread;
diff --git a/include/linux/oprofile.h b/include/linux/oprofile.h
index b2a0f15..4d7da32 100644
--- a/include/linux/oprofile.h
+++ b/include/linux/oprofile.h
@@ -138,9 +138,9 @@ int oprofilefs_create_ulong(struct dentry * root,
int oprofilefs_create_ro_ulong(struct dentry * root,
char const * name, ulong * val);
-/** Create a file for read-only access to an atomic_t. */
+/** Create a file for read-only access to an atomic_unchecked_t. */
int oprofilefs_create_ro_atomic(struct dentry * root,
- char const * name, atomic_t * val);
+ char const * name, atomic_unchecked_t * val);
/** create a directory */
struct dentry *oprofilefs_mkdir(struct dentry *parent, char const *name);
diff --git a/include/linux/padata.h b/include/linux/padata.h
index 4386946..f50c615 100644
--- a/include/linux/padata.h
+++ b/include/linux/padata.h
@@ -129,7 +129,7 @@ struct parallel_data {
struct padata_serial_queue __percpu *squeue;
atomic_t reorder_objects;
atomic_t refcnt;
- atomic_t seq_nr;
+ atomic_unchecked_t seq_nr;
struct padata_cpumask cpumask;
spinlock_t lock ____cacheline_aligned;
unsigned int processed;
diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h
index 2027809..7e35ed1 100644
--- a/include/linux/perf_event.h
+++ b/include/linux/perf_event.h
@@ -385,7 +385,7 @@ struct perf_event {
enum perf_event_active_state state;
unsigned int attach_state;
local64_t count;
- atomic64_t child_count;
+ atomic64_unchecked_t child_count;
/*
* These are the total time in nanoseconds that the event
@@ -436,8 +436,8 @@ struct perf_event {
* These accumulate total time (in nanoseconds) that children
* events have been enabled and running, respectively.
*/
- atomic64_t child_total_time_enabled;
- atomic64_t child_total_time_running;
+ atomic64_unchecked_t child_total_time_enabled;
+ atomic64_unchecked_t child_total_time_running;
/*
* Protect attach/detach and child_list:
@@ -859,7 +859,7 @@ static inline void perf_event_task_sched_out(struct task_struct *prev,
static inline u64 __perf_event_count(struct perf_event *event)
{
- return local64_read(&event->count) + atomic64_read(&event->child_count);
+ return local64_read(&event->count) + atomic64_read_unchecked(&event->child_count);
}
extern void perf_event_mmap(struct vm_area_struct *vma);
diff --git a/include/linux/sched.h b/include/linux/sched.h
index 20cf887..549ddc7 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -1770,7 +1770,7 @@ struct task_struct {
* Number of functions that haven't been traced
* because of depth overrun.
*/
- atomic_t trace_overrun;
+ atomic_unchecked_t trace_overrun;
/* Pause for the tracing */
atomic_t tracing_graph_pause;
#endif
diff --git a/include/video/udlfb.h b/include/video/udlfb.h
index f9466fa..f4e2b81 100644
--- a/include/video/udlfb.h
+++ b/include/video/udlfb.h
@@ -53,10 +53,10 @@ struct dlfb_data {
u32 pseudo_palette[256];
int blank_mode; /*one of FB_BLANK_ */
/* blit-only rendering path metrics, exposed through sysfs */
- atomic_t bytes_rendered; /* raw pixel-bytes driver asked to render */
- atomic_t bytes_identical; /* saved effort with backbuffer comparison */
- atomic_t bytes_sent; /* to usb, after compression including overhead */
- atomic_t cpu_kcycles_used; /* transpired during pixel processing */
+ atomic_unchecked_t bytes_rendered; /* raw pixel-bytes driver asked to render */
+ atomic_unchecked_t bytes_identical; /* saved effort with backbuffer comparison */
+ atomic_unchecked_t bytes_sent; /* to usb, after compression including overhead */
+ atomic_unchecked_t cpu_kcycles_used; /* transpired during pixel processing */
};
#define NR_USB_REQUEST_I2C_SUB_IO 0x02
diff --git a/kernel/audit.c b/kernel/audit.c
index f9e6065..3fcb6ab 100644
--- a/kernel/audit.c
+++ b/kernel/audit.c
@@ -124,7 +124,7 @@ u32 audit_sig_sid = 0;
3) suppressed due to audit_rate_limit
4) suppressed due to audit_backlog_limit
*/
-static atomic_t audit_lost = ATOMIC_INIT(0);
+static atomic_unchecked_t audit_lost = ATOMIC_INIT(0);
/* The netlink socket. */
static struct sock *audit_sock;
@@ -258,7 +258,7 @@ void audit_log_lost(const char *message)
unsigned long now;
int print;
- atomic_inc(&audit_lost);
+ atomic_inc_unchecked(&audit_lost);
print = (audit_failure == AUDIT_FAIL_PANIC || !audit_rate_limit);
@@ -275,7 +275,7 @@ void audit_log_lost(const char *message)
if (print) {
if (printk_ratelimit())
pr_warn("audit_lost=%u audit_rate_limit=%u audit_backlog_limit=%u\n",
- atomic_read(&audit_lost),
+ atomic_read_unchecked(&audit_lost),
audit_rate_limit,
audit_backlog_limit);
audit_panic(message);
@@ -833,7 +833,7 @@ static int audit_receive_msg(struct sk_buff *skb, struct nlmsghdr *nlh)
s.pid = audit_pid;
s.rate_limit = audit_rate_limit;
s.backlog_limit = audit_backlog_limit;
- s.lost = atomic_read(&audit_lost);
+ s.lost = atomic_read_unchecked(&audit_lost);
s.backlog = skb_queue_len(&audit_skb_queue);
s.feature_bitmap = AUDIT_FEATURE_BITMAP_ALL;
s.backlog_wait_time = audit_backlog_wait_time_master;
diff --git a/kernel/auditsc.c b/kernel/auditsc.c
index e85bdfd..f964d33 100644
--- a/kernel/auditsc.c
+++ b/kernel/auditsc.c
@@ -1952,7 +1952,7 @@ int auditsc_get_stamp(struct audit_context *ctx,
}
/* global counter which is incremented every time something logs in */
-static atomic_t session_id = ATOMIC_INIT(0);
+static atomic_unchecked_t session_id = ATOMIC_INIT(0);
static int audit_set_loginuid_perm(kuid_t loginuid)
{
@@ -2019,7 +2019,7 @@ int audit_set_loginuid(kuid_t loginuid)
/* are we setting or clearing? */
if (uid_valid(loginuid))
- sessionid = (unsigned int)atomic_inc_return(&session_id);
+ sessionid = (unsigned int)atomic_inc_return_unchecked(&session_id);
task->sessionid = sessionid;
task->loginuid = loginuid;
diff --git a/kernel/debug/debug_core.c b/kernel/debug/debug_core.c
index 0874e2e..5b32cc9 100644
--- a/kernel/debug/debug_core.c
+++ b/kernel/debug/debug_core.c
@@ -127,7 +127,7 @@ static DEFINE_RAW_SPINLOCK(dbg_slave_lock);
*/
static atomic_t masters_in_kgdb;
static atomic_t slaves_in_kgdb;
-static atomic_t kgdb_break_tasklet_var;
+static atomic_unchecked_t kgdb_break_tasklet_var;
atomic_t kgdb_setting_breakpoint;
struct task_struct *kgdb_usethread;
@@ -137,7 +137,7 @@ int kgdb_single_step;
static pid_t kgdb_sstep_pid;
/* to keep track of the CPU which is doing the single stepping*/
-atomic_t kgdb_cpu_doing_single_step = ATOMIC_INIT(-1);
+atomic_unchecked_t kgdb_cpu_doing_single_step = ATOMIC_INIT(-1);
/*
* If you are debugging a problem where roundup (the collection of
@@ -552,7 +552,7 @@ return_normal:
* kernel will only try for the value of sstep_tries before
* giving up and continuing on.
*/
- if (atomic_read(&kgdb_cpu_doing_single_step) != -1 &&
+ if (atomic_read_unchecked(&kgdb_cpu_doing_single_step) != -1 &&
(kgdb_info[cpu].task &&
kgdb_info[cpu].task->pid != kgdb_sstep_pid) && --sstep_tries) {
atomic_set(&kgdb_active, -1);
@@ -654,8 +654,8 @@ cpu_master_loop:
}
kgdb_restore:
- if (atomic_read(&kgdb_cpu_doing_single_step) != -1) {
- int sstep_cpu = atomic_read(&kgdb_cpu_doing_single_step);
+ if (atomic_read_unchecked(&kgdb_cpu_doing_single_step) != -1) {
+ int sstep_cpu = atomic_read_unchecked(&kgdb_cpu_doing_single_step);
if (kgdb_info[sstep_cpu].task)
kgdb_sstep_pid = kgdb_info[sstep_cpu].task->pid;
else
@@ -949,18 +949,18 @@ static void kgdb_unregister_callbacks(void)
static void kgdb_tasklet_bpt(unsigned long ing)
{
kgdb_breakpoint();
- atomic_set(&kgdb_break_tasklet_var, 0);
+ atomic_set_unchecked(&kgdb_break_tasklet_var, 0);
}
static DECLARE_TASKLET(kgdb_tasklet_breakpoint, kgdb_tasklet_bpt, 0);
void kgdb_schedule_breakpoint(void)
{
- if (atomic_read(&kgdb_break_tasklet_var) ||
+ if (atomic_read_unchecked(&kgdb_break_tasklet_var) ||
atomic_read(&kgdb_active) != -1 ||
atomic_read(&kgdb_setting_breakpoint))
return;
- atomic_inc(&kgdb_break_tasklet_var);
+ atomic_inc_unchecked(&kgdb_break_tasklet_var);
tasklet_schedule(&kgdb_tasklet_breakpoint);
}
EXPORT_SYMBOL_GPL(kgdb_schedule_breakpoint);
diff --git a/kernel/events/core.c b/kernel/events/core.c
index e6feb51..b83b393 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -307,7 +307,7 @@ void perf_sample_event_took(u64 sample_len_ns)
}
}
-static atomic64_t perf_event_id;
+static atomic64_unchecked_t perf_event_id;
static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
enum event_type_t event_type);
@@ -3753,9 +3753,9 @@ u64 perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running)
mutex_lock(&event->child_mutex);
total += perf_event_read(event);
*enabled += event->total_time_enabled +
- atomic64_read(&event->child_total_time_enabled);
+ atomic64_read_unchecked(&event->child_total_time_enabled);
*running += event->total_time_running +
- atomic64_read(&event->child_total_time_running);
+ atomic64_read_unchecked(&event->child_total_time_running);
list_for_each_entry(child, &event->child_list, child_list) {
total += perf_event_read(child);
@@ -4285,10 +4285,10 @@ void perf_event_update_userpage(struct perf_event *event)
userpg->offset -= local64_read(&event->hw.prev_count);
userpg->time_enabled = enabled +
- atomic64_read(&event->child_total_time_enabled);
+ atomic64_read_unchecked(&event->child_total_time_enabled);
userpg->time_running = running +
- atomic64_read(&event->child_total_time_running);
+ atomic64_read_unchecked(&event->child_total_time_running);
arch_perf_update_userpage(event, userpg, now);
@@ -5054,11 +5054,11 @@ static void perf_output_read_one(struct perf_output_handle *handle,
values[n++] = perf_event_count(event);
if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
values[n++] = enabled +
- atomic64_read(&event->child_total_time_enabled);
+ atomic64_read_unchecked(&event->child_total_time_enabled);
}
if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
values[n++] = running +
- atomic64_read(&event->child_total_time_running);
+ atomic64_read_unchecked(&event->child_total_time_running);
}
if (read_format & PERF_FORMAT_ID)
values[n++] = primary_event_id(event);
@@ -7588,7 +7588,7 @@ perf_event_alloc(struct perf_event_attr *attr, int cpu,
event->parent = parent_event;
event->ns = get_pid_ns(task_active_pid_ns(current));
- event->id = atomic64_inc_return(&perf_event_id);
+ event->id = atomic64_inc_return_unchecked(&perf_event_id);
event->state = PERF_EVENT_STATE_INACTIVE;
@@ -8395,10 +8395,10 @@ static void sync_child_event(struct perf_event *child_event,
/*
* Add back the child's count to the parent's count:
*/
- atomic64_add(child_val, &parent_event->child_count);
- atomic64_add(child_event->total_time_enabled,
+ atomic64_add_unchecked(child_val, &parent_event->child_count);
+ atomic64_add_unchecked(child_event->total_time_enabled,
&parent_event->child_total_time_enabled);
- atomic64_add(child_event->total_time_running,
+ atomic64_add_unchecked(child_event->total_time_running,
&parent_event->child_total_time_running);
/*
diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c
index f974485..c5b8afd 100644
--- a/kernel/irq/manage.c
+++ b/kernel/irq/manage.c
@@ -937,7 +937,7 @@ static int irq_thread(void *data)
action_ret = handler_fn(desc, action);
if (action_ret == IRQ_HANDLED)
- atomic_inc(&desc->threads_handled);
+ atomic_inc_unchecked(&desc->threads_handled);
wake_threads_waitq(desc);
}
diff --git a/kernel/irq/spurious.c b/kernel/irq/spurious.c
index e2514b0..de3dfe0 100644
--- a/kernel/irq/spurious.c
+++ b/kernel/irq/spurious.c
@@ -337,7 +337,7 @@ void note_interrupt(unsigned int irq, struct irq_desc *desc,
* count. We just care about the count being
* different than the one we saw before.
*/
- handled = atomic_read(&desc->threads_handled);
+ handled = atomic_read_unchecked(&desc->threads_handled);
handled |= SPURIOUS_DEFERRED;
if (handled != desc->threads_handled_last) {
action_ret = IRQ_HANDLED;
diff --git a/kernel/locking/lockdep.c b/kernel/locking/lockdep.c
index 8acfbf7..b9c128d 100644
--- a/kernel/locking/lockdep.c
+++ b/kernel/locking/lockdep.c
@@ -3102,7 +3102,7 @@ static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass,
if (!class)
return 0;
}
- atomic_inc((atomic_t *)&class->ops);
+ atomic_long_inc_unchecked((atomic_long_unchecked_t *)&class->ops);
if (very_verbose(class)) {
printk("\nacquire class [%p] %s", class->key, class->name);
if (class->name_version > 1)
diff --git a/kernel/locking/rtmutex-tester.c b/kernel/locking/rtmutex-tester.c
index 1d96dd0..994ff19 100644
--- a/kernel/locking/rtmutex-tester.c
+++ b/kernel/locking/rtmutex-tester.c
@@ -22,7 +22,7 @@
#define MAX_RT_TEST_MUTEXES 8
static spinlock_t rttest_lock;
-static atomic_t rttest_event;
+static atomic_unchecked_t rttest_event;
struct test_thread_data {
int opcode;
@@ -63,7 +63,7 @@ static int handle_op(struct test_thread_data *td, int lockwakeup)
case RTTEST_LOCKCONT:
td->mutexes[td->opdata] = 1;
- td->event = atomic_add_return(1, &rttest_event);
+ td->event = atomic_add_return_unchecked(1, &rttest_event);
return 0;
case RTTEST_RESET:
@@ -76,7 +76,7 @@ static int handle_op(struct test_thread_data *td, int lockwakeup)
return 0;
case RTTEST_RESETEVENT:
- atomic_set(&rttest_event, 0);
+ atomic_set_unchecked(&rttest_event, 0);
return 0;
default:
@@ -93,9 +93,9 @@ static int handle_op(struct test_thread_data *td, int lockwakeup)
return ret;
td->mutexes[id] = 1;
- td->event = atomic_add_return(1, &rttest_event);
+ td->event = atomic_add_return_unchecked(1, &rttest_event);
rt_mutex_lock(&mutexes[id]);
- td->event = atomic_add_return(1, &rttest_event);
+ td->event = atomic_add_return_unchecked(1, &rttest_event);
td->mutexes[id] = 4;
return 0;
@@ -106,9 +106,9 @@ static int handle_op(struct test_thread_data *td, int lockwakeup)
return ret;
td->mutexes[id] = 1;
- td->event = atomic_add_return(1, &rttest_event);
+ td->event = atomic_add_return_unchecked(1, &rttest_event);
ret = rt_mutex_lock_interruptible(&mutexes[id], 0);
- td->event = atomic_add_return(1, &rttest_event);
+ td->event = atomic_add_return_unchecked(1, &rttest_event);
td->mutexes[id] = ret ? 0 : 4;
return ret ? -EINTR : 0;
@@ -117,9 +117,9 @@ static int handle_op(struct test_thread_data *td, int lockwakeup)
if (id < 0 || id >= MAX_RT_TEST_MUTEXES || td->mutexes[id] != 4)
return ret;
- td->event = atomic_add_return(1, &rttest_event);
+ td->event = atomic_add_return_unchecked(1, &rttest_event);
rt_mutex_unlock(&mutexes[id]);
- td->event = atomic_add_return(1, &rttest_event);
+ td->event = atomic_add_return_unchecked(1, &rttest_event);
td->mutexes[id] = 0;
return 0;
@@ -166,7 +166,7 @@ void schedule_rt_mutex_test(struct rt_mutex *mutex)
break;
td->mutexes[dat] = 2;
- td->event = atomic_add_return(1, &rttest_event);
+ td->event = atomic_add_return_unchecked(1, &rttest_event);
break;
default:
@@ -186,7 +186,7 @@ void schedule_rt_mutex_test(struct rt_mutex *mutex)
return;
td->mutexes[dat] = 3;
- td->event = atomic_add_return(1, &rttest_event);
+ td->event = atomic_add_return_unchecked(1, &rttest_event);
break;
case RTTEST_LOCKNOWAIT:
@@ -198,7 +198,7 @@ void schedule_rt_mutex_test(struct rt_mutex *mutex)
return;
td->mutexes[dat] = 1;
- td->event = atomic_add_return(1, &rttest_event);
+ td->event = atomic_add_return_unchecked(1, &rttest_event);
return;
default:
diff --git a/kernel/padata.c b/kernel/padata.c
index b38bea9..91acfbe 100644
--- a/kernel/padata.c
+++ b/kernel/padata.c
@@ -54,7 +54,7 @@ static int padata_cpu_hash(struct parallel_data *pd)
* seq_nr mod. number of cpus in use.
*/
- seq_nr = atomic_inc_return(&pd->seq_nr);
+ seq_nr = atomic_inc_return_unchecked(&pd->seq_nr);
cpu_index = seq_nr % cpumask_weight(pd->cpumask.pcpu);
return padata_index_to_cpu(pd, cpu_index);
@@ -428,7 +428,7 @@ static struct parallel_data *padata_alloc_pd(struct padata_instance *pinst,
padata_init_pqueues(pd);
padata_init_squeues(pd);
setup_timer(&pd->timer, padata_reorder_timer, (unsigned long)pd);
- atomic_set(&pd->seq_nr, -1);
+ atomic_set_unchecked(&pd->seq_nr, -1);
atomic_set(&pd->reorder_objects, 0);
atomic_set(&pd->refcnt, 0);
pd->pinst = pinst;
diff --git a/kernel/profile.c b/kernel/profile.c
index a7bcd28..5b368fa 100644
--- a/kernel/profile.c
+++ b/kernel/profile.c
@@ -37,7 +37,7 @@ struct profile_hit {
#define NR_PROFILE_HIT (PAGE_SIZE/sizeof(struct profile_hit))
#define NR_PROFILE_GRP (NR_PROFILE_HIT/PROFILE_GRPSZ)
-static atomic_t *prof_buffer;
+static atomic_unchecked_t *prof_buffer;
static unsigned long prof_len, prof_shift;
int prof_on __read_mostly;
@@ -256,7 +256,7 @@ static void profile_flip_buffers(void)
hits[i].pc = 0;
continue;
}
- atomic_add(hits[i].hits, &prof_buffer[hits[i].pc]);
+ atomic_add_unchecked(hits[i].hits, &prof_buffer[hits[i].pc]);
hits[i].hits = hits[i].pc = 0;
}
}
@@ -317,9 +317,9 @@ static void do_profile_hits(int type, void *__pc, unsigned int nr_hits)
* Add the current hit(s) and flush the write-queue out
* to the global buffer:
*/
- atomic_add(nr_hits, &prof_buffer[pc]);
+ atomic_add_unchecked(nr_hits, &prof_buffer[pc]);
for (i = 0; i < NR_PROFILE_HIT; ++i) {
- atomic_add(hits[i].hits, &prof_buffer[hits[i].pc]);
+ atomic_add_unchecked(hits[i].hits, &prof_buffer[hits[i].pc]);
hits[i].pc = hits[i].hits = 0;
}
out:
@@ -394,7 +394,7 @@ static void do_profile_hits(int type, void *__pc, unsigned int nr_hits)
{
unsigned long pc;
pc = ((unsigned long)__pc - (unsigned long)_stext) >> prof_shift;
- atomic_add(nr_hits, &prof_buffer[min(pc, prof_len - 1)]);
+ atomic_add_unchecked(nr_hits, &prof_buffer[min(pc, prof_len - 1)]);
}
#endif /* !CONFIG_SMP */
@@ -489,7 +489,7 @@ read_profile(struct file *file, char __user *buf, size_t count, loff_t *ppos)
return -EFAULT;
buf++; p++; count--; read++;
}
- pnt = (char *)prof_buffer + p - sizeof(atomic_t);
+ pnt = (char *)prof_buffer + p - sizeof(atomic_unchecked_t);
if (copy_to_user(buf, (void *)pnt, count))
return -EFAULT;
read += count;
@@ -520,7 +520,7 @@ static ssize_t write_profile(struct file *file, const char __user *buf,
}
#endif
profile_discard_flip_buffers();
- memset(prof_buffer, 0, prof_len * sizeof(atomic_t));
+ memset(prof_buffer, 0, prof_len * sizeof(atomic_unchecked_t));
return count;
}
diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c
index 59e3268..d2eb3d9 100644
--- a/kernel/rcu/rcutorture.c
+++ b/kernel/rcu/rcutorture.c
@@ -134,12 +134,12 @@ static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1],
rcu_torture_count) = { 0 };
static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1],
rcu_torture_batch) = { 0 };
-static atomic_t rcu_torture_wcount[RCU_TORTURE_PIPE_LEN + 1];
-static atomic_t n_rcu_torture_alloc;
-static atomic_t n_rcu_torture_alloc_fail;
-static atomic_t n_rcu_torture_free;
-static atomic_t n_rcu_torture_mberror;
-static atomic_t n_rcu_torture_error;
+static atomic_unchecked_t rcu_torture_wcount[RCU_TORTURE_PIPE_LEN + 1];
+static atomic_unchecked_t n_rcu_torture_alloc;
+static atomic_unchecked_t n_rcu_torture_alloc_fail;
+static atomic_unchecked_t n_rcu_torture_free;
+static atomic_unchecked_t n_rcu_torture_mberror;
+static atomic_unchecked_t n_rcu_torture_error;
static long n_rcu_torture_barrier_error;
static long n_rcu_torture_boost_ktrerror;
static long n_rcu_torture_boost_rterror;
@@ -148,7 +148,7 @@ static long n_rcu_torture_boosts;
static long n_rcu_torture_timers;
static long n_barrier_attempts;
static long n_barrier_successes;
-static atomic_long_t n_cbfloods;
+static atomic_long_unchecked_t n_cbfloods;
static struct list_head rcu_torture_removed;
static int rcu_torture_writer_state;
@@ -211,11 +211,11 @@ rcu_torture_alloc(void)
spin_lock_bh(&rcu_torture_lock);
if (list_empty(&rcu_torture_freelist)) {
- atomic_inc(&n_rcu_torture_alloc_fail);
+ atomic_inc_unchecked(&n_rcu_torture_alloc_fail);
spin_unlock_bh(&rcu_torture_lock);
return NULL;
}
- atomic_inc(&n_rcu_torture_alloc);
+ atomic_inc_unchecked(&n_rcu_torture_alloc);
p = rcu_torture_freelist.next;
list_del_init(p);
spin_unlock_bh(&rcu_torture_lock);
@@ -228,7 +228,7 @@ rcu_torture_alloc(void)
static void
rcu_torture_free(struct rcu_torture *p)
{
- atomic_inc(&n_rcu_torture_free);
+ atomic_inc_unchecked(&n_rcu_torture_free);
spin_lock_bh(&rcu_torture_lock);
list_add_tail(&p->rtort_free, &rcu_torture_freelist);
spin_unlock_bh(&rcu_torture_lock);
@@ -309,7 +309,7 @@ rcu_torture_pipe_update_one(struct rcu_torture *rp)
i = rp->rtort_pipe_count;
if (i > RCU_TORTURE_PIPE_LEN)
i = RCU_TORTURE_PIPE_LEN;
- atomic_inc(&rcu_torture_wcount[i]);
+ atomic_inc_unchecked(&rcu_torture_wcount[i]);
if (++rp->rtort_pipe_count >= RCU_TORTURE_PIPE_LEN) {
rp->rtort_mbtest = 0;
return true;
@@ -830,7 +830,7 @@ rcu_torture_cbflood(void *arg)
VERBOSE_TOROUT_STRING("rcu_torture_cbflood task started");
do {
schedule_timeout_interruptible(cbflood_inter_holdoff);
- atomic_long_inc(&n_cbfloods);
+ atomic_long_inc_unchecked(&n_cbfloods);
WARN_ON(signal_pending(current));
for (i = 0; i < cbflood_n_burst; i++) {
for (j = 0; j < cbflood_n_per_burst; j++) {
@@ -957,7 +957,7 @@ rcu_torture_writer(void *arg)
i = old_rp->rtort_pipe_count;
if (i > RCU_TORTURE_PIPE_LEN)
i = RCU_TORTURE_PIPE_LEN;
- atomic_inc(&rcu_torture_wcount[i]);
+ atomic_inc_unchecked(&rcu_torture_wcount[i]);
old_rp->rtort_pipe_count++;
switch (synctype[torture_random(&rand) % nsynctypes]) {
case RTWS_DEF_FREE:
@@ -1095,7 +1095,7 @@ static void rcu_torture_timer(unsigned long unused)
return;
}
if (p->rtort_mbtest == 0)
- atomic_inc(&n_rcu_torture_mberror);
+ atomic_inc_unchecked(&n_rcu_torture_mberror);
spin_lock(&rand_lock);
cur_ops->read_delay(&rand);
n_rcu_torture_timers++;
@@ -1170,7 +1170,7 @@ rcu_torture_reader(void *arg)
continue;
}
if (p->rtort_mbtest == 0)
- atomic_inc(&n_rcu_torture_mberror);
+ atomic_inc_unchecked(&n_rcu_torture_mberror);
cur_ops->read_delay(&rand);
preempt_disable();
pipe_count = p->rtort_pipe_count;
@@ -1239,11 +1239,11 @@ rcu_torture_stats_print(void)
rcu_torture_current,
rcu_torture_current_version,
list_empty(&rcu_torture_freelist),
- atomic_read(&n_rcu_torture_alloc),
- atomic_read(&n_rcu_torture_alloc_fail),
- atomic_read(&n_rcu_torture_free));
+ atomic_read_unchecked(&n_rcu_torture_alloc),
+ atomic_read_unchecked(&n_rcu_torture_alloc_fail),
+ atomic_read_unchecked(&n_rcu_torture_free));
pr_cont("rtmbe: %d rtbke: %ld rtbre: %ld ",
- atomic_read(&n_rcu_torture_mberror),
+ atomic_read_unchecked(&n_rcu_torture_mberror),
n_rcu_torture_boost_ktrerror,
n_rcu_torture_boost_rterror);
pr_cont("rtbf: %ld rtb: %ld nt: %ld ",
@@ -1255,17 +1255,17 @@ rcu_torture_stats_print(void)
n_barrier_successes,
n_barrier_attempts,
n_rcu_torture_barrier_error);
- pr_cont("cbflood: %ld\n", atomic_long_read(&n_cbfloods));
+ pr_cont("cbflood: %ld\n", atomic_long_read_unchecked(&n_cbfloods));
pr_alert("%s%s ", torture_type, TORTURE_FLAG);
- if (atomic_read(&n_rcu_torture_mberror) != 0 ||
+ if (atomic_read_unchecked(&n_rcu_torture_mberror) != 0 ||
n_rcu_torture_barrier_error != 0 ||
n_rcu_torture_boost_ktrerror != 0 ||
n_rcu_torture_boost_rterror != 0 ||
n_rcu_torture_boost_failure != 0 ||
i > 1) {
pr_cont("%s", "!!! ");
- atomic_inc(&n_rcu_torture_error);
+ atomic_inc_unchecked(&n_rcu_torture_error);
WARN_ON_ONCE(1);
}
pr_cont("Reader Pipe: ");
@@ -1282,7 +1282,7 @@ rcu_torture_stats_print(void)
pr_alert("%s%s ", torture_type, TORTURE_FLAG);
pr_cont("Free-Block Circulation: ");
for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
- pr_cont(" %d", atomic_read(&rcu_torture_wcount[i]));
+ pr_cont(" %d", atomic_read_unchecked(&rcu_torture_wcount[i]));
}
pr_cont("\n");
@@ -1636,7 +1636,7 @@ rcu_torture_cleanup(void)
rcu_torture_stats_print(); /* -After- the stats thread is stopped! */
- if (atomic_read(&n_rcu_torture_error) || n_rcu_torture_barrier_error)
+ if (atomic_read_unchecked(&n_rcu_torture_error) || n_rcu_torture_barrier_error)
rcu_torture_print_module_parms(cur_ops, "End of test: FAILURE");
else if (torture_onoff_failures())
rcu_torture_print_module_parms(cur_ops,
@@ -1761,18 +1761,18 @@ rcu_torture_init(void)
rcu_torture_current = NULL;
rcu_torture_current_version = 0;
- atomic_set(&n_rcu_torture_alloc, 0);
- atomic_set(&n_rcu_torture_alloc_fail, 0);
- atomic_set(&n_rcu_torture_free, 0);
- atomic_set(&n_rcu_torture_mberror, 0);
- atomic_set(&n_rcu_torture_error, 0);
+ atomic_set_unchecked(&n_rcu_torture_alloc, 0);
+ atomic_set_unchecked(&n_rcu_torture_alloc_fail, 0);
+ atomic_set_unchecked(&n_rcu_torture_free, 0);
+ atomic_set_unchecked(&n_rcu_torture_mberror, 0);
+ atomic_set_unchecked(&n_rcu_torture_error, 0);
n_rcu_torture_barrier_error = 0;
n_rcu_torture_boost_ktrerror = 0;
n_rcu_torture_boost_rterror = 0;
n_rcu_torture_boost_failure = 0;
n_rcu_torture_boosts = 0;
for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
- atomic_set(&rcu_torture_wcount[i], 0);
+ atomic_set_unchecked(&rcu_torture_wcount[i], 0);
for_each_possible_cpu(cpu) {
for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
per_cpu(rcu_torture_count, cpu)[i] = 0;
diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 65137bc..0827851 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -326,7 +326,7 @@ static void rcu_momentary_dyntick_idle(void)
*/
rdtp = this_cpu_ptr(&rcu_dynticks);
smp_mb__before_atomic(); /* Earlier stuff before QS. */
- atomic_add(2, &rdtp->dynticks); /* QS. */
+ atomic_add_unchecked(2, &rdtp->dynticks); /* QS. */
smp_mb__after_atomic(); /* Later stuff after QS. */
break;
}
@@ -639,10 +639,10 @@ static void rcu_eqs_enter_common(long long oldval, bool user)
rcu_prepare_for_idle();
/* CPUs seeing atomic_inc() must see prior RCU read-side crit sects */
smp_mb__before_atomic(); /* See above. */
- atomic_inc(&rdtp->dynticks);
+ atomic_inc_unchecked(&rdtp->dynticks);
smp_mb__after_atomic(); /* Force ordering with next sojourn. */
WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
- atomic_read(&rdtp->dynticks) & 0x1);
+ atomic_read_unchecked(&rdtp->dynticks) & 0x1);
rcu_dynticks_task_enter();
/*
@@ -765,11 +765,11 @@ static void rcu_eqs_exit_common(long long oldval, int user)
rcu_dynticks_task_exit();
smp_mb__before_atomic(); /* Force ordering w/previous sojourn. */
- atomic_inc(&rdtp->dynticks);
+ atomic_inc_unchecked(&rdtp->dynticks);
/* CPUs seeing atomic_inc() must see later RCU read-side crit sects */
smp_mb__after_atomic(); /* See above. */
WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
- !(atomic_read(&rdtp->dynticks) & 0x1));
+ !(atomic_read_unchecked(&rdtp->dynticks) & 0x1));
rcu_cleanup_after_idle();
trace_rcu_dyntick(TPS("End"), oldval, rdtp->dynticks_nesting);
if (IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
@@ -905,12 +905,12 @@ void rcu_nmi_enter(void)
* to be in the outermost NMI handler that interrupted an RCU-idle
* period (observation due to Andy Lutomirski).
*/
- if (!(atomic_read(&rdtp->dynticks) & 0x1)) {
+ if (!(atomic_read_unchecked(&rdtp->dynticks) & 0x1)) {
smp_mb__before_atomic(); /* Force delay from prior write. */
- atomic_inc(&rdtp->dynticks);
+ atomic_inc_unchecked(&rdtp->dynticks);
/* atomic_inc() before later RCU read-side crit sects */
smp_mb__after_atomic(); /* See above. */
- WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks) & 0x1));
+ WARN_ON_ONCE(!(atomic_read_unchecked(&rdtp->dynticks) & 0x1));
incby = 1;
}
rdtp->dynticks_nmi_nesting += incby;
@@ -935,7 +935,7 @@ void rcu_nmi_exit(void)
* to us!)
*/
WARN_ON_ONCE(rdtp->dynticks_nmi_nesting <= 0);
- WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks) & 0x1));
+ WARN_ON_ONCE(!(atomic_read_unchecked(&rdtp->dynticks) & 0x1));
/*
* If the nesting level is not 1, the CPU wasn't RCU-idle, so
@@ -950,9 +950,9 @@ void rcu_nmi_exit(void)
rdtp->dynticks_nmi_nesting = 0;
/* CPUs seeing atomic_inc() must see prior RCU read-side crit sects */
smp_mb__before_atomic(); /* See above. */
- atomic_inc(&rdtp->dynticks);
+ atomic_inc_unchecked(&rdtp->dynticks);
smp_mb__after_atomic(); /* Force delay to next write. */
- WARN_ON_ONCE(atomic_read(&rdtp->dynticks) & 0x1);
+ WARN_ON_ONCE(atomic_read_unchecked(&rdtp->dynticks) & 0x1);
}
/**
@@ -965,7 +965,7 @@ void rcu_nmi_exit(void)
*/
bool notrace __rcu_is_watching(void)
{
- return atomic_read(this_cpu_ptr(&rcu_dynticks.dynticks)) & 0x1;
+ return atomic_read_unchecked(this_cpu_ptr(&rcu_dynticks.dynticks)) & 0x1;
}
/**
@@ -1048,7 +1048,7 @@ static int rcu_is_cpu_rrupt_from_idle(void)
static int dyntick_save_progress_counter(struct rcu_data *rdp,
bool *isidle, unsigned long *maxj)
{
- rdp->dynticks_snap = atomic_add_return(0, &rdp->dynticks->dynticks);
+ rdp->dynticks_snap = atomic_add_return_unchecked(0, &rdp->dynticks->dynticks);
rcu_sysidle_check_cpu(rdp, isidle, maxj);
if ((rdp->dynticks_snap & 0x1) == 0) {
trace_rcu_fqs(rdp->rsp->name, rdp->gpnum, rdp->cpu, TPS("dti"));
@@ -1074,7 +1074,7 @@ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp,
int *rcrmp;
unsigned int snap;
- curr = (unsigned int)atomic_add_return(0, &rdp->dynticks->dynticks);
+ curr = (unsigned int)atomic_add_return_unchecked(0, &rdp->dynticks->dynticks);
snap = (unsigned int)rdp->dynticks_snap;
/*
@@ -3319,11 +3319,11 @@ void synchronize_sched_expedited(void)
* counter wrap on a 32-bit system. Quite a few more CPUs would of
* course be required on a 64-bit system.
*/
- if (ULONG_CMP_GE((ulong)atomic_long_read(&rsp->expedited_start),
+ if (ULONG_CMP_GE((ulong)atomic_long_read_unchecked(&rsp->expedited_start),
(ulong)atomic_long_read(&rsp->expedited_done) +
ULONG_MAX / 8)) {
wait_rcu_gp(call_rcu_sched);
- atomic_long_inc(&rsp->expedited_wrap);
+ atomic_long_inc_return_unchecked(&rsp->expedited_wrap);
return;
}
@@ -3331,12 +3331,12 @@ void synchronize_sched_expedited(void)
* Take a ticket. Note that atomic_inc_return() implies a
* full memory barrier.
*/
- snap = atomic_long_inc_return(&rsp->expedited_start);
+ snap = atomic_long_inc_return_unchecked(&rsp->expedited_start);
firstsnap = snap;
if (!try_get_online_cpus()) {
/* CPU hotplug operation in flight, fall back to normal GP. */
wait_rcu_gp(call_rcu_sched);
- atomic_long_inc(&rsp->expedited_normal);
+ atomic_long_inc_unchecked(&rsp->expedited_normal);
return;
}
WARN_ON_ONCE(cpu_is_offline(raw_smp_processor_id()));
@@ -3349,7 +3349,7 @@ void synchronize_sched_expedited(void)
for_each_cpu(cpu, cm) {
struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu);
- if (!(atomic_add_return(0, &rdtp->dynticks) & 0x1))
+ if (!(atomic_add_return_unchecked(0, &rdtp->dynticks) & 0x1))
cpumask_clear_cpu(cpu, cm);
}
if (cpumask_weight(cm) == 0)
@@ -3364,14 +3364,14 @@ void synchronize_sched_expedited(void)
synchronize_sched_expedited_cpu_stop,
NULL) == -EAGAIN) {
put_online_cpus();
- atomic_long_inc(&rsp->expedited_tryfail);
+ atomic_long_inc_unchecked(&rsp->expedited_tryfail);
/* Check to see if someone else did our work for us. */
s = atomic_long_read(&rsp->expedited_done);
if (ULONG_CMP_GE((ulong)s, (ulong)firstsnap)) {
/* ensure test happens before caller kfree */
smp_mb__before_atomic(); /* ^^^ */
- atomic_long_inc(&rsp->expedited_workdone1);
+ atomic_long_inc_unchecked(&rsp->expedited_workdone1);
free_cpumask_var(cm);
return;
}
@@ -3381,7 +3381,7 @@ void synchronize_sched_expedited(void)
udelay(trycount * num_online_cpus());
} else {
wait_rcu_gp(call_rcu_sched);
- atomic_long_inc(&rsp->expedited_normal);
+ atomic_long_inc_unchecked(&rsp->expedited_normal);
free_cpumask_var(cm);
return;
}
@@ -3391,7 +3391,7 @@ void synchronize_sched_expedited(void)
if (ULONG_CMP_GE((ulong)s, (ulong)firstsnap)) {
/* ensure test happens before caller kfree */
smp_mb__before_atomic(); /* ^^^ */
- atomic_long_inc(&rsp->expedited_workdone2);
+ atomic_long_inc_unchecked(&rsp->expedited_workdone2);
free_cpumask_var(cm);
return;
}
@@ -3406,14 +3406,14 @@ void synchronize_sched_expedited(void)
if (!try_get_online_cpus()) {
/* CPU hotplug operation in flight, use normal GP. */
wait_rcu_gp(call_rcu_sched);
- atomic_long_inc(&rsp->expedited_normal);
+ atomic_long_inc_unchecked(&rsp->expedited_normal);
free_cpumask_var(cm);
return;
}
- snap = atomic_long_read(&rsp->expedited_start);
+ snap = atomic_long_read_unchecked(&rsp->expedited_start);
smp_mb(); /* ensure read is before try_stop_cpus(). */
}
- atomic_long_inc(&rsp->expedited_stoppedcpus);
+ atomic_long_inc_unchecked(&rsp->expedited_stoppedcpus);
all_cpus_idle:
free_cpumask_var(cm);
@@ -3425,16 +3425,16 @@ all_cpus_idle:
* than we did already did their update.
*/
do {
- atomic_long_inc(&rsp->expedited_done_tries);
+ atomic_long_inc_unchecked(&rsp->expedited_done_tries);
s = atomic_long_read(&rsp->expedited_done);
if (ULONG_CMP_GE((ulong)s, (ulong)snap)) {
/* ensure test happens before caller kfree */
smp_mb__before_atomic(); /* ^^^ */
- atomic_long_inc(&rsp->expedited_done_lost);
+ atomic_long_inc_unchecked(&rsp->expedited_done_lost);
break;
}
} while (atomic_long_cmpxchg(&rsp->expedited_done, s, snap) != s);
- atomic_long_inc(&rsp->expedited_done_exit);
+ atomic_long_inc_unchecked(&rsp->expedited_done_exit);
put_online_cpus();
}
@@ -3767,7 +3767,7 @@ rcu_boot_init_percpu_data(int cpu, struct rcu_state *rsp)
rdp->grpmask = 1UL << (cpu - rdp->mynode->grplo);
rdp->dynticks = &per_cpu(rcu_dynticks, cpu);
WARN_ON_ONCE(rdp->dynticks->dynticks_nesting != DYNTICK_TASK_EXIT_IDLE);
- WARN_ON_ONCE(atomic_read(&rdp->dynticks->dynticks) != 1);
+ WARN_ON_ONCE(atomic_read_unchecked(&rdp->dynticks->dynticks) != 1);
rdp->cpu = cpu;
rdp->rsp = rsp;
rcu_boot_init_nocb_percpu_data(rdp);
@@ -3798,8 +3798,8 @@ rcu_init_percpu_data(int cpu, struct rcu_state *rsp)
init_callback_list(rdp); /* Re-enable callbacks on this CPU. */
rdp->dynticks->dynticks_nesting = DYNTICK_TASK_EXIT_IDLE;
rcu_sysidle_init_percpu_data(rdp->dynticks);
- atomic_set(&rdp->dynticks->dynticks,
- (atomic_read(&rdp->dynticks->dynticks) & ~0x1) + 1);
+ atomic_set_unchecked(&rdp->dynticks->dynticks,
+ (atomic_read_unchecked(&rdp->dynticks->dynticks) & ~0x1) + 1);
raw_spin_unlock(&rnp->lock); /* irqs remain disabled. */
/*
diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h
index 4adb7ca..20910e6 100644
--- a/kernel/rcu/tree.h
+++ b/kernel/rcu/tree.h
@@ -108,11 +108,11 @@ struct rcu_dynticks {
long long dynticks_nesting; /* Track irq/process nesting level. */
/* Process level is worth LLONG_MAX/2. */
int dynticks_nmi_nesting; /* Track NMI nesting level. */
- atomic_t dynticks; /* Even value for idle, else odd. */
+ atomic_unchecked_t dynticks;/* Even value for idle, else odd. */
#ifdef CONFIG_NO_HZ_FULL_SYSIDLE
long long dynticks_idle_nesting;
/* irq/process nesting level from idle. */
- atomic_t dynticks_idle; /* Even value for idle, else odd. */
+ atomic_unchecked_t dynticks_idle;/* Even value for idle, else odd. */
/* "Idle" excludes userspace execution. */
unsigned long dynticks_idle_jiffies;
/* End of last non-NMI non-idle period. */
@@ -483,17 +483,17 @@ struct rcu_state {
/* _rcu_barrier(). */
/* End of fields guarded by barrier_mutex. */
- atomic_long_t expedited_start; /* Starting ticket. */
- atomic_long_t expedited_done; /* Done ticket. */
- atomic_long_t expedited_wrap; /* # near-wrap incidents. */
- atomic_long_t expedited_tryfail; /* # acquisition failures. */
- atomic_long_t expedited_workdone1; /* # done by others #1. */
- atomic_long_t expedited_workdone2; /* # done by others #2. */
- atomic_long_t expedited_normal; /* # fallbacks to normal. */
- atomic_long_t expedited_stoppedcpus; /* # successful stop_cpus. */
- atomic_long_t expedited_done_tries; /* # tries to update _done. */
- atomic_long_t expedited_done_lost; /* # times beaten to _done. */
- atomic_long_t expedited_done_exit; /* # times exited _done loop. */
+ atomic_long_unchecked_t expedited_start; /* Starting ticket. */
+ atomic_long_t expedited_done; /* Done ticket. */
+ atomic_long_unchecked_t expedited_wrap; /* # near-wrap incidents. */
+ atomic_long_unchecked_t expedited_tryfail; /* # acquisition failures. */
+ atomic_long_unchecked_t expedited_workdone1; /* # done by others #1. */
+ atomic_long_unchecked_t expedited_workdone2; /* # done by others #2. */
+ atomic_long_unchecked_t expedited_normal; /* # fallbacks to normal. */
+ atomic_long_unchecked_t expedited_stoppedcpus; /* # successful stop_cpus. */
+ atomic_long_unchecked_t expedited_done_tries; /* # tries to update _done. */
+ atomic_long_unchecked_t expedited_done_lost; /* # times beaten to _done. */
+ atomic_long_unchecked_t expedited_done_exit; /* # times exited _done loop. */
unsigned long jiffies_force_qs; /* Time at which to invoke */
/* force_quiescent_state(). */
diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h
index 013485f..a9fa510 100644
--- a/kernel/rcu/tree_plugin.h
+++ b/kernel/rcu/tree_plugin.h
@@ -1767,7 +1767,7 @@ static void print_cpu_stall_info(struct rcu_state *rsp, int cpu)
print_cpu_stall_fast_no_hz(fast_no_hz, cpu);
pr_err("\t%d: (%lu %s) idle=%03x/%llx/%d softirq=%u/%u fqs=%ld %s\n",
cpu, ticks_value, ticks_title,
- atomic_read(&rdtp->dynticks) & 0xfff,
+ atomic_read_unchecked(&rdtp->dynticks) & 0xfff,
rdtp->dynticks_nesting, rdtp->dynticks_nmi_nesting,
rdp->softirq_snap, kstat_softirqs_cpu(RCU_SOFTIRQ, cpu),
READ_ONCE(rsp->n_force_qs) - rsp->n_force_qs_gpstart,
@@ -2675,9 +2675,9 @@ static void rcu_sysidle_enter(int irq)
j = jiffies;
WRITE_ONCE(rdtp->dynticks_idle_jiffies, j);
smp_mb__before_atomic();
- atomic_inc(&rdtp->dynticks_idle);
+ atomic_inc_unchecked(&rdtp->dynticks_idle);
smp_mb__after_atomic();
- WARN_ON_ONCE(atomic_read(&rdtp->dynticks_idle) & 0x1);
+ WARN_ON_ONCE(atomic_read_unchecked(&rdtp->dynticks_idle) & 0x1);
}
/*
@@ -2748,9 +2748,9 @@ static void rcu_sysidle_exit(int irq)
/* Record end of idle period. */
smp_mb__before_atomic();
- atomic_inc(&rdtp->dynticks_idle);
+ atomic_inc_unchecked(&rdtp->dynticks_idle);
smp_mb__after_atomic();
- WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks_idle) & 0x1));
+ WARN_ON_ONCE(!(atomic_read_unchecked(&rdtp->dynticks_idle) & 0x1));
/*
* If we are the timekeeping CPU, we are permitted to be non-idle
@@ -2796,7 +2796,7 @@ static void rcu_sysidle_check_cpu(struct rcu_data *rdp, bool *isidle,
WARN_ON_ONCE(smp_processor_id() != tick_do_timer_cpu);
/* Pick up current idle and NMI-nesting counter and check. */
- cur = atomic_read(&rdtp->dynticks_idle);
+ cur = atomic_read_unchecked(&rdtp->dynticks_idle);
if (cur & 0x1) {
*isidle = false; /* We are not idle! */
return;
diff --git a/kernel/rcu/tree_trace.c b/kernel/rcu/tree_trace.c
index 3ea7ffc..cb06f2d 100644
--- a/kernel/rcu/tree_trace.c
+++ b/kernel/rcu/tree_trace.c
@@ -125,7 +125,7 @@ static void print_one_rcu_data(struct seq_file *m, struct rcu_data *rdp)
rdp->rcu_qs_ctr_snap == per_cpu(rcu_qs_ctr, rdp->cpu),
rdp->qs_pending);
seq_printf(m, " dt=%d/%llx/%d df=%lu",
- atomic_read(&rdp->dynticks->dynticks),
+ atomic_read_unchecked(&rdp->dynticks->dynticks),
rdp->dynticks->dynticks_nesting,
rdp->dynticks->dynticks_nmi_nesting,
rdp->dynticks_fqs);
@@ -186,17 +186,17 @@ static int show_rcuexp(struct seq_file *m, void *v)
struct rcu_state *rsp = (struct rcu_state *)m->private;
seq_printf(m, "s=%lu d=%lu w=%lu tf=%lu wd1=%lu wd2=%lu n=%lu sc=%lu dt=%lu dl=%lu dx=%lu\n",
- atomic_long_read(&rsp->expedited_start),
+ atomic_long_read_unchecked(&rsp->expedited_start),
atomic_long_read(&rsp->expedited_done),
- atomic_long_read(&rsp->expedited_wrap),
- atomic_long_read(&rsp->expedited_tryfail),
- atomic_long_read(&rsp->expedited_workdone1),
- atomic_long_read(&rsp->expedited_workdone2),
- atomic_long_read(&rsp->expedited_normal),
- atomic_long_read(&rsp->expedited_stoppedcpus),
- atomic_long_read(&rsp->expedited_done_tries),
- atomic_long_read(&rsp->expedited_done_lost),
- atomic_long_read(&rsp->expedited_done_exit));
+ atomic_long_read_unchecked(&rsp->expedited_wrap),
+ atomic_long_read_unchecked(&rsp->expedited_tryfail),
+ atomic_long_read_unchecked(&rsp->expedited_workdone1),
+ atomic_long_read_unchecked(&rsp->expedited_workdone2),
+ atomic_long_read_unchecked(&rsp->expedited_normal),
+ atomic_long_read_unchecked(&rsp->expedited_stoppedcpus),
+ atomic_long_read_unchecked(&rsp->expedited_done_tries),
+ atomic_long_read_unchecked(&rsp->expedited_done_lost),
+ atomic_long_read_unchecked(&rsp->expedited_done_exit));
return 0;
}
diff --git a/kernel/sched/auto_group.c b/kernel/sched/auto_group.c
index 750ed60..eb01466 100644
--- a/kernel/sched/auto_group.c
+++ b/kernel/sched/auto_group.c
@@ -9,7 +9,7 @@
unsigned int __read_mostly sysctl_sched_autogroup_enabled = 1;
static struct autogroup autogroup_default;
-static atomic_t autogroup_seq_nr;
+static atomic_unchecked_t autogroup_seq_nr;
void __init autogroup_init(struct task_struct *init_task)
{
@@ -77,7 +77,7 @@ static inline struct autogroup *autogroup_create(void)
kref_init(&ag->kref);
init_rwsem(&ag->lock);
- ag->id = atomic_inc_return(&autogroup_seq_nr);
+ ag->id = atomic_inc_return_unchecked(&autogroup_seq_nr);
ag->tg = tg;
#ifdef CONFIG_RT_GROUP_SCHED
/*
diff --git a/kernel/time/timer_stats.c b/kernel/time/timer_stats.c
index 1adecb4..56bce1f 100644
--- a/kernel/time/timer_stats.c
+++ b/kernel/time/timer_stats.c
@@ -116,7 +116,7 @@ static ktime_t time_start, time_stop;
static unsigned long nr_entries;
static struct entry entries[MAX_ENTRIES];
-static atomic_t overflow_count;
+static atomic_unchecked_t overflow_count;
/*
* The entries are in a hash-table, for fast lookup:
@@ -140,7 +140,7 @@ static void reset_entries(void)
nr_entries = 0;
memset(entries, 0, sizeof(entries));
memset(tstat_hash_table, 0, sizeof(tstat_hash_table));
- atomic_set(&overflow_count, 0);
+ atomic_set_unchecked(&overflow_count, 0);
}
static struct entry *alloc_entry(void)
@@ -261,7 +261,7 @@ void timer_stats_update_stats(void *timer, pid_t pid, void *startf,
if (likely(entry))
entry->count++;
else
- atomic_inc(&overflow_count);
+ atomic_inc_unchecked(&overflow_count);
out_unlock:
raw_spin_unlock_irqrestore(lock, flags);
@@ -300,8 +300,8 @@ static int tstats_show(struct seq_file *m, void *v)
seq_puts(m, "Timer Stats Version: v0.3\n");
seq_printf(m, "Sample period: %ld.%03ld s\n", period.tv_sec, ms);
- if (atomic_read(&overflow_count))
- seq_printf(m, "Overflow: %d entries\n", atomic_read(&overflow_count));
+ if (atomic_read_unchecked(&overflow_count))
+ seq_printf(m, "Overflow: %d entries\n", atomic_read_unchecked(&overflow_count));
seq_printf(m, "Collection: %s\n", timer_stats_active ? "active" : "inactive");
for (i = 0; i < nr_entries; i++) {
diff --git a/kernel/trace/blktrace.c b/kernel/trace/blktrace.c
index b3e6b39..719099b 100644
--- a/kernel/trace/blktrace.c
+++ b/kernel/trace/blktrace.c
@@ -328,7 +328,7 @@ static ssize_t blk_dropped_read(struct file *filp, char __user *buffer,
struct blk_trace *bt = filp->private_data;
char buf[16];
- snprintf(buf, sizeof(buf), "%u\n", atomic_read(&bt->dropped));
+ snprintf(buf, sizeof(buf), "%u\n", atomic_read_unchecked(&bt->dropped));
return simple_read_from_buffer(buffer, count, ppos, buf, strlen(buf));
}
@@ -386,7 +386,7 @@ static int blk_subbuf_start_callback(struct rchan_buf *buf, void *subbuf,
return 1;
bt = buf->chan->private_data;
- atomic_inc(&bt->dropped);
+ atomic_inc_unchecked(&bt->dropped);
return 0;
}
@@ -485,7 +485,7 @@ int do_blk_trace_setup(struct request_queue *q, char *name, dev_t dev,
bt->dir = dir;
bt->dev = dev;
- atomic_set(&bt->dropped, 0);
+ atomic_set_unchecked(&bt->dropped, 0);
INIT_LIST_HEAD(&bt->running_list);
ret = -EIO;
diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c
index eb11011..07bf7ce 100644
--- a/kernel/trace/ftrace.c
+++ b/kernel/trace/ftrace.c
@@ -5675,7 +5675,7 @@ static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list)
if (t->ret_stack == NULL) {
atomic_set(&t->tracing_graph_pause, 0);
- atomic_set(&t->trace_overrun, 0);
+ atomic_set_unchecked(&t->trace_overrun, 0);
t->curr_ret_stack = -1;
/* Make sure the tasks see the -1 first: */
smp_wmb();
@@ -5898,7 +5898,7 @@ static void
graph_init_task(struct task_struct *t, struct ftrace_ret_stack *ret_stack)
{
atomic_set(&t->tracing_graph_pause, 0);
- atomic_set(&t->trace_overrun, 0);
+ atomic_set_unchecked(&t->trace_overrun, 0);
t->ftrace_timestamp = 0;
/* make curr_ret_stack visible before we add the ret_stack */
smp_wmb();
diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c
index 6260717..b9bd83c 100644
--- a/kernel/trace/ring_buffer.c
+++ b/kernel/trace/ring_buffer.c
@@ -296,9 +296,9 @@ struct buffer_data_page {
*/
struct buffer_page {
struct list_head list; /* list of buffer pages */
- local_t write; /* index for next write */
+ local_unchecked_t write; /* index for next write */
unsigned read; /* index for next read */
- local_t entries; /* entries on this page */
+ local_unchecked_t entries; /* entries on this page */
unsigned long real_end; /* real end of data */
struct buffer_data_page *page; /* Actual data page */
};
@@ -437,11 +437,11 @@ struct ring_buffer_per_cpu {
unsigned long last_overrun;
local_t entries_bytes;
local_t entries;
- local_t overrun;
- local_t commit_overrun;
- local_t dropped_events;
+ local_unchecked_t overrun;
+ local_unchecked_t commit_overrun;
+ local_unchecked_t dropped_events;
local_t committing;
- local_t commits;
+ local_unchecked_t commits;
unsigned long read;
unsigned long read_bytes;
u64 write_stamp;
@@ -1011,8 +1011,8 @@ static int rb_tail_page_update(struct ring_buffer_per_cpu *cpu_buffer,
*
* We add a counter to the write field to denote this.
*/
- old_write = local_add_return(RB_WRITE_INTCNT, &next_page->write);
- old_entries = local_add_return(RB_WRITE_INTCNT, &next_page->entries);
+ old_write = local_add_return_unchecked(RB_WRITE_INTCNT, &next_page->write);
+ old_entries = local_add_return_unchecked(RB_WRITE_INTCNT, &next_page->entries);
/*
* Just make sure we have seen our old_write and synchronize
@@ -1040,8 +1040,8 @@ static int rb_tail_page_update(struct ring_buffer_per_cpu *cpu_buffer,
* cmpxchg to only update if an interrupt did not already
* do it for us. If the cmpxchg fails, we don't care.
*/
- (void)local_cmpxchg(&next_page->write, old_write, val);
- (void)local_cmpxchg(&next_page->entries, old_entries, eval);
+ (void)local_cmpxchg_unchecked(&next_page->write, old_write, val);
+ (void)local_cmpxchg_unchecked(&next_page->entries, old_entries, eval);
/*
* No need to worry about races with clearing out the commit.
@@ -1409,12 +1409,12 @@ static void rb_reset_cpu(struct ring_buffer_per_cpu *cpu_buffer);
static inline unsigned long rb_page_entries(struct buffer_page *bpage)
{
- return local_read(&bpage->entries) & RB_WRITE_MASK;
+ return local_read_unchecked(&bpage->entries) & RB_WRITE_MASK;
}
static inline unsigned long rb_page_write(struct buffer_page *bpage)
{
- return local_read(&bpage->write) & RB_WRITE_MASK;
+ return local_read_unchecked(&bpage->write) & RB_WRITE_MASK;
}
static int
@@ -1509,7 +1509,7 @@ rb_remove_pages(struct ring_buffer_per_cpu *cpu_buffer, unsigned int nr_pages)
* bytes consumed in ring buffer from here.
* Increment overrun to account for the lost events.
*/
- local_add(page_entries, &cpu_buffer->overrun);
+ local_add_unchecked(page_entries, &cpu_buffer->overrun);
local_sub(BUF_PAGE_SIZE, &cpu_buffer->entries_bytes);
}
@@ -2071,7 +2071,7 @@ rb_handle_head_page(struct ring_buffer_per_cpu *cpu_buffer,
* it is our responsibility to update
* the counters.
*/
- local_add(entries, &cpu_buffer->overrun);
+ local_add_unchecked(entries, &cpu_buffer->overrun);
local_sub(BUF_PAGE_SIZE, &cpu_buffer->entries_bytes);
/*
@@ -2221,7 +2221,7 @@ rb_reset_tail(struct ring_buffer_per_cpu *cpu_buffer,
if (tail == BUF_PAGE_SIZE)
tail_page->real_end = 0;
- local_sub(length, &tail_page->write);
+ local_sub_unchecked(length, &tail_page->write);
return;
}
@@ -2256,7 +2256,7 @@ rb_reset_tail(struct ring_buffer_per_cpu *cpu_buffer,
rb_event_set_padding(event);
/* Set the write back to the previous setting */
- local_sub(length, &tail_page->write);
+ local_sub_unchecked(length, &tail_page->write);
return;
}
@@ -2268,7 +2268,7 @@ rb_reset_tail(struct ring_buffer_per_cpu *cpu_buffer,
/* Set write to end of buffer */
length = (tail + length) - BUF_PAGE_SIZE;
- local_sub(length, &tail_page->write);
+ local_sub_unchecked(length, &tail_page->write);
}
/*
@@ -2294,7 +2294,7 @@ rb_move_tail(struct ring_buffer_per_cpu *cpu_buffer,
* about it.
*/
if (unlikely(next_page == commit_page)) {
- local_inc(&cpu_buffer->commit_overrun);
+ local_inc_unchecked(&cpu_buffer->commit_overrun);
goto out_reset;
}
@@ -2324,7 +2324,7 @@ rb_move_tail(struct ring_buffer_per_cpu *cpu_buffer,
* this is easy, just stop here.
*/
if (!(buffer->flags & RB_FL_OVERWRITE)) {
- local_inc(&cpu_buffer->dropped_events);
+ local_inc_unchecked(&cpu_buffer->dropped_events);
goto out_reset;
}
@@ -2350,7 +2350,7 @@ rb_move_tail(struct ring_buffer_per_cpu *cpu_buffer,
cpu_buffer->tail_page) &&
(cpu_buffer->commit_page ==
cpu_buffer->reader_page))) {
- local_inc(&cpu_buffer->commit_overrun);
+ local_inc_unchecked(&cpu_buffer->commit_overrun);
goto out_reset;
}
}
@@ -2398,7 +2398,7 @@ __rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer,
length += RB_LEN_TIME_EXTEND;
tail_page = cpu_buffer->tail_page;
- write = local_add_return(length, &tail_page->write);
+ write = local_add_return_unchecked(length, &tail_page->write);
/* set write to only the index of the write */
write &= RB_WRITE_MASK;
@@ -2422,7 +2422,7 @@ __rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer,
kmemcheck_annotate_bitfield(event, bitfield);
rb_update_event(cpu_buffer, event, length, add_timestamp, delta);
- local_inc(&tail_page->entries);
+ local_inc_unchecked(&tail_page->entries);
/*
* If this is the first commit on the page, then update
@@ -2455,7 +2455,7 @@ rb_try_to_discard(struct ring_buffer_per_cpu *cpu_buffer,
if (bpage->page == (void *)addr && rb_page_write(bpage) == old_index) {
unsigned long write_mask =
- local_read(&bpage->write) & ~RB_WRITE_MASK;
+ local_read_unchecked(&bpage->write) & ~RB_WRITE_MASK;
unsigned long event_length = rb_event_length(event);
/*
* This is on the tail page. It is possible that
@@ -2465,7 +2465,7 @@ rb_try_to_discard(struct ring_buffer_per_cpu *cpu_buffer,
*/
old_index += write_mask;
new_index += write_mask;
- index = local_cmpxchg(&bpage->write, old_index, new_index);
+ index = local_cmpxchg_unchecked(&bpage->write, old_index, new_index);
if (index == old_index) {
/* update counters */
local_sub(event_length, &cpu_buffer->entries_bytes);
@@ -2480,7 +2480,7 @@ rb_try_to_discard(struct ring_buffer_per_cpu *cpu_buffer,
static void rb_start_commit(struct ring_buffer_per_cpu *cpu_buffer)
{
local_inc(&cpu_buffer->committing);
- local_inc(&cpu_buffer->commits);
+ local_inc_unchecked(&cpu_buffer->commits);
}
static inline void rb_end_commit(struct ring_buffer_per_cpu *cpu_buffer)
@@ -2492,7 +2492,7 @@ static inline void rb_end_commit(struct ring_buffer_per_cpu *cpu_buffer)
return;
again:
- commits = local_read(&cpu_buffer->commits);
+ commits = local_read_unchecked(&cpu_buffer->commits);
/* synchronize with interrupts */
barrier();
if (local_read(&cpu_buffer->committing) == 1)
@@ -2508,7 +2508,7 @@ static inline void rb_end_commit(struct ring_buffer_per_cpu *cpu_buffer)
* updating of the commit page and the clearing of the
* committing counter.
*/
- if (unlikely(local_read(&cpu_buffer->commits) != commits) &&
+ if (unlikely(local_read_unchecked(&cpu_buffer->commits) != commits) &&
!local_read(&cpu_buffer->committing)) {
local_inc(&cpu_buffer->committing);
goto again;
@@ -2538,7 +2538,7 @@ rb_reserve_next_event(struct ring_buffer *buffer,
barrier();
if (unlikely(ACCESS_ONCE(cpu_buffer->buffer) != buffer)) {
local_dec(&cpu_buffer->committing);
- local_dec(&cpu_buffer->commits);
+ local_dec_unchecked(&cpu_buffer->commits);
return NULL;
}
#endif
@@ -2852,7 +2852,7 @@ rb_decrement_entry(struct ring_buffer_per_cpu *cpu_buffer,
/* Do the likely case first */
if (likely(bpage->page == (void *)addr)) {
- local_dec(&bpage->entries);
+ local_dec_unchecked(&bpage->entries);
return;
}
@@ -2864,7 +2864,7 @@ rb_decrement_entry(struct ring_buffer_per_cpu *cpu_buffer,
start = bpage;
do {
if (bpage->page == (void *)addr) {
- local_dec(&bpage->entries);
+ local_dec_unchecked(&bpage->entries);
return;
}
rb_inc_page(cpu_buffer, &bpage);
@@ -3152,7 +3152,7 @@ static inline unsigned long
rb_num_of_entries(struct ring_buffer_per_cpu *cpu_buffer)
{
return local_read(&cpu_buffer->entries) -
- (local_read(&cpu_buffer->overrun) + cpu_buffer->read);
+ (local_read_unchecked(&cpu_buffer->overrun) + cpu_buffer->read);
}
/**
@@ -3241,7 +3241,7 @@ unsigned long ring_buffer_overrun_cpu(struct ring_buffer *buffer, int cpu)
return 0;
cpu_buffer = buffer->buffers[cpu];
- ret = local_read(&cpu_buffer->overrun);
+ ret = local_read_unchecked(&cpu_buffer->overrun);
return ret;
}
@@ -3264,7 +3264,7 @@ ring_buffer_commit_overrun_cpu(struct ring_buffer *buffer, int cpu)
return 0;
cpu_buffer = buffer->buffers[cpu];
- ret = local_read(&cpu_buffer->commit_overrun);
+ ret = local_read_unchecked(&cpu_buffer->commit_overrun);
return ret;
}
@@ -3286,7 +3286,7 @@ ring_buffer_dropped_events_cpu(struct ring_buffer *buffer, int cpu)
return 0;
cpu_buffer = buffer->buffers[cpu];
- ret = local_read(&cpu_buffer->dropped_events);
+ ret = local_read_unchecked(&cpu_buffer->dropped_events);
return ret;
}
@@ -3349,7 +3349,7 @@ unsigned long ring_buffer_overruns(struct ring_buffer *buffer)
/* if you care about this being correct, lock the buffer */
for_each_buffer_cpu(buffer, cpu) {
cpu_buffer = buffer->buffers[cpu];
- overruns += local_read(&cpu_buffer->overrun);
+ overruns += local_read_unchecked(&cpu_buffer->overrun);
}
return overruns;
@@ -3520,8 +3520,8 @@ rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer)
/*
* Reset the reader page to size zero.
*/
- local_set(&cpu_buffer->reader_page->write, 0);
- local_set(&cpu_buffer->reader_page->entries, 0);
+ local_set_unchecked(&cpu_buffer->reader_page->write, 0);
+ local_set_unchecked(&cpu_buffer->reader_page->entries, 0);
local_set(&cpu_buffer->reader_page->page->commit, 0);
cpu_buffer->reader_page->real_end = 0;
@@ -3555,7 +3555,7 @@ rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer)
* want to compare with the last_overrun.
*/
smp_mb();
- overwrite = local_read(&(cpu_buffer->overrun));
+ overwrite = local_read_unchecked(&(cpu_buffer->overrun));
/*
* Here's the tricky part.
@@ -4137,8 +4137,8 @@ rb_reset_cpu(struct ring_buffer_per_cpu *cpu_buffer)
cpu_buffer->head_page
= list_entry(cpu_buffer->pages, struct buffer_page, list);
- local_set(&cpu_buffer->head_page->write, 0);
- local_set(&cpu_buffer->head_page->entries, 0);
+ local_set_unchecked(&cpu_buffer->head_page->write, 0);
+ local_set_unchecked(&cpu_buffer->head_page->entries, 0);
local_set(&cpu_buffer->head_page->page->commit, 0);
cpu_buffer->head_page->read = 0;
@@ -4148,18 +4148,18 @@ rb_reset_cpu(struct ring_buffer_per_cpu *cpu_buffer)
INIT_LIST_HEAD(&cpu_buffer->reader_page->list);
INIT_LIST_HEAD(&cpu_buffer->new_pages);
- local_set(&cpu_buffer->reader_page->write, 0);
- local_set(&cpu_buffer->reader_page->entries, 0);
+ local_set_unchecked(&cpu_buffer->reader_page->write, 0);
+ local_set_unchecked(&cpu_buffer->reader_page->entries, 0);
local_set(&cpu_buffer->reader_page->page->commit, 0);
cpu_buffer->reader_page->read = 0;
local_set(&cpu_buffer->entries_bytes, 0);
- local_set(&cpu_buffer->overrun, 0);
- local_set(&cpu_buffer->commit_overrun, 0);
- local_set(&cpu_buffer->dropped_events, 0);
+ local_set_unchecked(&cpu_buffer->overrun, 0);
+ local_set_unchecked(&cpu_buffer->commit_overrun, 0);
+ local_set_unchecked(&cpu_buffer->dropped_events, 0);
local_set(&cpu_buffer->entries, 0);
local_set(&cpu_buffer->committing, 0);
- local_set(&cpu_buffer->commits, 0);
+ local_set_unchecked(&cpu_buffer->commits, 0);
cpu_buffer->read = 0;
cpu_buffer->read_bytes = 0;
@@ -4549,8 +4549,8 @@ int ring_buffer_read_page(struct ring_buffer *buffer,
rb_init_page(bpage);
bpage = reader->page;
reader->page = *data_page;
- local_set(&reader->write, 0);
- local_set(&reader->entries, 0);
+ local_set_unchecked(&reader->write, 0);
+ local_set_unchecked(&reader->entries, 0);
reader->read = 0;
*data_page = bpage;
diff --git a/kernel/trace/trace_clock.c b/kernel/trace/trace_clock.c
index 0f06532..247c8e7 100644
--- a/kernel/trace/trace_clock.c
+++ b/kernel/trace/trace_clock.c
@@ -127,7 +127,7 @@ u64 notrace trace_clock_global(void)
}
EXPORT_SYMBOL_GPL(trace_clock_global);
-static atomic64_t trace_counter;
+static atomic64_unchecked_t trace_counter;
/*
* trace_clock_counter(): simply an atomic counter.
@@ -136,5 +136,5 @@ static atomic64_t trace_counter;
*/
u64 notrace trace_clock_counter(void)
{
- return atomic64_add_return(1, &trace_counter);
+ return atomic64_inc_return_unchecked(&trace_counter);
}
diff --git a/kernel/trace/trace_functions_graph.c b/kernel/trace/trace_functions_graph.c
index 8968bf7..e6623fc 100644
--- a/kernel/trace/trace_functions_graph.c
+++ b/kernel/trace/trace_functions_graph.c
@@ -132,7 +132,7 @@ ftrace_push_return_trace(unsigned long ret, unsigned long func, int *depth,
/* The return trace stack is full */
if (current->curr_ret_stack == FTRACE_RETFUNC_DEPTH - 1) {
- atomic_inc(¤t->trace_overrun);
+ atomic_inc_unchecked(¤t->trace_overrun);
return -EBUSY;
}
@@ -229,7 +229,7 @@ ftrace_pop_return_trace(struct ftrace_graph_ret *trace, unsigned long *ret,
*ret = current->ret_stack[index].ret;
trace->func = current->ret_stack[index].func;
trace->calltime = current->ret_stack[index].calltime;
- trace->overrun = atomic_read(¤t->trace_overrun);
+ trace->overrun = atomic_read_unchecked(¤t->trace_overrun);
trace->depth = index;
}
diff --git a/kernel/trace/trace_mmiotrace.c b/kernel/trace/trace_mmiotrace.c
index 638e110..99b73b2 100644
--- a/kernel/trace/trace_mmiotrace.c
+++ b/kernel/trace/trace_mmiotrace.c
@@ -24,7 +24,7 @@ struct header_iter {
static struct trace_array *mmio_trace_array;
static bool overrun_detected;
static unsigned long prev_overruns;
-static atomic_t dropped_count;
+static atomic_unchecked_t dropped_count;
static void mmio_reset_data(struct trace_array *tr)
{
@@ -124,7 +124,7 @@ static void mmio_close(struct trace_iterator *iter)
static unsigned long count_overruns(struct trace_iterator *iter)
{
- unsigned long cnt = atomic_xchg(&dropped_count, 0);
+ unsigned long cnt = atomic_xchg_unchecked(&dropped_count, 0);
unsigned long over = ring_buffer_overruns(iter->trace_buffer->buffer);
if (over > prev_overruns)
@@ -307,7 +307,7 @@ static void __trace_mmiotrace_rw(struct trace_array *tr,
event = trace_buffer_lock_reserve(buffer, TRACE_MMIO_RW,
sizeof(*entry), 0, pc);
if (!event) {
- atomic_inc(&dropped_count);
+ atomic_inc_unchecked(&dropped_count);
return;
}
entry = ring_buffer_event_data(event);
@@ -337,7 +337,7 @@ static void __trace_mmiotrace_map(struct trace_array *tr,
event = trace_buffer_lock_reserve(buffer, TRACE_MMIO_MAP,
sizeof(*entry), 0, pc);
if (!event) {
- atomic_inc(&dropped_count);
+ atomic_inc_unchecked(&dropped_count);
return;
}
entry = ring_buffer_event_data(event);
--
2.5.0
^ permalink raw reply related [flat|nested] 32+ messages in thread* [kernel-hardening][RFC PATCH v2 04/12] mm: opt out of PAX_REFCOUNT protection
2015-12-17 14:57 [kernel-hardening][RFC PATCH v2 00/12] Add PAX_REFCOUNT overflow protection David Windsor
` (2 preceding siblings ...)
2015-12-17 14:57 ` [kernel-hardening][RFC PATCH v2 03/12] kernel: opt out of PAX_REFCOUNT protection David Windsor
@ 2015-12-17 14:57 ` David Windsor
2015-12-17 14:57 ` [kernel-hardening][RFC PATCH v2 05/12] fs: " David Windsor
` (8 subsequent siblings)
12 siblings, 0 replies; 32+ messages in thread
From: David Windsor @ 2015-12-17 14:57 UTC (permalink / raw)
To: kernel-hardening; +Cc: David Windsor
Signed-off-by: David Windsor <dave@progbits.org>
---
include/linux/mm.h | 2 +-
include/linux/mmzone.h | 2 +-
include/linux/slab_def.h | 8 ++++----
include/linux/vmstat.h | 20 ++++++++++----------
mm/backing-dev.c | 4 ++--
mm/memory-failure.c | 30 +++++++++++++++---------------
mm/page_alloc.c | 6 +++---
mm/slab.c | 20 ++++++++++++--------
mm/sparse.c | 2 +-
mm/swapfile.c | 12 ++++++------
mm/vmstat.c | 12 ++++++------
11 files changed, 61 insertions(+), 57 deletions(-)
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 2b05068..54936fc 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -2207,7 +2207,7 @@ extern int get_hwpoison_page(struct page *page);
extern int sysctl_memory_failure_early_kill;
extern int sysctl_memory_failure_recovery;
extern void shake_page(struct page *p, int access);
-extern atomic_long_t num_poisoned_pages;
+extern atomic_long_unchecked_t num_poisoned_pages;
extern int soft_offline_page(struct page *page, int flags);
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 754c259..7b65ac6 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -526,7 +526,7 @@ struct zone {
ZONE_PADDING(_pad3_)
/* Zone statistics */
- atomic_long_t vm_stat[NR_VM_ZONE_STAT_ITEMS];
+ atomic_long_unchecked_t vm_stat[NR_VM_ZONE_STAT_ITEMS];
} ____cacheline_internodealigned_in_smp;
enum zone_flags {
diff --git a/include/linux/slab_def.h b/include/linux/slab_def.h
index 33d0490..e3c7936 100644
--- a/include/linux/slab_def.h
+++ b/include/linux/slab_def.h
@@ -56,10 +56,10 @@ struct kmem_cache {
unsigned long node_allocs;
unsigned long node_frees;
unsigned long node_overflow;
- atomic_t allochit;
- atomic_t allocmiss;
- atomic_t freehit;
- atomic_t freemiss;
+ atomic_unchecked_t allochit;
+ atomic_unchecked_t allocmiss;
+ atomic_unchecked_t freehit;
+ atomic_unchecked_t freemiss;
/*
* If debugging is enabled, then the allocator can add additional
diff --git a/include/linux/vmstat.h b/include/linux/vmstat.h
index 82e7db7..39e551e 100644
--- a/include/linux/vmstat.h
+++ b/include/linux/vmstat.h
@@ -108,18 +108,18 @@ static inline void vm_events_fold_cpu(int cpu)
/*
* Zone based page accounting with per cpu differentials.
*/
-extern atomic_long_t vm_stat[NR_VM_ZONE_STAT_ITEMS];
+extern atomic_long_unchecked_t vm_stat[NR_VM_ZONE_STAT_ITEMS];
static inline void zone_page_state_add(long x, struct zone *zone,
enum zone_stat_item item)
{
- atomic_long_add(x, &zone->vm_stat[item]);
- atomic_long_add(x, &vm_stat[item]);
+ atomic_long_add_unchecked(x, &zone->vm_stat[item]);
+ atomic_long_add_unchecked(x, &vm_stat[item]);
}
static inline unsigned long global_page_state(enum zone_stat_item item)
{
- long x = atomic_long_read(&vm_stat[item]);
+ long x = atomic_long_read_unchecked(&vm_stat[item]);
#ifdef CONFIG_SMP
if (x < 0)
x = 0;
@@ -130,7 +130,7 @@ static inline unsigned long global_page_state(enum zone_stat_item item)
static inline unsigned long zone_page_state(struct zone *zone,
enum zone_stat_item item)
{
- long x = atomic_long_read(&zone->vm_stat[item]);
+ long x = atomic_long_read_unchecked(&zone->vm_stat[item]);
#ifdef CONFIG_SMP
if (x < 0)
x = 0;
@@ -147,7 +147,7 @@ static inline unsigned long zone_page_state(struct zone *zone,
static inline unsigned long zone_page_state_snapshot(struct zone *zone,
enum zone_stat_item item)
{
- long x = atomic_long_read(&zone->vm_stat[item]);
+ long x = atomic_long_read_unchecked(&zone->vm_stat[item]);
#ifdef CONFIG_SMP
int cpu;
@@ -234,14 +234,14 @@ static inline void __mod_zone_page_state(struct zone *zone,
static inline void __inc_zone_state(struct zone *zone, enum zone_stat_item item)
{
- atomic_long_inc(&zone->vm_stat[item]);
- atomic_long_inc(&vm_stat[item]);
+ atomic_long_inc_unchecked(&zone->vm_stat[item]);
+ atomic_long_inc_unchecked(&vm_stat[item]);
}
static inline void __dec_zone_state(struct zone *zone, enum zone_stat_item item)
{
- atomic_long_dec(&zone->vm_stat[item]);
- atomic_long_dec(&vm_stat[item]);
+ atomic_long_dec_unchecked(&zone->vm_stat[item]);
+ atomic_long_dec_unchecked(&vm_stat[item]);
}
static inline void __inc_zone_page_state(struct page *page,
diff --git a/mm/backing-dev.c b/mm/backing-dev.c
index dc07d88..3929c29 100644
--- a/mm/backing-dev.c
+++ b/mm/backing-dev.c
@@ -12,7 +12,7 @@
#include <linux/device.h>
#include <trace/events/writeback.h>
-static atomic_long_t bdi_seq = ATOMIC_LONG_INIT(0);
+static atomic_long_unchecked_t bdi_seq = ATOMIC_LONG_INIT(0);
struct backing_dev_info noop_backing_dev_info = {
.name = "noop",
@@ -865,7 +865,7 @@ int bdi_setup_and_register(struct backing_dev_info *bdi, char *name)
return err;
err = bdi_register(bdi, NULL, "%.28s-%ld", name,
- atomic_long_inc_return(&bdi_seq));
+ atomic_long_inc_return_unchecked(&bdi_seq));
if (err) {
bdi_destroy(bdi);
return err;
diff --git a/mm/memory-failure.c b/mm/memory-failure.c
index 1f4446a..e837835 100644
--- a/mm/memory-failure.c
+++ b/mm/memory-failure.c
@@ -63,7 +63,7 @@ int sysctl_memory_failure_early_kill __read_mostly = 0;
int sysctl_memory_failure_recovery __read_mostly = 1;
-atomic_long_t num_poisoned_pages __read_mostly = ATOMIC_LONG_INIT(0);
+atomic_long_unchecked_t num_poisoned_pages __read_mostly = ATOMIC_LONG_INIT(0);
#if defined(CONFIG_HWPOISON_INJECT) || defined(CONFIG_HWPOISON_INJECT_MODULE)
@@ -1100,7 +1100,7 @@ int memory_failure(unsigned long pfn, int trapno, int flags)
nr_pages = 1 << compound_order(hpage);
else /* normal page or thp */
nr_pages = 1;
- atomic_long_add(nr_pages, &num_poisoned_pages);
+ atomic_long_add_unchecked(nr_pages, &num_poisoned_pages);
/*
* We need/can do nothing about count=0 pages.
@@ -1128,7 +1128,7 @@ int memory_failure(unsigned long pfn, int trapno, int flags)
if (PageHWPoison(hpage)) {
if ((hwpoison_filter(p) && TestClearPageHWPoison(p))
|| (p != hpage && TestSetPageHWPoison(hpage))) {
- atomic_long_sub(nr_pages, &num_poisoned_pages);
+ atomic_long_sub_unchecked(nr_pages, &num_poisoned_pages);
unlock_page(hpage);
return 0;
}
@@ -1152,7 +1152,7 @@ int memory_failure(unsigned long pfn, int trapno, int flags)
else
pr_err("MCE: %#lx: thp split failed\n", pfn);
if (TestClearPageHWPoison(p))
- atomic_long_sub(nr_pages, &num_poisoned_pages);
+ atomic_long_sub_unchecked(nr_pages, &num_poisoned_pages);
put_page(p);
if (p != hpage)
put_page(hpage);
@@ -1214,14 +1214,14 @@ int memory_failure(unsigned long pfn, int trapno, int flags)
*/
if (!PageHWPoison(p)) {
printk(KERN_ERR "MCE %#lx: just unpoisoned\n", pfn);
- atomic_long_sub(nr_pages, &num_poisoned_pages);
+ atomic_long_sub_unchecked(nr_pages, &num_poisoned_pages);
unlock_page(hpage);
put_page(hpage);
return 0;
}
if (hwpoison_filter(p)) {
if (TestClearPageHWPoison(p))
- atomic_long_sub(nr_pages, &num_poisoned_pages);
+ atomic_long_sub_unchecked(nr_pages, &num_poisoned_pages);
unlock_page(hpage);
put_page(hpage);
return 0;
@@ -1450,7 +1450,7 @@ int unpoison_memory(unsigned long pfn)
return 0;
}
if (TestClearPageHWPoison(p))
- atomic_long_dec(&num_poisoned_pages);
+ atomic_long_dec_unchecked(&num_poisoned_pages);
pr_info("MCE: Software-unpoisoned free page %#lx\n", pfn);
return 0;
}
@@ -1464,7 +1464,7 @@ int unpoison_memory(unsigned long pfn)
*/
if (TestClearPageHWPoison(page)) {
pr_info("MCE: Software-unpoisoned page %#lx\n", pfn);
- atomic_long_sub(nr_pages, &num_poisoned_pages);
+ atomic_long_sub_unchecked(nr_pages, &num_poisoned_pages);
freeit = 1;
if (PageHuge(page))
clear_page_hwpoison_huge_page(page);
@@ -1600,11 +1600,11 @@ static int soft_offline_huge_page(struct page *page, int flags)
if (PageHuge(page)) {
set_page_hwpoison_huge_page(hpage);
dequeue_hwpoisoned_huge_page(hpage);
- atomic_long_add(1 << compound_order(hpage),
+ atomic_long_add_unchecked(1 << compound_order(hpage),
&num_poisoned_pages);
} else {
SetPageHWPoison(page);
- atomic_long_inc(&num_poisoned_pages);
+ atomic_long_inc_unchecked(&num_poisoned_pages);
}
}
return ret;
@@ -1643,7 +1643,7 @@ static int __soft_offline_page(struct page *page, int flags)
put_page(page);
pr_info("soft_offline: %#lx: invalidated\n", pfn);
SetPageHWPoison(page);
- atomic_long_inc(&num_poisoned_pages);
+ atomic_long_inc_unchecked(&num_poisoned_pages);
return 0;
}
@@ -1664,7 +1664,7 @@ static int __soft_offline_page(struct page *page, int flags)
page_is_file_cache(page));
list_add(&page->lru, &pagelist);
if (!TestSetPageHWPoison(page))
- atomic_long_inc(&num_poisoned_pages);
+ atomic_long_inc_unchecked(&num_poisoned_pages);
ret = migrate_pages(&pagelist, new_page, NULL, MPOL_MF_MOVE_ALL,
MIGRATE_SYNC, MR_MEMORY_FAILURE);
if (ret) {
@@ -1680,7 +1680,7 @@ static int __soft_offline_page(struct page *page, int flags)
if (ret > 0)
ret = -EIO;
if (TestClearPageHWPoison(page))
- atomic_long_dec(&num_poisoned_pages);
+ atomic_long_dec_unchecked(&num_poisoned_pages);
}
} else {
pr_info("soft offline: %#lx: isolation failed: %d, page count %d, type %lx\n",
@@ -1742,11 +1742,11 @@ int soft_offline_page(struct page *page, int flags)
if (PageHuge(page)) {
set_page_hwpoison_huge_page(hpage);
if (!dequeue_hwpoisoned_huge_page(hpage))
- atomic_long_add(1 << compound_order(hpage),
+ atomic_long_add_unchecked(1 << compound_order(hpage),
&num_poisoned_pages);
} else {
if (!TestSetPageHWPoison(page))
- atomic_long_inc(&num_poisoned_pages);
+ atomic_long_inc_unchecked(&num_poisoned_pages);
}
}
return ret;
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 5b5240b..da8c7af 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -2116,7 +2116,7 @@ struct page *buffered_rmqueue(struct zone *preferred_zone,
}
__mod_zone_page_state(zone, NR_ALLOC_BATCH, -(1 << order));
- if (atomic_long_read(&zone->vm_stat[NR_ALLOC_BATCH]) <= 0 &&
+ if (atomic_long_read_unchecked(&zone->vm_stat[NR_ALLOC_BATCH]) <= 0 &&
!test_bit(ZONE_FAIR_DEPLETED, &zone->flags))
set_bit(ZONE_FAIR_DEPLETED, &zone->flags);
@@ -2435,7 +2435,7 @@ static void reset_alloc_batches(struct zone *preferred_zone)
do {
mod_zone_page_state(zone, NR_ALLOC_BATCH,
high_wmark_pages(zone) - low_wmark_pages(zone) -
- atomic_long_read(&zone->vm_stat[NR_ALLOC_BATCH]));
+ atomic_long_read_unchecked(&zone->vm_stat[NR_ALLOC_BATCH]));
clear_bit(ZONE_FAIR_DEPLETED, &zone->flags);
} while (zone++ != preferred_zone);
}
@@ -6184,7 +6184,7 @@ static void __setup_per_zone_wmarks(void)
__mod_zone_page_state(zone, NR_ALLOC_BATCH,
high_wmark_pages(zone) - low_wmark_pages(zone) -
- atomic_long_read(&zone->vm_stat[NR_ALLOC_BATCH]));
+ atomic_long_read_unchecked(&zone->vm_stat[NR_ALLOC_BATCH]));
setup_zone_migrate_reserve(zone);
spin_unlock_irqrestore(&zone->lock, flags);
diff --git a/mm/slab.c b/mm/slab.c
index ae36028..154e594 100644
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -314,10 +314,12 @@ static void kmem_cache_node_init(struct kmem_cache_node *parent)
if ((x)->max_freeable < i) \
(x)->max_freeable = i; \
} while (0)
-#define STATS_INC_ALLOCHIT(x) atomic_inc(&(x)->allochit)
-#define STATS_INC_ALLOCMISS(x) atomic_inc(&(x)->allocmiss)
-#define STATS_INC_FREEHIT(x) atomic_inc(&(x)->freehit)
-#define STATS_INC_FREEMISS(x) atomic_inc(&(x)->freemiss)
+#define STATS_INC_ALLOCHIT(x) atomic_inc_unchecked(&(x)->allochit)
+#define STATS_INC_ALLOCMISS(x) atomic_inc_unchecked(&(x)->allocmiss)
+#define STATS_INC_FREEHIT(x) atomic_inc_unchecked(&(x)->freehit)
+#define STATS_INC_FREEMISS(x) atomic_inc_unchecked(&(x)->freemiss)
+#define STATS_INC_SANITIZED(x) atomic_inc_unchecked(&(x)->sanitized)
+#define STATS_INC_NOT_SANITIZED(x) atomic_inc_unchecked(&(x)->not_sanitized)
#else
#define STATS_INC_ACTIVE(x) do { } while (0)
#define STATS_DEC_ACTIVE(x) do { } while (0)
@@ -334,6 +336,8 @@ static void kmem_cache_node_init(struct kmem_cache_node *parent)
#define STATS_INC_ALLOCMISS(x) do { } while (0)
#define STATS_INC_FREEHIT(x) do { } while (0)
#define STATS_INC_FREEMISS(x) do { } while (0)
+#define STATS_INC_SANITIZED(x) do { } while (0)
+#define STATS_INC_NOT_SANITIZED(x) do { } while (0)
#endif
#if DEBUG
@@ -4003,10 +4007,10 @@ void slabinfo_show_stats(struct seq_file *m, struct kmem_cache *cachep)
}
/* cpu stats */
{
- unsigned long allochit = atomic_read(&cachep->allochit);
- unsigned long allocmiss = atomic_read(&cachep->allocmiss);
- unsigned long freehit = atomic_read(&cachep->freehit);
- unsigned long freemiss = atomic_read(&cachep->freemiss);
+ unsigned long allochit = atomic_read_unchecked(&cachep->allochit);
+ unsigned long allocmiss = atomic_read_unchecked(&cachep->allocmiss);
+ unsigned long freehit = atomic_read_unchecked(&cachep->freehit);
+ unsigned long freemiss = atomic_read_unchecked(&cachep->freemiss);
seq_printf(m, " : cpustat %6lu %6lu %6lu %6lu",
allochit, allocmiss, freehit, freemiss);
diff --git a/mm/sparse.c b/mm/sparse.c
index d1b48b6..6e8590e 100644
--- a/mm/sparse.c
+++ b/mm/sparse.c
@@ -750,7 +750,7 @@ static void clear_hwpoisoned_pages(struct page *memmap, int nr_pages)
for (i = 0; i < PAGES_PER_SECTION; i++) {
if (PageHWPoison(&memmap[i])) {
- atomic_long_sub(1, &num_poisoned_pages);
+ atomic_long_sub_unchecked(1, &num_poisoned_pages);
ClearPageHWPoison(&memmap[i]);
}
}
diff --git a/mm/swapfile.c b/mm/swapfile.c
index 41e4581..6c452c9 100644
--- a/mm/swapfile.c
+++ b/mm/swapfile.c
@@ -84,7 +84,7 @@ static DEFINE_MUTEX(swapon_mutex);
static DECLARE_WAIT_QUEUE_HEAD(proc_poll_wait);
/* Activity counter to indicate that a swapon or swapoff has occurred */
-static atomic_t proc_poll_event = ATOMIC_INIT(0);
+static atomic_unchecked_t proc_poll_event = ATOMIC_INIT(0);
static inline unsigned char swap_count(unsigned char ent)
{
@@ -1944,7 +1944,7 @@ SYSCALL_DEFINE1(swapoff, const char __user *, specialfile)
spin_unlock(&swap_lock);
err = 0;
- atomic_inc(&proc_poll_event);
+ atomic_inc_unchecked(&proc_poll_event);
wake_up_interruptible(&proc_poll_wait);
out_dput:
@@ -1961,8 +1961,8 @@ static unsigned swaps_poll(struct file *file, poll_table *wait)
poll_wait(file, &proc_poll_wait, wait);
- if (seq->poll_event != atomic_read(&proc_poll_event)) {
- seq->poll_event = atomic_read(&proc_poll_event);
+ if (seq->poll_event != atomic_read_unchecked(&proc_poll_event)) {
+ seq->poll_event = atomic_read_unchecked(&proc_poll_event);
return POLLIN | POLLRDNORM | POLLERR | POLLPRI;
}
@@ -2060,7 +2060,7 @@ static int swaps_open(struct inode *inode, struct file *file)
return ret;
seq = file->private_data;
- seq->poll_event = atomic_read(&proc_poll_event);
+ seq->poll_event = atomic_read_unchecked(&proc_poll_event);
return 0;
}
@@ -2520,7 +2520,7 @@ SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags)
(frontswap_map) ? "FS" : "");
mutex_unlock(&swapon_mutex);
- atomic_inc(&proc_poll_event);
+ atomic_inc_unchecked(&proc_poll_event);
wake_up_interruptible(&proc_poll_wait);
if (S_ISREG(inode->i_mode))
diff --git a/mm/vmstat.c b/mm/vmstat.c
index 4f5cd97..35d42ba 100644
--- a/mm/vmstat.c
+++ b/mm/vmstat.c
@@ -86,7 +86,7 @@ void vm_events_fold_cpu(int cpu)
*
* vm_stat contains the global counters
*/
-atomic_long_t vm_stat[NR_VM_ZONE_STAT_ITEMS] __cacheline_aligned_in_smp;
+atomic_long_unchecked_t vm_stat[NR_VM_ZONE_STAT_ITEMS] __cacheline_aligned_in_smp;
EXPORT_SYMBOL(vm_stat);
#ifdef CONFIG_SMP
@@ -438,7 +438,7 @@ static int fold_diff(int *diff)
for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
if (diff[i]) {
- atomic_long_add(diff[i], &vm_stat[i]);
+ atomic_long_add_unchecked(diff[i], &vm_stat[i]);
changes++;
}
return changes;
@@ -476,7 +476,7 @@ static int refresh_cpu_vm_stats(void)
v = this_cpu_xchg(p->vm_stat_diff[i], 0);
if (v) {
- atomic_long_add(v, &zone->vm_stat[i]);
+ atomic_long_add_unchecked(v, &zone->vm_stat[i]);
global_diff[i] += v;
#ifdef CONFIG_NUMA
/* 3 seconds idle till flush */
@@ -540,7 +540,7 @@ void cpu_vm_stats_fold(int cpu)
v = p->vm_stat_diff[i];
p->vm_stat_diff[i] = 0;
- atomic_long_add(v, &zone->vm_stat[i]);
+ atomic_long_add_unchecked(v, &zone->vm_stat[i]);
global_diff[i] += v;
}
}
@@ -560,8 +560,8 @@ void drain_zonestat(struct zone *zone, struct per_cpu_pageset *pset)
if (pset->vm_stat_diff[i]) {
int v = pset->vm_stat_diff[i];
pset->vm_stat_diff[i] = 0;
- atomic_long_add(v, &zone->vm_stat[i]);
- atomic_long_add(v, &vm_stat[i]);
+ atomic_long_add_unchecked(v, &zone->vm_stat[i]);
+ atomic_long_add_unchecked(v, &vm_stat[i]);
}
}
#endif
--
2.5.0
^ permalink raw reply related [flat|nested] 32+ messages in thread* [kernel-hardening][RFC PATCH v2 05/12] fs: opt out of PAX_REFCOUNT protection
2015-12-17 14:57 [kernel-hardening][RFC PATCH v2 00/12] Add PAX_REFCOUNT overflow protection David Windsor
` (3 preceding siblings ...)
2015-12-17 14:57 ` [kernel-hardening][RFC PATCH v2 04/12] mm: " David Windsor
@ 2015-12-17 14:57 ` David Windsor
2015-12-17 14:57 ` [kernel-hardening][RFC PATCH v2 06/12] net: " David Windsor
` (7 subsequent siblings)
12 siblings, 0 replies; 32+ messages in thread
From: David Windsor @ 2015-12-17 14:57 UTC (permalink / raw)
To: kernel-hardening; +Cc: David Windsor
Signed-off-by: David Windsor <dave@progbits.org>
---
fs/afs/inode.c | 4 +-
fs/btrfs/delayed-inode.c | 6 +-
fs/btrfs/delayed-inode.h | 4 +-
fs/cachefiles/internal.h | 12 +-
fs/cachefiles/namei.c | 2 +-
fs/cachefiles/proc.c | 12 +-
fs/ceph/super.c | 4 +-
fs/cifs/cifs_debug.c | 12 +-
fs/cifs/cifsfs.c | 4 +-
fs/cifs/cifsglob.h | 54 +++----
fs/cifs/misc.c | 4 +-
fs/cifs/smb1ops.c | 80 +++++-----
fs/cifs/smb2ops.c | 84 +++++-----
fs/coda/cache.c | 10 +-
fs/coredump.c | 6 +-
fs/ext4/ext4.h | 20 +--
fs/ext4/mballoc.c | 44 +++---
fs/fscache/cookie.c | 40 ++---
fs/fscache/internal.h | 202 ++++++++++++------------
fs/fscache/object.c | 26 ++--
fs/fscache/operation.c | 38 ++---
fs/fscache/page.c | 110 ++++++-------
fs/fscache/stats.c | 348 +++++++++++++++++++++---------------------
fs/inode.c | 4 +-
fs/kernfs/file.c | 12 +-
fs/lockd/clntproc.c | 4 +-
fs/namespace.c | 4 +-
fs/nfs/inode.c | 6 +-
fs/notify/notification.c | 4 +-
fs/ocfs2/localalloc.c | 2 +-
fs/ocfs2/ocfs2.h | 10 +-
fs/ocfs2/suballoc.c | 12 +-
fs/ocfs2/super.c | 20 +--
fs/proc/meminfo.c | 2 +-
fs/quota/netlink.c | 4 +-
fs/reiserfs/do_balan.c | 2 +-
fs/reiserfs/procfs.c | 2 +-
fs/reiserfs/reiserfs.h | 4 +-
include/linux/fscache-cache.h | 2 +-
39 files changed, 613 insertions(+), 607 deletions(-)
diff --git a/fs/afs/inode.c b/fs/afs/inode.c
index e06f5a2..81d07ac 100644
--- a/fs/afs/inode.c
+++ b/fs/afs/inode.c
@@ -141,7 +141,7 @@ struct inode *afs_iget_autocell(struct inode *dir, const char *dev_name,
struct afs_vnode *vnode;
struct super_block *sb;
struct inode *inode;
- static atomic_t afs_autocell_ino;
+ static atomic_unchecked_t afs_autocell_ino;
_enter("{%x:%u},%*.*s,",
AFS_FS_I(dir)->fid.vid, AFS_FS_I(dir)->fid.vnode,
@@ -154,7 +154,7 @@ struct inode *afs_iget_autocell(struct inode *dir, const char *dev_name,
data.fid.unique = 0;
data.fid.vnode = 0;
- inode = iget5_locked(sb, atomic_inc_return(&afs_autocell_ino),
+ inode = iget5_locked(sb, atomic_inc_return_unchecked(&afs_autocell_ino),
afs_iget5_autocell_test, afs_iget5_set,
&data);
if (!inode) {
diff --git a/fs/btrfs/delayed-inode.c b/fs/btrfs/delayed-inode.c
index a2ae427..53c2e98 100644
--- a/fs/btrfs/delayed-inode.c
+++ b/fs/btrfs/delayed-inode.c
@@ -462,7 +462,7 @@ static int __btrfs_add_delayed_deletion_item(struct btrfs_delayed_node *node,
static void finish_one_item(struct btrfs_delayed_root *delayed_root)
{
- int seq = atomic_inc_return(&delayed_root->items_seq);
+ int seq = atomic_inc_return_unchecked(&delayed_root->items_seq);
if ((atomic_dec_return(&delayed_root->items) <
BTRFS_DELAYED_BACKGROUND || seq % BTRFS_DELAYED_BATCH == 0) &&
waitqueue_active(&delayed_root->wait))
@@ -1412,7 +1412,7 @@ void btrfs_assert_delayed_root_empty(struct btrfs_root *root)
static int could_end_wait(struct btrfs_delayed_root *delayed_root, int seq)
{
- int val = atomic_read(&delayed_root->items_seq);
+ int val = atomic_read_unchecked(&delayed_root->items_seq);
if (val < seq || val >= seq + BTRFS_DELAYED_BATCH)
return 1;
@@ -1437,7 +1437,7 @@ void btrfs_balance_delayed_items(struct btrfs_root *root)
int seq;
int ret;
- seq = atomic_read(&delayed_root->items_seq);
+ seq = atomic_read_unchecked(&delayed_root->items_seq);
ret = btrfs_wq_run_delayed_node(delayed_root, fs_info, 0);
if (ret)
diff --git a/fs/btrfs/delayed-inode.h b/fs/btrfs/delayed-inode.h
index f70119f..ab5894d 100644
--- a/fs/btrfs/delayed-inode.h
+++ b/fs/btrfs/delayed-inode.h
@@ -43,7 +43,7 @@ struct btrfs_delayed_root {
*/
struct list_head prepare_list;
atomic_t items; /* for delayed items */
- atomic_t items_seq; /* for delayed items */
+ atomic_unchecked_t items_seq; /* for delayed items */
int nodes; /* for delayed nodes */
wait_queue_head_t wait;
};
@@ -90,7 +90,7 @@ static inline void btrfs_init_delayed_root(
struct btrfs_delayed_root *delayed_root)
{
atomic_set(&delayed_root->items, 0);
- atomic_set(&delayed_root->items_seq, 0);
+ atomic_set_unchecked(&delayed_root->items_seq, 0);
delayed_root->nodes = 0;
spin_lock_init(&delayed_root->lock);
init_waitqueue_head(&delayed_root->wait);
diff --git a/fs/cachefiles/internal.h b/fs/cachefiles/internal.h
index aecd085..3584e2f 100644
--- a/fs/cachefiles/internal.h
+++ b/fs/cachefiles/internal.h
@@ -65,7 +65,7 @@ struct cachefiles_cache {
wait_queue_head_t daemon_pollwq; /* poll waitqueue for daemon */
struct rb_root active_nodes; /* active nodes (can't be culled) */
rwlock_t active_lock; /* lock for active_nodes */
- atomic_t gravecounter; /* graveyard uniquifier */
+ atomic_unchecked_t gravecounter; /* graveyard uniquifier */
unsigned frun_percent; /* when to stop culling (% files) */
unsigned fcull_percent; /* when to start culling (% files) */
unsigned fstop_percent; /* when to stop allocating (% files) */
@@ -177,19 +177,19 @@ extern int cachefiles_check_in_use(struct cachefiles_cache *cache,
* proc.c
*/
#ifdef CONFIG_CACHEFILES_HISTOGRAM
-extern atomic_t cachefiles_lookup_histogram[HZ];
-extern atomic_t cachefiles_mkdir_histogram[HZ];
-extern atomic_t cachefiles_create_histogram[HZ];
+extern atomic_unchecked_t cachefiles_lookup_histogram[HZ];
+extern atomic_unchecked_t cachefiles_mkdir_histogram[HZ];
+extern atomic_unchecked_t cachefiles_create_histogram[HZ];
extern int __init cachefiles_proc_init(void);
extern void cachefiles_proc_cleanup(void);
static inline
-void cachefiles_hist(atomic_t histogram[], unsigned long start_jif)
+void cachefiles_hist(atomic_unchecked_t histogram[], unsigned long start_jif)
{
unsigned long jif = jiffies - start_jif;
if (jif >= HZ)
jif = HZ - 1;
- atomic_inc(&histogram[jif]);
+ atomic_inc_unchecked(&histogram[jif]);
}
#else
diff --git a/fs/cachefiles/namei.c b/fs/cachefiles/namei.c
index fc1056f..501a546 100644
--- a/fs/cachefiles/namei.c
+++ b/fs/cachefiles/namei.c
@@ -312,7 +312,7 @@ try_again:
/* first step is to make up a grave dentry in the graveyard */
sprintf(nbuffer, "%08x%08x",
(uint32_t) get_seconds(),
- (uint32_t) atomic_inc_return(&cache->gravecounter));
+ (uint32_t) atomic_inc_return_unchecked(&cache->gravecounter));
/* do the multiway lock magic */
trap = lock_rename(cache->graveyard, dir);
diff --git a/fs/cachefiles/proc.c b/fs/cachefiles/proc.c
index eccd339..4c1d995 100644
--- a/fs/cachefiles/proc.c
+++ b/fs/cachefiles/proc.c
@@ -14,9 +14,9 @@
#include <linux/seq_file.h>
#include "internal.h"
-atomic_t cachefiles_lookup_histogram[HZ];
-atomic_t cachefiles_mkdir_histogram[HZ];
-atomic_t cachefiles_create_histogram[HZ];
+atomic_unchecked_t cachefiles_lookup_histogram[HZ];
+atomic_unchecked_t cachefiles_mkdir_histogram[HZ];
+atomic_unchecked_t cachefiles_create_histogram[HZ];
/*
* display the latency histogram
@@ -35,9 +35,9 @@ static int cachefiles_histogram_show(struct seq_file *m, void *v)
return 0;
default:
index = (unsigned long) v - 3;
- x = atomic_read(&cachefiles_lookup_histogram[index]);
- y = atomic_read(&cachefiles_mkdir_histogram[index]);
- z = atomic_read(&cachefiles_create_histogram[index]);
+ x = atomic_read_unchecked(&cachefiles_lookup_histogram[index]);
+ y = atomic_read_unchecked(&cachefiles_mkdir_histogram[index]);
+ z = atomic_read_unchecked(&cachefiles_create_histogram[index]);
if (x == 0 && y == 0 && z == 0)
return 0;
diff --git a/fs/ceph/super.c b/fs/ceph/super.c
index 7b6bfcb..f8d5416 100644
--- a/fs/ceph/super.c
+++ b/fs/ceph/super.c
@@ -906,7 +906,7 @@ static int ceph_compare_super(struct super_block *sb, void *data)
/*
* construct our own bdi so we can control readahead, etc.
*/
-static atomic_long_t bdi_seq = ATOMIC_LONG_INIT(0);
+static atomic_long_unchecked_t bdi_seq = ATOMIC_LONG_INIT(0);
static int ceph_register_bdi(struct super_block *sb,
struct ceph_fs_client *fsc)
@@ -923,7 +923,7 @@ static int ceph_register_bdi(struct super_block *sb,
VM_MAX_READAHEAD * 1024 / PAGE_CACHE_SIZE;
err = bdi_register(&fsc->backing_dev_info, NULL, "ceph-%ld",
- atomic_long_inc_return(&bdi_seq));
+ atomic_long_inc_return_unchecked(&bdi_seq));
if (!err)
sb->s_bdi = &fsc->backing_dev_info;
return err;
diff --git a/fs/cifs/cifs_debug.c b/fs/cifs/cifs_debug.c
index 7febcf2..62a5721 100644
--- a/fs/cifs/cifs_debug.c
+++ b/fs/cifs/cifs_debug.c
@@ -269,8 +269,8 @@ static ssize_t cifs_stats_proc_write(struct file *file,
if (strtobool(&c, &bv) == 0) {
#ifdef CONFIG_CIFS_STATS2
- atomic_set(&totBufAllocCount, 0);
- atomic_set(&totSmBufAllocCount, 0);
+ atomic_set_unchecked(&totBufAllocCount, 0);
+ atomic_set_unchecked(&totSmBufAllocCount, 0);
#endif /* CONFIG_CIFS_STATS2 */
spin_lock(&cifs_tcp_ses_lock);
list_for_each(tmp1, &cifs_tcp_ses_list) {
@@ -283,7 +283,7 @@ static ssize_t cifs_stats_proc_write(struct file *file,
tcon = list_entry(tmp3,
struct cifs_tcon,
tcon_list);
- atomic_set(&tcon->num_smbs_sent, 0);
+ atomic_set_unchecked(&tcon->num_smbs_sent, 0);
if (server->ops->clear_stats)
server->ops->clear_stats(tcon);
}
@@ -315,8 +315,8 @@ static int cifs_stats_proc_show(struct seq_file *m, void *v)
smBufAllocCount.counter, cifs_min_small);
#ifdef CONFIG_CIFS_STATS2
seq_printf(m, "Total Large %d Small %d Allocations\n",
- atomic_read(&totBufAllocCount),
- atomic_read(&totSmBufAllocCount));
+ atomic_read_unchecked(&totBufAllocCount),
+ atomic_read_unchecked(&totSmBufAllocCount));
#endif /* CONFIG_CIFS_STATS2 */
seq_printf(m, "Operations (MIDs): %d\n", atomic_read(&midCount));
@@ -345,7 +345,7 @@ static int cifs_stats_proc_show(struct seq_file *m, void *v)
if (tcon->need_reconnect)
seq_puts(m, "\tDISCONNECTED ");
seq_printf(m, "\nSMBs: %d",
- atomic_read(&tcon->num_smbs_sent));
+ atomic_read_unchecked(&tcon->num_smbs_sent));
if (server->ops->print_stats)
server->ops->print_stats(m, tcon);
}
diff --git a/fs/cifs/cifsfs.c b/fs/cifs/cifsfs.c
index 6a1119e..b29f3b3 100644
--- a/fs/cifs/cifsfs.c
+++ b/fs/cifs/cifsfs.c
@@ -1194,8 +1194,8 @@ init_cifs(void)
atomic_set(&bufAllocCount, 0);
atomic_set(&smBufAllocCount, 0);
#ifdef CONFIG_CIFS_STATS2
- atomic_set(&totBufAllocCount, 0);
- atomic_set(&totSmBufAllocCount, 0);
+ atomic_set_unchecked(&totBufAllocCount, 0);
+ atomic_set_unchecked(&totSmBufAllocCount, 0);
#endif /* CONFIG_CIFS_STATS2 */
atomic_set(&midCount, 0);
diff --git a/fs/cifs/cifsglob.h b/fs/cifs/cifsglob.h
index b406a32..243eb1c 100644
--- a/fs/cifs/cifsglob.h
+++ b/fs/cifs/cifsglob.h
@@ -832,35 +832,35 @@ struct cifs_tcon {
__u16 Flags; /* optional support bits */
enum statusEnum tidStatus;
#ifdef CONFIG_CIFS_STATS
- atomic_t num_smbs_sent;
+ atomic_unchecked_t num_smbs_sent;
union {
struct {
- atomic_t num_writes;
- atomic_t num_reads;
- atomic_t num_flushes;
- atomic_t num_oplock_brks;
- atomic_t num_opens;
- atomic_t num_closes;
- atomic_t num_deletes;
- atomic_t num_mkdirs;
- atomic_t num_posixopens;
- atomic_t num_posixmkdirs;
- atomic_t num_rmdirs;
- atomic_t num_renames;
- atomic_t num_t2renames;
- atomic_t num_ffirst;
- atomic_t num_fnext;
- atomic_t num_fclose;
- atomic_t num_hardlinks;
- atomic_t num_symlinks;
- atomic_t num_locks;
- atomic_t num_acl_get;
- atomic_t num_acl_set;
+ atomic_unchecked_t num_writes;
+ atomic_unchecked_t num_reads;
+ atomic_unchecked_t num_flushes;
+ atomic_unchecked_t num_oplock_brks;
+ atomic_unchecked_t num_opens;
+ atomic_unchecked_t num_closes;
+ atomic_unchecked_t num_deletes;
+ atomic_unchecked_t num_mkdirs;
+ atomic_unchecked_t num_posixopens;
+ atomic_unchecked_t num_posixmkdirs;
+ atomic_unchecked_t num_rmdirs;
+ atomic_unchecked_t num_renames;
+ atomic_unchecked_t num_t2renames;
+ atomic_unchecked_t num_ffirst;
+ atomic_unchecked_t num_fnext;
+ atomic_unchecked_t num_fclose;
+ atomic_unchecked_t num_hardlinks;
+ atomic_unchecked_t num_symlinks;
+ atomic_unchecked_t num_locks;
+ atomic_unchecked_t num_acl_get;
+ atomic_unchecked_t num_acl_set;
} cifs_stats;
#ifdef CONFIG_CIFS_SMB2
struct {
- atomic_t smb2_com_sent[NUMBER_OF_SMB2_COMMANDS];
- atomic_t smb2_com_failed[NUMBER_OF_SMB2_COMMANDS];
+ atomic_unchecked_t smb2_com_sent[NUMBER_OF_SMB2_COMMANDS];
+ atomic_unchecked_t smb2_com_failed[NUMBER_OF_SMB2_COMMANDS];
} smb2_stats;
#endif /* CONFIG_CIFS_SMB2 */
} stats;
@@ -1207,7 +1207,7 @@ convert_delimiter(char *path, char delim)
}
#ifdef CONFIG_CIFS_STATS
-#define cifs_stats_inc atomic_inc
+#define cifs_stats_inc atomic_inc_unchecked
static inline void cifs_stats_bytes_written(struct cifs_tcon *tcon,
unsigned int bytes)
@@ -1574,8 +1574,8 @@ GLOBAL_EXTERN atomic_t tconInfoReconnectCount;
/* Various Debug counters */
GLOBAL_EXTERN atomic_t bufAllocCount; /* current number allocated */
#ifdef CONFIG_CIFS_STATS2
-GLOBAL_EXTERN atomic_t totBufAllocCount; /* total allocated over all time */
-GLOBAL_EXTERN atomic_t totSmBufAllocCount;
+GLOBAL_EXTERN atomic_unchecked_t totBufAllocCount; /* total allocated over all time */
+GLOBAL_EXTERN atomic_unchecked_t totSmBufAllocCount;
#endif
GLOBAL_EXTERN atomic_t smBufAllocCount;
GLOBAL_EXTERN atomic_t midCount;
diff --git a/fs/cifs/misc.c b/fs/cifs/misc.c
index 8442b8b..ea6986f 100644
--- a/fs/cifs/misc.c
+++ b/fs/cifs/misc.c
@@ -170,7 +170,7 @@ cifs_buf_get(void)
memset(ret_buf, 0, buf_size + 3);
atomic_inc(&bufAllocCount);
#ifdef CONFIG_CIFS_STATS2
- atomic_inc(&totBufAllocCount);
+ atomic_inc_unchecked(&totBufAllocCount);
#endif /* CONFIG_CIFS_STATS2 */
}
@@ -205,7 +205,7 @@ cifs_small_buf_get(void)
/* memset(ret_buf, 0, sizeof(struct smb_hdr) + 27);*/
atomic_inc(&smBufAllocCount);
#ifdef CONFIG_CIFS_STATS2
- atomic_inc(&totSmBufAllocCount);
+ atomic_inc_unchecked(&totSmBufAllocCount);
#endif /* CONFIG_CIFS_STATS2 */
}
diff --git a/fs/cifs/smb1ops.c b/fs/cifs/smb1ops.c
index fc537c2..47d654c 100644
--- a/fs/cifs/smb1ops.c
+++ b/fs/cifs/smb1ops.c
@@ -622,27 +622,27 @@ static void
cifs_clear_stats(struct cifs_tcon *tcon)
{
#ifdef CONFIG_CIFS_STATS
- atomic_set(&tcon->stats.cifs_stats.num_writes, 0);
- atomic_set(&tcon->stats.cifs_stats.num_reads, 0);
- atomic_set(&tcon->stats.cifs_stats.num_flushes, 0);
- atomic_set(&tcon->stats.cifs_stats.num_oplock_brks, 0);
- atomic_set(&tcon->stats.cifs_stats.num_opens, 0);
- atomic_set(&tcon->stats.cifs_stats.num_posixopens, 0);
- atomic_set(&tcon->stats.cifs_stats.num_posixmkdirs, 0);
- atomic_set(&tcon->stats.cifs_stats.num_closes, 0);
- atomic_set(&tcon->stats.cifs_stats.num_deletes, 0);
- atomic_set(&tcon->stats.cifs_stats.num_mkdirs, 0);
- atomic_set(&tcon->stats.cifs_stats.num_rmdirs, 0);
- atomic_set(&tcon->stats.cifs_stats.num_renames, 0);
- atomic_set(&tcon->stats.cifs_stats.num_t2renames, 0);
- atomic_set(&tcon->stats.cifs_stats.num_ffirst, 0);
- atomic_set(&tcon->stats.cifs_stats.num_fnext, 0);
- atomic_set(&tcon->stats.cifs_stats.num_fclose, 0);
- atomic_set(&tcon->stats.cifs_stats.num_hardlinks, 0);
- atomic_set(&tcon->stats.cifs_stats.num_symlinks, 0);
- atomic_set(&tcon->stats.cifs_stats.num_locks, 0);
- atomic_set(&tcon->stats.cifs_stats.num_acl_get, 0);
- atomic_set(&tcon->stats.cifs_stats.num_acl_set, 0);
+ atomic_set_unchecked(&tcon->stats.cifs_stats.num_writes, 0);
+ atomic_set_unchecked(&tcon->stats.cifs_stats.num_reads, 0);
+ atomic_set_unchecked(&tcon->stats.cifs_stats.num_flushes, 0);
+ atomic_set_unchecked(&tcon->stats.cifs_stats.num_oplock_brks, 0);
+ atomic_set_unchecked(&tcon->stats.cifs_stats.num_opens, 0);
+ atomic_set_unchecked(&tcon->stats.cifs_stats.num_posixopens, 0);
+ atomic_set_unchecked(&tcon->stats.cifs_stats.num_posixmkdirs, 0);
+ atomic_set_unchecked(&tcon->stats.cifs_stats.num_closes, 0);
+ atomic_set_unchecked(&tcon->stats.cifs_stats.num_deletes, 0);
+ atomic_set_unchecked(&tcon->stats.cifs_stats.num_mkdirs, 0);
+ atomic_set_unchecked(&tcon->stats.cifs_stats.num_rmdirs, 0);
+ atomic_set_unchecked(&tcon->stats.cifs_stats.num_renames, 0);
+ atomic_set_unchecked(&tcon->stats.cifs_stats.num_t2renames, 0);
+ atomic_set_unchecked(&tcon->stats.cifs_stats.num_ffirst, 0);
+ atomic_set_unchecked(&tcon->stats.cifs_stats.num_fnext, 0);
+ atomic_set_unchecked(&tcon->stats.cifs_stats.num_fclose, 0);
+ atomic_set_unchecked(&tcon->stats.cifs_stats.num_hardlinks, 0);
+ atomic_set_unchecked(&tcon->stats.cifs_stats.num_symlinks, 0);
+ atomic_set_unchecked(&tcon->stats.cifs_stats.num_locks, 0);
+ atomic_set_unchecked(&tcon->stats.cifs_stats.num_acl_get, 0);
+ atomic_set_unchecked(&tcon->stats.cifs_stats.num_acl_set, 0);
#endif
}
@@ -651,36 +651,36 @@ cifs_print_stats(struct seq_file *m, struct cifs_tcon *tcon)
{
#ifdef CONFIG_CIFS_STATS
seq_printf(m, " Oplocks breaks: %d",
- atomic_read(&tcon->stats.cifs_stats.num_oplock_brks));
+ atomic_read_unchecked(&tcon->stats.cifs_stats.num_oplock_brks));
seq_printf(m, "\nReads: %d Bytes: %llu",
- atomic_read(&tcon->stats.cifs_stats.num_reads),
+ atomic_read_unchecked(&tcon->stats.cifs_stats.num_reads),
(long long)(tcon->bytes_read));
seq_printf(m, "\nWrites: %d Bytes: %llu",
- atomic_read(&tcon->stats.cifs_stats.num_writes),
+ atomic_read_unchecked(&tcon->stats.cifs_stats.num_writes),
(long long)(tcon->bytes_written));
seq_printf(m, "\nFlushes: %d",
- atomic_read(&tcon->stats.cifs_stats.num_flushes));
+ atomic_read_unchecked(&tcon->stats.cifs_stats.num_flushes));
seq_printf(m, "\nLocks: %d HardLinks: %d Symlinks: %d",
- atomic_read(&tcon->stats.cifs_stats.num_locks),
- atomic_read(&tcon->stats.cifs_stats.num_hardlinks),
- atomic_read(&tcon->stats.cifs_stats.num_symlinks));
+ atomic_read_unchecked(&tcon->stats.cifs_stats.num_locks),
+ atomic_read_unchecked(&tcon->stats.cifs_stats.num_hardlinks),
+ atomic_read_unchecked(&tcon->stats.cifs_stats.num_symlinks));
seq_printf(m, "\nOpens: %d Closes: %d Deletes: %d",
- atomic_read(&tcon->stats.cifs_stats.num_opens),
- atomic_read(&tcon->stats.cifs_stats.num_closes),
- atomic_read(&tcon->stats.cifs_stats.num_deletes));
+ atomic_read_unchecked(&tcon->stats.cifs_stats.num_opens),
+ atomic_read_unchecked(&tcon->stats.cifs_stats.num_closes),
+ atomic_read_unchecked(&tcon->stats.cifs_stats.num_deletes));
seq_printf(m, "\nPosix Opens: %d Posix Mkdirs: %d",
- atomic_read(&tcon->stats.cifs_stats.num_posixopens),
- atomic_read(&tcon->stats.cifs_stats.num_posixmkdirs));
+ atomic_read_unchecked(&tcon->stats.cifs_stats.num_posixopens),
+ atomic_read_unchecked(&tcon->stats.cifs_stats.num_posixmkdirs));
seq_printf(m, "\nMkdirs: %d Rmdirs: %d",
- atomic_read(&tcon->stats.cifs_stats.num_mkdirs),
- atomic_read(&tcon->stats.cifs_stats.num_rmdirs));
+ atomic_read_unchecked(&tcon->stats.cifs_stats.num_mkdirs),
+ atomic_read_unchecked(&tcon->stats.cifs_stats.num_rmdirs));
seq_printf(m, "\nRenames: %d T2 Renames %d",
- atomic_read(&tcon->stats.cifs_stats.num_renames),
- atomic_read(&tcon->stats.cifs_stats.num_t2renames));
+ atomic_read_unchecked(&tcon->stats.cifs_stats.num_renames),
+ atomic_read_unchecked(&tcon->stats.cifs_stats.num_t2renames));
seq_printf(m, "\nFindFirst: %d FNext %d FClose %d",
- atomic_read(&tcon->stats.cifs_stats.num_ffirst),
- atomic_read(&tcon->stats.cifs_stats.num_fnext),
- atomic_read(&tcon->stats.cifs_stats.num_fclose));
+ atomic_read_unchecked(&tcon->stats.cifs_stats.num_ffirst),
+ atomic_read_unchecked(&tcon->stats.cifs_stats.num_fnext),
+ atomic_read_unchecked(&tcon->stats.cifs_stats.num_fclose));
#endif
}
diff --git a/fs/cifs/smb2ops.c b/fs/cifs/smb2ops.c
index 18da19f..38a3a79 100644
--- a/fs/cifs/smb2ops.c
+++ b/fs/cifs/smb2ops.c
@@ -422,8 +422,8 @@ smb2_clear_stats(struct cifs_tcon *tcon)
#ifdef CONFIG_CIFS_STATS
int i;
for (i = 0; i < NUMBER_OF_SMB2_COMMANDS; i++) {
- atomic_set(&tcon->stats.smb2_stats.smb2_com_sent[i], 0);
- atomic_set(&tcon->stats.smb2_stats.smb2_com_failed[i], 0);
+ atomic_set_unchecked(&tcon->stats.smb2_stats.smb2_com_sent[i], 0);
+ atomic_set_unchecked(&tcon->stats.smb2_stats.smb2_com_failed[i], 0);
}
#endif
}
@@ -463,65 +463,65 @@ static void
smb2_print_stats(struct seq_file *m, struct cifs_tcon *tcon)
{
#ifdef CONFIG_CIFS_STATS
- atomic_t *sent = tcon->stats.smb2_stats.smb2_com_sent;
- atomic_t *failed = tcon->stats.smb2_stats.smb2_com_failed;
+ atomic_unchecked_t *sent = tcon->stats.smb2_stats.smb2_com_sent;
+ atomic_unchecked_t *failed = tcon->stats.smb2_stats.smb2_com_failed;
seq_printf(m, "\nNegotiates: %d sent %d failed",
- atomic_read(&sent[SMB2_NEGOTIATE_HE]),
- atomic_read(&failed[SMB2_NEGOTIATE_HE]));
+ atomic_read_unchecked(&sent[SMB2_NEGOTIATE_HE]),
+ atomic_read_unchecked(&failed[SMB2_NEGOTIATE_HE]));
seq_printf(m, "\nSessionSetups: %d sent %d failed",
- atomic_read(&sent[SMB2_SESSION_SETUP_HE]),
- atomic_read(&failed[SMB2_SESSION_SETUP_HE]));
+ atomic_read_unchecked(&sent[SMB2_SESSION_SETUP_HE]),
+ atomic_read_unchecked(&failed[SMB2_SESSION_SETUP_HE]));
seq_printf(m, "\nLogoffs: %d sent %d failed",
- atomic_read(&sent[SMB2_LOGOFF_HE]),
- atomic_read(&failed[SMB2_LOGOFF_HE]));
+ atomic_read_unchecked(&sent[SMB2_LOGOFF_HE]),
+ atomic_read_unchecked(&failed[SMB2_LOGOFF_HE]));
seq_printf(m, "\nTreeConnects: %d sent %d failed",
- atomic_read(&sent[SMB2_TREE_CONNECT_HE]),
- atomic_read(&failed[SMB2_TREE_CONNECT_HE]));
+ atomic_read_unchecked(&sent[SMB2_TREE_CONNECT_HE]),
+ atomic_read_unchecked(&failed[SMB2_TREE_CONNECT_HE]));
seq_printf(m, "\nTreeDisconnects: %d sent %d failed",
- atomic_read(&sent[SMB2_TREE_DISCONNECT_HE]),
- atomic_read(&failed[SMB2_TREE_DISCONNECT_HE]));
+ atomic_read_unchecked(&sent[SMB2_TREE_DISCONNECT_HE]),
+ atomic_read_unchecked(&failed[SMB2_TREE_DISCONNECT_HE]));
seq_printf(m, "\nCreates: %d sent %d failed",
- atomic_read(&sent[SMB2_CREATE_HE]),
- atomic_read(&failed[SMB2_CREATE_HE]));
+ atomic_read_unchecked(&sent[SMB2_CREATE_HE]),
+ atomic_read_unchecked(&failed[SMB2_CREATE_HE]));
seq_printf(m, "\nCloses: %d sent %d failed",
- atomic_read(&sent[SMB2_CLOSE_HE]),
- atomic_read(&failed[SMB2_CLOSE_HE]));
+ atomic_read_unchecked(&sent[SMB2_CLOSE_HE]),
+ atomic_read_unchecked(&failed[SMB2_CLOSE_HE]));
seq_printf(m, "\nFlushes: %d sent %d failed",
- atomic_read(&sent[SMB2_FLUSH_HE]),
- atomic_read(&failed[SMB2_FLUSH_HE]));
+ atomic_read_unchecked(&sent[SMB2_FLUSH_HE]),
+ atomic_read_unchecked(&failed[SMB2_FLUSH_HE]));
seq_printf(m, "\nReads: %d sent %d failed",
- atomic_read(&sent[SMB2_READ_HE]),
- atomic_read(&failed[SMB2_READ_HE]));
+ atomic_read_unchecked(&sent[SMB2_READ_HE]),
+ atomic_read_unchecked(&failed[SMB2_READ_HE]));
seq_printf(m, "\nWrites: %d sent %d failed",
- atomic_read(&sent[SMB2_WRITE_HE]),
- atomic_read(&failed[SMB2_WRITE_HE]));
+ atomic_read_unchecked(&sent[SMB2_WRITE_HE]),
+ atomic_read_unchecked(&failed[SMB2_WRITE_HE]));
seq_printf(m, "\nLocks: %d sent %d failed",
- atomic_read(&sent[SMB2_LOCK_HE]),
- atomic_read(&failed[SMB2_LOCK_HE]));
+ atomic_read_unchecked(&sent[SMB2_LOCK_HE]),
+ atomic_read_unchecked(&failed[SMB2_LOCK_HE]));
seq_printf(m, "\nIOCTLs: %d sent %d failed",
- atomic_read(&sent[SMB2_IOCTL_HE]),
- atomic_read(&failed[SMB2_IOCTL_HE]));
+ atomic_read_unchecked(&sent[SMB2_IOCTL_HE]),
+ atomic_read_unchecked(&failed[SMB2_IOCTL_HE]));
seq_printf(m, "\nCancels: %d sent %d failed",
- atomic_read(&sent[SMB2_CANCEL_HE]),
- atomic_read(&failed[SMB2_CANCEL_HE]));
+ atomic_read_unchecked(&sent[SMB2_CANCEL_HE]),
+ atomic_read_unchecked(&failed[SMB2_CANCEL_HE]));
seq_printf(m, "\nEchos: %d sent %d failed",
- atomic_read(&sent[SMB2_ECHO_HE]),
- atomic_read(&failed[SMB2_ECHO_HE]));
+ atomic_read_unchecked(&sent[SMB2_ECHO_HE]),
+ atomic_read_unchecked(&failed[SMB2_ECHO_HE]));
seq_printf(m, "\nQueryDirectories: %d sent %d failed",
- atomic_read(&sent[SMB2_QUERY_DIRECTORY_HE]),
- atomic_read(&failed[SMB2_QUERY_DIRECTORY_HE]));
+ atomic_read_unchecked(&sent[SMB2_QUERY_DIRECTORY_HE]),
+ atomic_read_unchecked(&failed[SMB2_QUERY_DIRECTORY_HE]));
seq_printf(m, "\nChangeNotifies: %d sent %d failed",
- atomic_read(&sent[SMB2_CHANGE_NOTIFY_HE]),
- atomic_read(&failed[SMB2_CHANGE_NOTIFY_HE]));
+ atomic_read_unchecked(&sent[SMB2_CHANGE_NOTIFY_HE]),
+ atomic_read_unchecked(&failed[SMB2_CHANGE_NOTIFY_HE]));
seq_printf(m, "\nQueryInfos: %d sent %d failed",
- atomic_read(&sent[SMB2_QUERY_INFO_HE]),
- atomic_read(&failed[SMB2_QUERY_INFO_HE]));
+ atomic_read_unchecked(&sent[SMB2_QUERY_INFO_HE]),
+ atomic_read_unchecked(&failed[SMB2_QUERY_INFO_HE]));
seq_printf(m, "\nSetInfos: %d sent %d failed",
- atomic_read(&sent[SMB2_SET_INFO_HE]),
- atomic_read(&failed[SMB2_SET_INFO_HE]));
+ atomic_read_unchecked(&sent[SMB2_SET_INFO_HE]),
+ atomic_read_unchecked(&failed[SMB2_SET_INFO_HE]));
seq_printf(m, "\nOplockBreaks: %d sent %d failed",
- atomic_read(&sent[SMB2_OPLOCK_BREAK_HE]),
- atomic_read(&failed[SMB2_OPLOCK_BREAK_HE]));
+ atomic_read_unchecked(&sent[SMB2_OPLOCK_BREAK_HE]),
+ atomic_read_unchecked(&failed[SMB2_OPLOCK_BREAK_HE]));
#endif
}
diff --git a/fs/coda/cache.c b/fs/coda/cache.c
index 5bb630a..043dc70 100644
--- a/fs/coda/cache.c
+++ b/fs/coda/cache.c
@@ -24,7 +24,7 @@
#include "coda_linux.h"
#include "coda_cache.h"
-static atomic_t permission_epoch = ATOMIC_INIT(0);
+static atomic_unchecked_t permission_epoch = ATOMIC_INIT(0);
/* replace or extend an acl cache hit */
void coda_cache_enter(struct inode *inode, int mask)
@@ -32,7 +32,7 @@ void coda_cache_enter(struct inode *inode, int mask)
struct coda_inode_info *cii = ITOC(inode);
spin_lock(&cii->c_lock);
- cii->c_cached_epoch = atomic_read(&permission_epoch);
+ cii->c_cached_epoch = atomic_read_unchecked(&permission_epoch);
if (!uid_eq(cii->c_uid, current_fsuid())) {
cii->c_uid = current_fsuid();
cii->c_cached_perm = mask;
@@ -46,14 +46,14 @@ void coda_cache_clear_inode(struct inode *inode)
{
struct coda_inode_info *cii = ITOC(inode);
spin_lock(&cii->c_lock);
- cii->c_cached_epoch = atomic_read(&permission_epoch) - 1;
+ cii->c_cached_epoch = atomic_read_unchecked(&permission_epoch) - 1;
spin_unlock(&cii->c_lock);
}
/* remove all acl caches */
void coda_cache_clear_all(struct super_block *sb)
{
- atomic_inc(&permission_epoch);
+ atomic_inc_unchecked(&permission_epoch);
}
@@ -66,7 +66,7 @@ int coda_cache_check(struct inode *inode, int mask)
spin_lock(&cii->c_lock);
hit = (mask & cii->c_cached_perm) == mask &&
uid_eq(cii->c_uid, current_fsuid()) &&
- cii->c_cached_epoch == atomic_read(&permission_epoch);
+ cii->c_cached_epoch == atomic_read_unchecked(&permission_epoch);
spin_unlock(&cii->c_lock);
return hit;
diff --git a/fs/coredump.c b/fs/coredump.c
index a8f7564..488c681 100644
--- a/fs/coredump.c
+++ b/fs/coredump.c
@@ -518,7 +518,7 @@ void do_coredump(const siginfo_t *siginfo)
/* require nonrelative corefile path and be extra careful */
bool need_suid_safe = false;
bool core_dumped = false;
- static atomic_t core_dump_count = ATOMIC_INIT(0);
+ static atomic_unchecked_t core_dump_count = ATOMIC_INIT(0);
struct coredump_params cprm = {
.siginfo = siginfo,
.regs = signal_pt_regs(),
@@ -597,7 +597,7 @@ void do_coredump(const siginfo_t *siginfo)
}
cprm.limit = RLIM_INFINITY;
- dump_count = atomic_inc_return(&core_dump_count);
+ dump_count = atomic_inc_return_unchecked(&core_dump_count);
if (core_pipe_limit && (core_pipe_limit < dump_count)) {
printk(KERN_WARNING "Pid %d(%s) over core_pipe_limit\n",
task_tgid_vnr(current), current->comm);
@@ -718,7 +718,7 @@ close_fail:
filp_close(cprm.file, NULL);
fail_dropcount:
if (ispipe)
- atomic_dec(&core_dump_count);
+ atomic_dec_unchecked(&core_dump_count);
fail_unlock:
kfree(cn.corename);
coredump_finish(mm, core_dumped);
diff --git a/fs/ext4/ext4.h b/fs/ext4/ext4.h
index f5e9f04..91296b9 100644
--- a/fs/ext4/ext4.h
+++ b/fs/ext4/ext4.h
@@ -1305,19 +1305,19 @@ struct ext4_sb_info {
unsigned long s_mb_last_start;
/* stats for buddy allocator */
- atomic_t s_bal_reqs; /* number of reqs with len > 1 */
- atomic_t s_bal_success; /* we found long enough chunks */
- atomic_t s_bal_allocated; /* in blocks */
- atomic_t s_bal_ex_scanned; /* total extents scanned */
- atomic_t s_bal_goals; /* goal hits */
- atomic_t s_bal_breaks; /* too long searches */
- atomic_t s_bal_2orders; /* 2^order hits */
+ atomic_unchecked_t s_bal_reqs; /* number of reqs with len > 1 */
+ atomic_unchecked_t s_bal_success; /* we found long enough chunks */
+ atomic_unchecked_t s_bal_allocated; /* in blocks */
+ atomic_unchecked_t s_bal_ex_scanned; /* total extents scanned */
+ atomic_unchecked_t s_bal_goals; /* goal hits */
+ atomic_unchecked_t s_bal_breaks; /* too long searches */
+ atomic_unchecked_t s_bal_2orders; /* 2^order hits */
spinlock_t s_bal_lock;
unsigned long s_mb_buddies_generated;
unsigned long long s_mb_generation_time;
- atomic_t s_mb_lost_chunks;
- atomic_t s_mb_preallocated;
- atomic_t s_mb_discarded;
+ atomic_unchecked_t s_mb_lost_chunks;
+ atomic_unchecked_t s_mb_preallocated;
+ atomic_unchecked_t s_mb_discarded;
atomic_t s_lock_busy;
/* locality groups */
diff --git a/fs/ext4/mballoc.c b/fs/ext4/mballoc.c
index 34b610e..ecc47cb 100644
--- a/fs/ext4/mballoc.c
+++ b/fs/ext4/mballoc.c
@@ -1905,7 +1905,7 @@ void ext4_mb_simple_scan_group(struct ext4_allocation_context *ac,
BUG_ON(ac->ac_b_ex.fe_len != ac->ac_g_ex.fe_len);
if (EXT4_SB(sb)->s_mb_stats)
- atomic_inc(&EXT4_SB(sb)->s_bal_2orders);
+ atomic_inc_unchecked(&EXT4_SB(sb)->s_bal_2orders);
break;
}
@@ -2228,7 +2228,7 @@ repeat:
ac->ac_status = AC_STATUS_CONTINUE;
ac->ac_flags |= EXT4_MB_HINT_FIRST;
cr = 3;
- atomic_inc(&sbi->s_mb_lost_chunks);
+ atomic_inc_unchecked(&sbi->s_mb_lost_chunks);
goto repeat;
}
}
@@ -2732,25 +2732,25 @@ int ext4_mb_release(struct super_block *sb)
if (sbi->s_mb_stats) {
ext4_msg(sb, KERN_INFO,
"mballoc: %u blocks %u reqs (%u success)",
- atomic_read(&sbi->s_bal_allocated),
- atomic_read(&sbi->s_bal_reqs),
- atomic_read(&sbi->s_bal_success));
+ atomic_read_unchecked(&sbi->s_bal_allocated),
+ atomic_read_unchecked(&sbi->s_bal_reqs),
+ atomic_read_unchecked(&sbi->s_bal_success));
ext4_msg(sb, KERN_INFO,
"mballoc: %u extents scanned, %u goal hits, "
"%u 2^N hits, %u breaks, %u lost",
- atomic_read(&sbi->s_bal_ex_scanned),
- atomic_read(&sbi->s_bal_goals),
- atomic_read(&sbi->s_bal_2orders),
- atomic_read(&sbi->s_bal_breaks),
- atomic_read(&sbi->s_mb_lost_chunks));
+ atomic_read_unchecked(&sbi->s_bal_ex_scanned),
+ atomic_read_unchecked(&sbi->s_bal_goals),
+ atomic_read_unchecked(&sbi->s_bal_2orders),
+ atomic_read_unchecked(&sbi->s_bal_breaks),
+ atomic_read_unchecked(&sbi->s_mb_lost_chunks));
ext4_msg(sb, KERN_INFO,
"mballoc: %lu generated and it took %Lu",
sbi->s_mb_buddies_generated,
sbi->s_mb_generation_time);
ext4_msg(sb, KERN_INFO,
"mballoc: %u preallocated, %u discarded",
- atomic_read(&sbi->s_mb_preallocated),
- atomic_read(&sbi->s_mb_discarded));
+ atomic_read_unchecked(&sbi->s_mb_preallocated),
+ atomic_read_unchecked(&sbi->s_mb_discarded));
}
free_percpu(sbi->s_locality_groups);
@@ -3206,16 +3206,16 @@ static void ext4_mb_collect_stats(struct ext4_allocation_context *ac)
struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);
if (sbi->s_mb_stats && ac->ac_g_ex.fe_len > 1) {
- atomic_inc(&sbi->s_bal_reqs);
- atomic_add(ac->ac_b_ex.fe_len, &sbi->s_bal_allocated);
+ atomic_inc_unchecked(&sbi->s_bal_reqs);
+ atomic_add_unchecked(ac->ac_b_ex.fe_len, &sbi->s_bal_allocated);
if (ac->ac_b_ex.fe_len >= ac->ac_o_ex.fe_len)
- atomic_inc(&sbi->s_bal_success);
- atomic_add(ac->ac_found, &sbi->s_bal_ex_scanned);
+ atomic_inc_unchecked(&sbi->s_bal_success);
+ atomic_add_unchecked(ac->ac_found, &sbi->s_bal_ex_scanned);
if (ac->ac_g_ex.fe_start == ac->ac_b_ex.fe_start &&
ac->ac_g_ex.fe_group == ac->ac_b_ex.fe_group)
- atomic_inc(&sbi->s_bal_goals);
+ atomic_inc_unchecked(&sbi->s_bal_goals);
if (ac->ac_found > sbi->s_mb_max_to_scan)
- atomic_inc(&sbi->s_bal_breaks);
+ atomic_inc_unchecked(&sbi->s_bal_breaks);
}
if (ac->ac_op == EXT4_MB_HISTORY_ALLOC)
@@ -3642,7 +3642,7 @@ ext4_mb_new_inode_pa(struct ext4_allocation_context *ac)
trace_ext4_mb_new_inode_pa(ac, pa);
ext4_mb_use_inode_pa(ac, pa);
- atomic_add(pa->pa_free, &sbi->s_mb_preallocated);
+ atomic_add_unchecked(pa->pa_free, &sbi->s_mb_preallocated);
ei = EXT4_I(ac->ac_inode);
grp = ext4_get_group_info(sb, ac->ac_b_ex.fe_group);
@@ -3702,7 +3702,7 @@ ext4_mb_new_group_pa(struct ext4_allocation_context *ac)
trace_ext4_mb_new_group_pa(ac, pa);
ext4_mb_use_group_pa(ac, pa);
- atomic_add(pa->pa_free, &EXT4_SB(sb)->s_mb_preallocated);
+ atomic_add_unchecked(pa->pa_free, &EXT4_SB(sb)->s_mb_preallocated);
grp = ext4_get_group_info(sb, ac->ac_b_ex.fe_group);
lg = ac->ac_lg;
@@ -3791,7 +3791,7 @@ ext4_mb_release_inode_pa(struct ext4_buddy *e4b, struct buffer_head *bitmap_bh,
* from the bitmap and continue.
*/
}
- atomic_add(free, &sbi->s_mb_discarded);
+ atomic_add_unchecked(free, &sbi->s_mb_discarded);
return err;
}
@@ -3809,7 +3809,7 @@ ext4_mb_release_group_pa(struct ext4_buddy *e4b,
ext4_get_group_no_and_offset(sb, pa->pa_pstart, &group, &bit);
BUG_ON(group != e4b->bd_group && pa->pa_len != 0);
mb_free_blocks(pa->pa_inode, e4b, bit, pa->pa_len);
- atomic_add(pa->pa_len, &EXT4_SB(sb)->s_mb_discarded);
+ atomic_add_unchecked(pa->pa_len, &EXT4_SB(sb)->s_mb_discarded);
trace_ext4_mballoc_discard(sb, NULL, group, bit, pa->pa_len);
return 0;
diff --git a/fs/fscache/cookie.c b/fs/fscache/cookie.c
index d403c69..30be0a9 100644
--- a/fs/fscache/cookie.c
+++ b/fs/fscache/cookie.c
@@ -19,7 +19,7 @@
struct kmem_cache *fscache_cookie_jar;
-static atomic_t fscache_object_debug_id = ATOMIC_INIT(0);
+static atomic_unchecked_t fscache_object_debug_id = ATOMIC_INIT(0);
static int fscache_acquire_non_index_cookie(struct fscache_cookie *cookie);
static int fscache_alloc_object(struct fscache_cache *cache,
@@ -69,11 +69,11 @@ struct fscache_cookie *__fscache_acquire_cookie(
parent ? (char *) parent->def->name : "<no-parent>",
def->name, netfs_data, enable);
- fscache_stat(&fscache_n_acquires);
+ fscache_stat_unchecked(&fscache_n_acquires);
/* if there's no parent cookie, then we don't create one here either */
if (!parent) {
- fscache_stat(&fscache_n_acquires_null);
+ fscache_stat_unchecked(&fscache_n_acquires_null);
_leave(" [no parent]");
return NULL;
}
@@ -88,7 +88,7 @@ struct fscache_cookie *__fscache_acquire_cookie(
/* allocate and initialise a cookie */
cookie = kmem_cache_alloc(fscache_cookie_jar, GFP_KERNEL);
if (!cookie) {
- fscache_stat(&fscache_n_acquires_oom);
+ fscache_stat_unchecked(&fscache_n_acquires_oom);
_leave(" [ENOMEM]");
return NULL;
}
@@ -115,13 +115,13 @@ struct fscache_cookie *__fscache_acquire_cookie(
switch (cookie->def->type) {
case FSCACHE_COOKIE_TYPE_INDEX:
- fscache_stat(&fscache_n_cookie_index);
+ fscache_stat_unchecked(&fscache_n_cookie_index);
break;
case FSCACHE_COOKIE_TYPE_DATAFILE:
- fscache_stat(&fscache_n_cookie_data);
+ fscache_stat_unchecked(&fscache_n_cookie_data);
break;
default:
- fscache_stat(&fscache_n_cookie_special);
+ fscache_stat_unchecked(&fscache_n_cookie_special);
break;
}
@@ -135,7 +135,7 @@ struct fscache_cookie *__fscache_acquire_cookie(
} else {
atomic_dec(&parent->n_children);
__fscache_cookie_put(cookie);
- fscache_stat(&fscache_n_acquires_nobufs);
+ fscache_stat_unchecked(&fscache_n_acquires_nobufs);
_leave(" = NULL");
return NULL;
}
@@ -144,7 +144,7 @@ struct fscache_cookie *__fscache_acquire_cookie(
}
}
- fscache_stat(&fscache_n_acquires_ok);
+ fscache_stat_unchecked(&fscache_n_acquires_ok);
_leave(" = %p", cookie);
return cookie;
}
@@ -213,7 +213,7 @@ static int fscache_acquire_non_index_cookie(struct fscache_cookie *cookie)
cache = fscache_select_cache_for_object(cookie->parent);
if (!cache) {
up_read(&fscache_addremove_sem);
- fscache_stat(&fscache_n_acquires_no_cache);
+ fscache_stat_unchecked(&fscache_n_acquires_no_cache);
_leave(" = -ENOMEDIUM [no cache]");
return -ENOMEDIUM;
}
@@ -297,14 +297,14 @@ static int fscache_alloc_object(struct fscache_cache *cache,
object = cache->ops->alloc_object(cache, cookie);
fscache_stat_d(&fscache_n_cop_alloc_object);
if (IS_ERR(object)) {
- fscache_stat(&fscache_n_object_no_alloc);
+ fscache_stat_unchecked(&fscache_n_object_no_alloc);
ret = PTR_ERR(object);
goto error;
}
- fscache_stat(&fscache_n_object_alloc);
+ fscache_stat_unchecked(&fscache_n_object_alloc);
- object->debug_id = atomic_inc_return(&fscache_object_debug_id);
+ object->debug_id = atomic_inc_return_unchecked(&fscache_object_debug_id);
_debug("ALLOC OBJ%x: %s {%lx}",
object->debug_id, cookie->def->name, object->events);
@@ -419,7 +419,7 @@ void __fscache_invalidate(struct fscache_cookie *cookie)
_enter("{%s}", cookie->def->name);
- fscache_stat(&fscache_n_invalidates);
+ fscache_stat_unchecked(&fscache_n_invalidates);
/* Only permit invalidation of data files. Invalidating an index will
* require the caller to release all its attachments to the tree rooted
@@ -477,10 +477,10 @@ void __fscache_update_cookie(struct fscache_cookie *cookie)
{
struct fscache_object *object;
- fscache_stat(&fscache_n_updates);
+ fscache_stat_unchecked(&fscache_n_updates);
if (!cookie) {
- fscache_stat(&fscache_n_updates_null);
+ fscache_stat_unchecked(&fscache_n_updates_null);
_leave(" [no cookie]");
return;
}
@@ -581,12 +581,12 @@ EXPORT_SYMBOL(__fscache_disable_cookie);
*/
void __fscache_relinquish_cookie(struct fscache_cookie *cookie, bool retire)
{
- fscache_stat(&fscache_n_relinquishes);
+ fscache_stat_unchecked(&fscache_n_relinquishes);
if (retire)
- fscache_stat(&fscache_n_relinquishes_retire);
+ fscache_stat_unchecked(&fscache_n_relinquishes_retire);
if (!cookie) {
- fscache_stat(&fscache_n_relinquishes_null);
+ fscache_stat_unchecked(&fscache_n_relinquishes_null);
_leave(" [no cookie]");
return;
}
@@ -687,7 +687,7 @@ int __fscache_check_consistency(struct fscache_cookie *cookie)
if (test_bit(FSCACHE_IOERROR, &object->cache->flags))
goto inconsistent;
- op->debug_id = atomic_inc_return(&fscache_op_debug_id);
+ op->debug_id = atomic_inc_return_unchecked(&fscache_op_debug_id);
__fscache_use_cookie(cookie);
if (fscache_submit_op(object, op) < 0)
diff --git a/fs/fscache/internal.h b/fs/fscache/internal.h
index 97ec451..f722cee 100644
--- a/fs/fscache/internal.h
+++ b/fs/fscache/internal.h
@@ -136,8 +136,8 @@ extern void fscache_operation_gc(struct work_struct *);
extern int fscache_wait_for_deferred_lookup(struct fscache_cookie *);
extern int fscache_wait_for_operation_activation(struct fscache_object *,
struct fscache_operation *,
- atomic_t *,
- atomic_t *);
+ atomic_unchecked_t *,
+ atomic_unchecked_t *);
extern void fscache_invalidate_writes(struct fscache_cookie *);
/*
@@ -155,102 +155,102 @@ extern void fscache_proc_cleanup(void);
* stats.c
*/
#ifdef CONFIG_FSCACHE_STATS
-extern atomic_t fscache_n_ops_processed[FSCACHE_MAX_THREADS];
-extern atomic_t fscache_n_objs_processed[FSCACHE_MAX_THREADS];
-
-extern atomic_t fscache_n_op_pend;
-extern atomic_t fscache_n_op_run;
-extern atomic_t fscache_n_op_enqueue;
-extern atomic_t fscache_n_op_deferred_release;
-extern atomic_t fscache_n_op_initialised;
-extern atomic_t fscache_n_op_release;
-extern atomic_t fscache_n_op_gc;
-extern atomic_t fscache_n_op_cancelled;
-extern atomic_t fscache_n_op_rejected;
-
-extern atomic_t fscache_n_attr_changed;
-extern atomic_t fscache_n_attr_changed_ok;
-extern atomic_t fscache_n_attr_changed_nobufs;
-extern atomic_t fscache_n_attr_changed_nomem;
-extern atomic_t fscache_n_attr_changed_calls;
-
-extern atomic_t fscache_n_allocs;
-extern atomic_t fscache_n_allocs_ok;
-extern atomic_t fscache_n_allocs_wait;
-extern atomic_t fscache_n_allocs_nobufs;
-extern atomic_t fscache_n_allocs_intr;
-extern atomic_t fscache_n_allocs_object_dead;
-extern atomic_t fscache_n_alloc_ops;
-extern atomic_t fscache_n_alloc_op_waits;
-
-extern atomic_t fscache_n_retrievals;
-extern atomic_t fscache_n_retrievals_ok;
-extern atomic_t fscache_n_retrievals_wait;
-extern atomic_t fscache_n_retrievals_nodata;
-extern atomic_t fscache_n_retrievals_nobufs;
-extern atomic_t fscache_n_retrievals_intr;
-extern atomic_t fscache_n_retrievals_nomem;
-extern atomic_t fscache_n_retrievals_object_dead;
-extern atomic_t fscache_n_retrieval_ops;
-extern atomic_t fscache_n_retrieval_op_waits;
-
-extern atomic_t fscache_n_stores;
-extern atomic_t fscache_n_stores_ok;
-extern atomic_t fscache_n_stores_again;
-extern atomic_t fscache_n_stores_nobufs;
-extern atomic_t fscache_n_stores_oom;
-extern atomic_t fscache_n_store_ops;
-extern atomic_t fscache_n_store_calls;
-extern atomic_t fscache_n_store_pages;
-extern atomic_t fscache_n_store_radix_deletes;
-extern atomic_t fscache_n_store_pages_over_limit;
-
-extern atomic_t fscache_n_store_vmscan_not_storing;
-extern atomic_t fscache_n_store_vmscan_gone;
-extern atomic_t fscache_n_store_vmscan_busy;
-extern atomic_t fscache_n_store_vmscan_cancelled;
-extern atomic_t fscache_n_store_vmscan_wait;
-
-extern atomic_t fscache_n_marks;
-extern atomic_t fscache_n_uncaches;
-
-extern atomic_t fscache_n_acquires;
-extern atomic_t fscache_n_acquires_null;
-extern atomic_t fscache_n_acquires_no_cache;
-extern atomic_t fscache_n_acquires_ok;
-extern atomic_t fscache_n_acquires_nobufs;
-extern atomic_t fscache_n_acquires_oom;
-
-extern atomic_t fscache_n_invalidates;
-extern atomic_t fscache_n_invalidates_run;
-
-extern atomic_t fscache_n_updates;
-extern atomic_t fscache_n_updates_null;
-extern atomic_t fscache_n_updates_run;
-
-extern atomic_t fscache_n_relinquishes;
-extern atomic_t fscache_n_relinquishes_null;
-extern atomic_t fscache_n_relinquishes_waitcrt;
-extern atomic_t fscache_n_relinquishes_retire;
-
-extern atomic_t fscache_n_cookie_index;
-extern atomic_t fscache_n_cookie_data;
-extern atomic_t fscache_n_cookie_special;
-
-extern atomic_t fscache_n_object_alloc;
-extern atomic_t fscache_n_object_no_alloc;
-extern atomic_t fscache_n_object_lookups;
-extern atomic_t fscache_n_object_lookups_negative;
-extern atomic_t fscache_n_object_lookups_positive;
-extern atomic_t fscache_n_object_lookups_timed_out;
-extern atomic_t fscache_n_object_created;
-extern atomic_t fscache_n_object_avail;
-extern atomic_t fscache_n_object_dead;
-
-extern atomic_t fscache_n_checkaux_none;
-extern atomic_t fscache_n_checkaux_okay;
-extern atomic_t fscache_n_checkaux_update;
-extern atomic_t fscache_n_checkaux_obsolete;
+extern atomic_unchecked_t fscache_n_ops_processed[FSCACHE_MAX_THREADS];
+extern atomic_unchecked_t fscache_n_objs_processed[FSCACHE_MAX_THREADS];
+
+extern atomic_unchecked_t fscache_n_op_pend;
+extern atomic_unchecked_t fscache_n_op_run;
+extern atomic_unchecked_t fscache_n_op_enqueue;
+extern atomic_unchecked_t fscache_n_op_deferred_release;
+extern atomic_unchecked_t fscache_n_op_initialised;
+extern atomic_unchecked_t fscache_n_op_release;
+extern atomic_unchecked_t fscache_n_op_gc;
+extern atomic_unchecked_t fscache_n_op_cancelled;
+extern atomic_unchecked_t fscache_n_op_rejected;
+
+extern atomic_unchecked_t fscache_n_attr_changed;
+extern atomic_unchecked_t fscache_n_attr_changed_ok;
+extern atomic_unchecked_t fscache_n_attr_changed_nobufs;
+extern atomic_unchecked_t fscache_n_attr_changed_nomem;
+extern atomic_unchecked_t fscache_n_attr_changed_calls;
+
+extern atomic_unchecked_t fscache_n_allocs;
+extern atomic_unchecked_t fscache_n_allocs_ok;
+extern atomic_unchecked_t fscache_n_allocs_wait;
+extern atomic_unchecked_t fscache_n_allocs_nobufs;
+extern atomic_unchecked_t fscache_n_allocs_intr;
+extern atomic_unchecked_t fscache_n_allocs_object_dead;
+extern atomic_unchecked_t fscache_n_alloc_ops;
+extern atomic_unchecked_t fscache_n_alloc_op_waits;
+
+extern atomic_unchecked_t fscache_n_retrievals;
+extern atomic_unchecked_t fscache_n_retrievals_ok;
+extern atomic_unchecked_t fscache_n_retrievals_wait;
+extern atomic_unchecked_t fscache_n_retrievals_nodata;
+extern atomic_unchecked_t fscache_n_retrievals_nobufs;
+extern atomic_unchecked_t fscache_n_retrievals_intr;
+extern atomic_unchecked_t fscache_n_retrievals_nomem;
+extern atomic_unchecked_t fscache_n_retrievals_object_dead;
+extern atomic_unchecked_t fscache_n_retrieval_ops;
+extern atomic_unchecked_t fscache_n_retrieval_op_waits;
+
+extern atomic_unchecked_t fscache_n_stores;
+extern atomic_unchecked_t fscache_n_stores_ok;
+extern atomic_unchecked_t fscache_n_stores_again;
+extern atomic_unchecked_t fscache_n_stores_nobufs;
+extern atomic_unchecked_t fscache_n_stores_oom;
+extern atomic_unchecked_t fscache_n_store_ops;
+extern atomic_unchecked_t fscache_n_store_calls;
+extern atomic_unchecked_t fscache_n_store_pages;
+extern atomic_unchecked_t fscache_n_store_radix_deletes;
+extern atomic_unchecked_t fscache_n_store_pages_over_limit;
+
+extern atomic_unchecked_t fscache_n_store_vmscan_not_storing;
+extern atomic_unchecked_t fscache_n_store_vmscan_gone;
+extern atomic_unchecked_t fscache_n_store_vmscan_busy;
+extern atomic_unchecked_t fscache_n_store_vmscan_cancelled;
+extern atomic_unchecked_t fscache_n_store_vmscan_wait;
+
+extern atomic_unchecked_t fscache_n_marks;
+extern atomic_unchecked_t fscache_n_uncaches;
+
+extern atomic_unchecked_t fscache_n_acquires;
+extern atomic_unchecked_t fscache_n_acquires_null;
+extern atomic_unchecked_t fscache_n_acquires_no_cache;
+extern atomic_unchecked_t fscache_n_acquires_ok;
+extern atomic_unchecked_t fscache_n_acquires_nobufs;
+extern atomic_unchecked_t fscache_n_acquires_oom;
+
+extern atomic_unchecked_t fscache_n_invalidates;
+extern atomic_unchecked_t fscache_n_invalidates_run;
+
+extern atomic_unchecked_t fscache_n_updates;
+extern atomic_unchecked_t fscache_n_updates_null;
+extern atomic_unchecked_t fscache_n_updates_run;
+
+extern atomic_unchecked_t fscache_n_relinquishes;
+extern atomic_unchecked_t fscache_n_relinquishes_null;
+extern atomic_unchecked_t fscache_n_relinquishes_waitcrt;
+extern atomic_unchecked_t fscache_n_relinquishes_retire;
+
+extern atomic_unchecked_t fscache_n_cookie_index;
+extern atomic_unchecked_t fscache_n_cookie_data;
+extern atomic_unchecked_t fscache_n_cookie_special;
+
+extern atomic_unchecked_t fscache_n_object_alloc;
+extern atomic_unchecked_t fscache_n_object_no_alloc;
+extern atomic_unchecked_t fscache_n_object_lookups;
+extern atomic_unchecked_t fscache_n_object_lookups_negative;
+extern atomic_unchecked_t fscache_n_object_lookups_positive;
+extern atomic_unchecked_t fscache_n_object_lookups_timed_out;
+extern atomic_unchecked_t fscache_n_object_created;
+extern atomic_unchecked_t fscache_n_object_avail;
+extern atomic_unchecked_t fscache_n_object_dead;
+
+extern atomic_unchecked_t fscache_n_checkaux_none;
+extern atomic_unchecked_t fscache_n_checkaux_okay;
+extern atomic_unchecked_t fscache_n_checkaux_update;
+extern atomic_unchecked_t fscache_n_checkaux_obsolete;
extern atomic_t fscache_n_cop_alloc_object;
extern atomic_t fscache_n_cop_lookup_object;
@@ -280,6 +280,11 @@ static inline void fscache_stat(atomic_t *stat)
atomic_inc(stat);
}
+static inline void fscache_stat_unchecked(atomic_unchecked_t *stat)
+{
+ atomic_inc_unchecked(stat);
+}
+
static inline void fscache_stat_d(atomic_t *stat)
{
atomic_dec(stat);
@@ -292,6 +297,7 @@ extern const struct file_operations fscache_stats_fops;
#define __fscache_stat(stat) (NULL)
#define fscache_stat(stat) do {} while (0)
+#define fscache_stat_unchecked(stat) do {} while (0)
#define fscache_stat_d(stat) do {} while (0)
#endif
diff --git a/fs/fscache/object.c b/fs/fscache/object.c
index 9e792e3..6b2affb 100644
--- a/fs/fscache/object.c
+++ b/fs/fscache/object.c
@@ -465,7 +465,7 @@ static const struct fscache_state *fscache_look_up_object(struct fscache_object
_debug("LOOKUP \"%s\" in \"%s\"",
cookie->def->name, object->cache->tag->name);
- fscache_stat(&fscache_n_object_lookups);
+ fscache_stat_unchecked(&fscache_n_object_lookups);
fscache_stat(&fscache_n_cop_lookup_object);
ret = object->cache->ops->lookup_object(object);
fscache_stat_d(&fscache_n_cop_lookup_object);
@@ -475,7 +475,7 @@ static const struct fscache_state *fscache_look_up_object(struct fscache_object
if (ret == -ETIMEDOUT) {
/* probably stuck behind another object, so move this one to
* the back of the queue */
- fscache_stat(&fscache_n_object_lookups_timed_out);
+ fscache_stat_unchecked(&fscache_n_object_lookups_timed_out);
_leave(" [timeout]");
return NO_TRANSIT;
}
@@ -503,7 +503,7 @@ void fscache_object_lookup_negative(struct fscache_object *object)
_enter("{OBJ%x,%s}", object->debug_id, object->state->name);
if (!test_and_set_bit(FSCACHE_OBJECT_IS_LOOKED_UP, &object->flags)) {
- fscache_stat(&fscache_n_object_lookups_negative);
+ fscache_stat_unchecked(&fscache_n_object_lookups_negative);
/* Allow write requests to begin stacking up and read requests to begin
* returning ENODATA.
@@ -538,7 +538,7 @@ void fscache_obtained_object(struct fscache_object *object)
/* if we were still looking up, then we must have a positive lookup
* result, in which case there may be data available */
if (!test_and_set_bit(FSCACHE_OBJECT_IS_LOOKED_UP, &object->flags)) {
- fscache_stat(&fscache_n_object_lookups_positive);
+ fscache_stat_unchecked(&fscache_n_object_lookups_positive);
/* We do (presumably) have data */
clear_bit_unlock(FSCACHE_COOKIE_NO_DATA_YET, &cookie->flags);
@@ -550,7 +550,7 @@ void fscache_obtained_object(struct fscache_object *object)
clear_bit_unlock(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags);
wake_up_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP);
} else {
- fscache_stat(&fscache_n_object_created);
+ fscache_stat_unchecked(&fscache_n_object_created);
}
set_bit(FSCACHE_OBJECT_IS_AVAILABLE, &object->flags);
@@ -586,7 +586,7 @@ static const struct fscache_state *fscache_object_available(struct fscache_objec
fscache_stat_d(&fscache_n_cop_lookup_complete);
fscache_hist(fscache_obj_instantiate_histogram, object->lookup_jif);
- fscache_stat(&fscache_n_object_avail);
+ fscache_stat_unchecked(&fscache_n_object_avail);
_leave("");
return transit_to(JUMPSTART_DEPS);
@@ -735,7 +735,7 @@ static const struct fscache_state *fscache_drop_object(struct fscache_object *ob
/* this just shifts the object release to the work processor */
fscache_put_object(object);
- fscache_stat(&fscache_n_object_dead);
+ fscache_stat_unchecked(&fscache_n_object_dead);
_leave("");
return transit_to(OBJECT_DEAD);
@@ -900,7 +900,7 @@ enum fscache_checkaux fscache_check_aux(struct fscache_object *object,
enum fscache_checkaux result;
if (!object->cookie->def->check_aux) {
- fscache_stat(&fscache_n_checkaux_none);
+ fscache_stat_unchecked(&fscache_n_checkaux_none);
return FSCACHE_CHECKAUX_OKAY;
}
@@ -909,17 +909,17 @@ enum fscache_checkaux fscache_check_aux(struct fscache_object *object,
switch (result) {
/* entry okay as is */
case FSCACHE_CHECKAUX_OKAY:
- fscache_stat(&fscache_n_checkaux_okay);
+ fscache_stat_unchecked(&fscache_n_checkaux_okay);
break;
/* entry requires update */
case FSCACHE_CHECKAUX_NEEDS_UPDATE:
- fscache_stat(&fscache_n_checkaux_update);
+ fscache_stat_unchecked(&fscache_n_checkaux_update);
break;
/* entry requires deletion */
case FSCACHE_CHECKAUX_OBSOLETE:
- fscache_stat(&fscache_n_checkaux_obsolete);
+ fscache_stat_unchecked(&fscache_n_checkaux_obsolete);
break;
default:
@@ -1007,7 +1007,7 @@ static const struct fscache_state *fscache_invalidate_object(struct fscache_obje
{
const struct fscache_state *s;
- fscache_stat(&fscache_n_invalidates_run);
+ fscache_stat_unchecked(&fscache_n_invalidates_run);
fscache_stat(&fscache_n_cop_invalidate_object);
s = _fscache_invalidate_object(object, event);
fscache_stat_d(&fscache_n_cop_invalidate_object);
@@ -1022,7 +1022,7 @@ static const struct fscache_state *fscache_update_object(struct fscache_object *
{
_enter("{OBJ%x},%d", object->debug_id, event);
- fscache_stat(&fscache_n_updates_run);
+ fscache_stat_unchecked(&fscache_n_updates_run);
fscache_stat(&fscache_n_cop_update_object);
object->cache->ops->update_object(object);
fscache_stat_d(&fscache_n_cop_update_object);
diff --git a/fs/fscache/operation.c b/fs/fscache/operation.c
index de67745..6a3a9b6 100644
--- a/fs/fscache/operation.c
+++ b/fs/fscache/operation.c
@@ -17,7 +17,7 @@
#include <linux/slab.h>
#include "internal.h"
-atomic_t fscache_op_debug_id;
+atomic_unchecked_t fscache_op_debug_id;
EXPORT_SYMBOL(fscache_op_debug_id);
static void fscache_operation_dummy_cancel(struct fscache_operation *op)
@@ -40,12 +40,12 @@ void fscache_operation_init(struct fscache_operation *op,
INIT_WORK(&op->work, fscache_op_work_func);
atomic_set(&op->usage, 1);
op->state = FSCACHE_OP_ST_INITIALISED;
- op->debug_id = atomic_inc_return(&fscache_op_debug_id);
+ op->debug_id = atomic_inc_return_unchecked(&fscache_op_debug_id);
op->processor = processor;
op->cancel = cancel ?: fscache_operation_dummy_cancel;
op->release = release;
INIT_LIST_HEAD(&op->pend_link);
- fscache_stat(&fscache_n_op_initialised);
+ fscache_stat_unchecked(&fscache_n_op_initialised);
}
EXPORT_SYMBOL(fscache_operation_init);
@@ -68,7 +68,7 @@ void fscache_enqueue_operation(struct fscache_operation *op)
ASSERTCMP(atomic_read(&op->usage), >, 0);
ASSERTCMP(op->state, ==, FSCACHE_OP_ST_IN_PROGRESS);
- fscache_stat(&fscache_n_op_enqueue);
+ fscache_stat_unchecked(&fscache_n_op_enqueue);
switch (op->flags & FSCACHE_OP_TYPE) {
case FSCACHE_OP_ASYNC:
_debug("queue async");
@@ -101,7 +101,7 @@ static void fscache_run_op(struct fscache_object *object,
wake_up_bit(&op->flags, FSCACHE_OP_WAITING);
if (op->processor)
fscache_enqueue_operation(op);
- fscache_stat(&fscache_n_op_run);
+ fscache_stat_unchecked(&fscache_n_op_run);
}
/*
@@ -169,7 +169,7 @@ int fscache_submit_exclusive_op(struct fscache_object *object,
op->state = FSCACHE_OP_ST_PENDING;
flags = READ_ONCE(object->flags);
if (unlikely(!(flags & BIT(FSCACHE_OBJECT_IS_LIVE)))) {
- fscache_stat(&fscache_n_op_rejected);
+ fscache_stat_unchecked(&fscache_n_op_rejected);
op->cancel(op);
op->state = FSCACHE_OP_ST_CANCELLED;
ret = -ENOBUFS;
@@ -185,11 +185,11 @@ int fscache_submit_exclusive_op(struct fscache_object *object,
if (object->n_in_progress > 0) {
atomic_inc(&op->usage);
list_add_tail(&op->pend_link, &object->pending_ops);
- fscache_stat(&fscache_n_op_pend);
+ fscache_stat_unchecked(&fscache_n_op_pend);
} else if (!list_empty(&object->pending_ops)) {
atomic_inc(&op->usage);
list_add_tail(&op->pend_link, &object->pending_ops);
- fscache_stat(&fscache_n_op_pend);
+ fscache_stat_unchecked(&fscache_n_op_pend);
fscache_start_operations(object);
} else {
ASSERTCMP(object->n_in_progress, ==, 0);
@@ -205,7 +205,7 @@ int fscache_submit_exclusive_op(struct fscache_object *object,
object->n_exclusive++; /* reads and writes must wait */
atomic_inc(&op->usage);
list_add_tail(&op->pend_link, &object->pending_ops);
- fscache_stat(&fscache_n_op_pend);
+ fscache_stat_unchecked(&fscache_n_op_pend);
ret = 0;
} else if (flags & BIT(FSCACHE_OBJECT_KILLED_BY_CACHE)) {
op->cancel(op);
@@ -254,7 +254,7 @@ int fscache_submit_op(struct fscache_object *object,
op->state = FSCACHE_OP_ST_PENDING;
flags = READ_ONCE(object->flags);
if (unlikely(!(flags & BIT(FSCACHE_OBJECT_IS_LIVE)))) {
- fscache_stat(&fscache_n_op_rejected);
+ fscache_stat_unchecked(&fscache_n_op_rejected);
op->cancel(op);
op->state = FSCACHE_OP_ST_CANCELLED;
ret = -ENOBUFS;
@@ -269,11 +269,11 @@ int fscache_submit_op(struct fscache_object *object,
if (object->n_exclusive > 0) {
atomic_inc(&op->usage);
list_add_tail(&op->pend_link, &object->pending_ops);
- fscache_stat(&fscache_n_op_pend);
+ fscache_stat_unchecked(&fscache_n_op_pend);
} else if (!list_empty(&object->pending_ops)) {
atomic_inc(&op->usage);
list_add_tail(&op->pend_link, &object->pending_ops);
- fscache_stat(&fscache_n_op_pend);
+ fscache_stat_unchecked(&fscache_n_op_pend);
fscache_start_operations(object);
} else {
ASSERTCMP(object->n_exclusive, ==, 0);
@@ -285,7 +285,7 @@ int fscache_submit_op(struct fscache_object *object,
object->n_ops++;
atomic_inc(&op->usage);
list_add_tail(&op->pend_link, &object->pending_ops);
- fscache_stat(&fscache_n_op_pend);
+ fscache_stat_unchecked(&fscache_n_op_pend);
ret = 0;
} else if (flags & BIT(FSCACHE_OBJECT_KILLED_BY_CACHE)) {
op->cancel(op);
@@ -369,7 +369,7 @@ int fscache_cancel_op(struct fscache_operation *op,
list_del_init(&op->pend_link);
put = true;
- fscache_stat(&fscache_n_op_cancelled);
+ fscache_stat_unchecked(&fscache_n_op_cancelled);
op->cancel(op);
op->state = FSCACHE_OP_ST_CANCELLED;
if (test_bit(FSCACHE_OP_EXCLUSIVE, &op->flags))
@@ -385,7 +385,7 @@ int fscache_cancel_op(struct fscache_operation *op,
if (object->n_in_progress == 0)
fscache_start_operations(object);
- fscache_stat(&fscache_n_op_cancelled);
+ fscache_stat_unchecked(&fscache_n_op_cancelled);
op->cancel(op);
op->state = FSCACHE_OP_ST_CANCELLED;
if (test_bit(FSCACHE_OP_EXCLUSIVE, &op->flags))
@@ -416,7 +416,7 @@ void fscache_cancel_all_ops(struct fscache_object *object)
while (!list_empty(&object->pending_ops)) {
op = list_entry(object->pending_ops.next,
struct fscache_operation, pend_link);
- fscache_stat(&fscache_n_op_cancelled);
+ fscache_stat_unchecked(&fscache_n_op_cancelled);
list_del_init(&op->pend_link);
ASSERTCMP(op->state, ==, FSCACHE_OP_ST_PENDING);
@@ -493,7 +493,7 @@ void fscache_put_operation(struct fscache_operation *op)
op->state != FSCACHE_OP_ST_COMPLETE,
op->state, ==, FSCACHE_OP_ST_CANCELLED);
- fscache_stat(&fscache_n_op_release);
+ fscache_stat_unchecked(&fscache_n_op_release);
if (op->release) {
op->release(op);
@@ -513,7 +513,7 @@ void fscache_put_operation(struct fscache_operation *op)
* lock, and defer it otherwise */
if (!spin_trylock(&object->lock)) {
_debug("defer put");
- fscache_stat(&fscache_n_op_deferred_release);
+ fscache_stat_unchecked(&fscache_n_op_deferred_release);
cache = object->cache;
spin_lock(&cache->op_gc_list_lock);
@@ -567,7 +567,7 @@ void fscache_operation_gc(struct work_struct *work)
_debug("GC DEFERRED REL OBJ%x OP%x",
object->debug_id, op->debug_id);
- fscache_stat(&fscache_n_op_gc);
+ fscache_stat_unchecked(&fscache_n_op_gc);
ASSERTCMP(atomic_read(&op->usage), ==, 0);
ASSERTCMP(op->state, ==, FSCACHE_OP_ST_DEAD);
diff --git a/fs/fscache/page.c b/fs/fscache/page.c
index 483bbc6..ba36737 100644
--- a/fs/fscache/page.c
+++ b/fs/fscache/page.c
@@ -74,7 +74,7 @@ try_again:
val = radix_tree_lookup(&cookie->stores, page->index);
if (!val) {
rcu_read_unlock();
- fscache_stat(&fscache_n_store_vmscan_not_storing);
+ fscache_stat_unchecked(&fscache_n_store_vmscan_not_storing);
__fscache_uncache_page(cookie, page);
return true;
}
@@ -104,11 +104,11 @@ try_again:
spin_unlock(&cookie->stores_lock);
if (xpage) {
- fscache_stat(&fscache_n_store_vmscan_cancelled);
- fscache_stat(&fscache_n_store_radix_deletes);
+ fscache_stat_unchecked(&fscache_n_store_vmscan_cancelled);
+ fscache_stat_unchecked(&fscache_n_store_radix_deletes);
ASSERTCMP(xpage, ==, page);
} else {
- fscache_stat(&fscache_n_store_vmscan_gone);
+ fscache_stat_unchecked(&fscache_n_store_vmscan_gone);
}
wake_up_bit(&cookie->flags, 0);
@@ -123,11 +123,11 @@ page_busy:
* sleeping on memory allocation, so we may need to impose a timeout
* too. */
if (!(gfp & __GFP_WAIT) || !(gfp & __GFP_FS)) {
- fscache_stat(&fscache_n_store_vmscan_busy);
+ fscache_stat_unchecked(&fscache_n_store_vmscan_busy);
return false;
}
- fscache_stat(&fscache_n_store_vmscan_wait);
+ fscache_stat_unchecked(&fscache_n_store_vmscan_wait);
if (!release_page_wait_timeout(cookie, page))
_debug("fscache writeout timeout page: %p{%lx}",
page, page->index);
@@ -156,7 +156,7 @@ static void fscache_end_page_write(struct fscache_object *object,
FSCACHE_COOKIE_STORING_TAG);
if (!radix_tree_tag_get(&cookie->stores, page->index,
FSCACHE_COOKIE_PENDING_TAG)) {
- fscache_stat(&fscache_n_store_radix_deletes);
+ fscache_stat_unchecked(&fscache_n_store_radix_deletes);
xpage = radix_tree_delete(&cookie->stores, page->index);
}
spin_unlock(&cookie->stores_lock);
@@ -177,7 +177,7 @@ static void fscache_attr_changed_op(struct fscache_operation *op)
_enter("{OBJ%x OP%x}", object->debug_id, op->debug_id);
- fscache_stat(&fscache_n_attr_changed_calls);
+ fscache_stat_unchecked(&fscache_n_attr_changed_calls);
if (fscache_object_is_active(object)) {
fscache_stat(&fscache_n_cop_attr_changed);
@@ -204,11 +204,11 @@ int __fscache_attr_changed(struct fscache_cookie *cookie)
ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
- fscache_stat(&fscache_n_attr_changed);
+ fscache_stat_unchecked(&fscache_n_attr_changed);
op = kzalloc(sizeof(*op), GFP_KERNEL);
if (!op) {
- fscache_stat(&fscache_n_attr_changed_nomem);
+ fscache_stat_unchecked(&fscache_n_attr_changed_nomem);
_leave(" = -ENOMEM");
return -ENOMEM;
}
@@ -230,7 +230,7 @@ int __fscache_attr_changed(struct fscache_cookie *cookie)
if (fscache_submit_exclusive_op(object, op) < 0)
goto nobufs_dec;
spin_unlock(&cookie->lock);
- fscache_stat(&fscache_n_attr_changed_ok);
+ fscache_stat_unchecked(&fscache_n_attr_changed_ok);
fscache_put_operation(op);
_leave(" = 0");
return 0;
@@ -242,7 +242,7 @@ nobufs:
fscache_put_operation(op);
if (wake_cookie)
__fscache_wake_unused_cookie(cookie);
- fscache_stat(&fscache_n_attr_changed_nobufs);
+ fscache_stat_unchecked(&fscache_n_attr_changed_nobufs);
_leave(" = %d", -ENOBUFS);
return -ENOBUFS;
}
@@ -293,7 +293,7 @@ static struct fscache_retrieval *fscache_alloc_retrieval(
/* allocate a retrieval operation and attempt to submit it */
op = kzalloc(sizeof(*op), GFP_NOIO);
if (!op) {
- fscache_stat(&fscache_n_retrievals_nomem);
+ fscache_stat_unchecked(&fscache_n_retrievals_nomem);
return NULL;
}
@@ -332,12 +332,12 @@ int fscache_wait_for_deferred_lookup(struct fscache_cookie *cookie)
return 0;
}
- fscache_stat(&fscache_n_retrievals_wait);
+ fscache_stat_unchecked(&fscache_n_retrievals_wait);
jif = jiffies;
if (wait_on_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP,
TASK_INTERRUPTIBLE) != 0) {
- fscache_stat(&fscache_n_retrievals_intr);
+ fscache_stat_unchecked(&fscache_n_retrievals_intr);
_leave(" = -ERESTARTSYS");
return -ERESTARTSYS;
}
@@ -355,8 +355,8 @@ int fscache_wait_for_deferred_lookup(struct fscache_cookie *cookie)
*/
int fscache_wait_for_operation_activation(struct fscache_object *object,
struct fscache_operation *op,
- atomic_t *stat_op_waits,
- atomic_t *stat_object_dead)
+ atomic_unchecked_t *stat_op_waits,
+ atomic_unchecked_t *stat_object_dead)
{
int ret;
@@ -365,7 +365,7 @@ int fscache_wait_for_operation_activation(struct fscache_object *object,
_debug(">>> WT");
if (stat_op_waits)
- fscache_stat(stat_op_waits);
+ fscache_stat_unchecked(stat_op_waits);
if (wait_on_bit(&op->flags, FSCACHE_OP_WAITING,
TASK_INTERRUPTIBLE) != 0) {
ret = fscache_cancel_op(op, false);
@@ -382,7 +382,7 @@ int fscache_wait_for_operation_activation(struct fscache_object *object,
check_if_dead:
if (op->state == FSCACHE_OP_ST_CANCELLED) {
if (stat_object_dead)
- fscache_stat(stat_object_dead);
+ fscache_stat_unchecked(stat_object_dead);
_leave(" = -ENOBUFS [cancelled]");
return -ENOBUFS;
}
@@ -391,7 +391,7 @@ check_if_dead:
enum fscache_operation_state state = op->state;
fscache_cancel_op(op, true);
if (stat_object_dead)
- fscache_stat(stat_object_dead);
+ fscache_stat_unchecked(stat_object_dead);
_leave(" = -ENOBUFS [obj dead %d]", state);
return -ENOBUFS;
}
@@ -420,7 +420,7 @@ int __fscache_read_or_alloc_page(struct fscache_cookie *cookie,
_enter("%p,%p,,,", cookie, page);
- fscache_stat(&fscache_n_retrievals);
+ fscache_stat_unchecked(&fscache_n_retrievals);
if (hlist_empty(&cookie->backing_objects))
goto nobufs;
@@ -462,7 +462,7 @@ int __fscache_read_or_alloc_page(struct fscache_cookie *cookie,
goto nobufs_unlock_dec;
spin_unlock(&cookie->lock);
- fscache_stat(&fscache_n_retrieval_ops);
+ fscache_stat_unchecked(&fscache_n_retrieval_ops);
/* we wait for the operation to become active, and then process it
* *here*, in this thread, and not in the thread pool */
@@ -488,15 +488,15 @@ int __fscache_read_or_alloc_page(struct fscache_cookie *cookie,
error:
if (ret == -ENOMEM)
- fscache_stat(&fscache_n_retrievals_nomem);
+ fscache_stat_unchecked(&fscache_n_retrievals_nomem);
else if (ret == -ERESTARTSYS)
- fscache_stat(&fscache_n_retrievals_intr);
+ fscache_stat_unchecked(&fscache_n_retrievals_intr);
else if (ret == -ENODATA)
- fscache_stat(&fscache_n_retrievals_nodata);
+ fscache_stat_unchecked(&fscache_n_retrievals_nodata);
else if (ret < 0)
- fscache_stat(&fscache_n_retrievals_nobufs);
+ fscache_stat_unchecked(&fscache_n_retrievals_nobufs);
else
- fscache_stat(&fscache_n_retrievals_ok);
+ fscache_stat_unchecked(&fscache_n_retrievals_ok);
fscache_put_retrieval(op);
_leave(" = %d", ret);
@@ -511,7 +511,7 @@ nobufs_unlock:
__fscache_wake_unused_cookie(cookie);
fscache_put_retrieval(op);
nobufs:
- fscache_stat(&fscache_n_retrievals_nobufs);
+ fscache_stat_unchecked(&fscache_n_retrievals_nobufs);
_leave(" = -ENOBUFS");
return -ENOBUFS;
}
@@ -550,7 +550,7 @@ int __fscache_read_or_alloc_pages(struct fscache_cookie *cookie,
_enter("%p,,%d,,,", cookie, *nr_pages);
- fscache_stat(&fscache_n_retrievals);
+ fscache_stat_unchecked(&fscache_n_retrievals);
if (hlist_empty(&cookie->backing_objects))
goto nobufs;
@@ -588,7 +588,7 @@ int __fscache_read_or_alloc_pages(struct fscache_cookie *cookie,
goto nobufs_unlock_dec;
spin_unlock(&cookie->lock);
- fscache_stat(&fscache_n_retrieval_ops);
+ fscache_stat_unchecked(&fscache_n_retrieval_ops);
/* we wait for the operation to become active, and then process it
* *here*, in this thread, and not in the thread pool */
@@ -614,15 +614,15 @@ int __fscache_read_or_alloc_pages(struct fscache_cookie *cookie,
error:
if (ret == -ENOMEM)
- fscache_stat(&fscache_n_retrievals_nomem);
+ fscache_stat_unchecked(&fscache_n_retrievals_nomem);
else if (ret == -ERESTARTSYS)
- fscache_stat(&fscache_n_retrievals_intr);
+ fscache_stat_unchecked(&fscache_n_retrievals_intr);
else if (ret == -ENODATA)
- fscache_stat(&fscache_n_retrievals_nodata);
+ fscache_stat_unchecked(&fscache_n_retrievals_nodata);
else if (ret < 0)
- fscache_stat(&fscache_n_retrievals_nobufs);
+ fscache_stat_unchecked(&fscache_n_retrievals_nobufs);
else
- fscache_stat(&fscache_n_retrievals_ok);
+ fscache_stat_unchecked(&fscache_n_retrievals_ok);
fscache_put_retrieval(op);
_leave(" = %d", ret);
@@ -637,7 +637,7 @@ nobufs_unlock:
if (wake_cookie)
__fscache_wake_unused_cookie(cookie);
nobufs:
- fscache_stat(&fscache_n_retrievals_nobufs);
+ fscache_stat_unchecked(&fscache_n_retrievals_nobufs);
_leave(" = -ENOBUFS");
return -ENOBUFS;
}
@@ -662,7 +662,7 @@ int __fscache_alloc_page(struct fscache_cookie *cookie,
_enter("%p,%p,,,", cookie, page);
- fscache_stat(&fscache_n_allocs);
+ fscache_stat_unchecked(&fscache_n_allocs);
if (hlist_empty(&cookie->backing_objects))
goto nobufs;
@@ -696,7 +696,7 @@ int __fscache_alloc_page(struct fscache_cookie *cookie,
goto nobufs_unlock_dec;
spin_unlock(&cookie->lock);
- fscache_stat(&fscache_n_alloc_ops);
+ fscache_stat_unchecked(&fscache_n_alloc_ops);
ret = fscache_wait_for_operation_activation(
object, &op->op,
@@ -712,11 +712,11 @@ int __fscache_alloc_page(struct fscache_cookie *cookie,
error:
if (ret == -ERESTARTSYS)
- fscache_stat(&fscache_n_allocs_intr);
+ fscache_stat_unchecked(&fscache_n_allocs_intr);
else if (ret < 0)
- fscache_stat(&fscache_n_allocs_nobufs);
+ fscache_stat_unchecked(&fscache_n_allocs_nobufs);
else
- fscache_stat(&fscache_n_allocs_ok);
+ fscache_stat_unchecked(&fscache_n_allocs_ok);
fscache_put_retrieval(op);
_leave(" = %d", ret);
@@ -730,7 +730,7 @@ nobufs_unlock:
if (wake_cookie)
__fscache_wake_unused_cookie(cookie);
nobufs:
- fscache_stat(&fscache_n_allocs_nobufs);
+ fscache_stat_unchecked(&fscache_n_allocs_nobufs);
_leave(" = -ENOBUFS");
return -ENOBUFS;
}
@@ -806,7 +806,7 @@ static void fscache_write_op(struct fscache_operation *_op)
spin_lock(&cookie->stores_lock);
- fscache_stat(&fscache_n_store_calls);
+ fscache_stat_unchecked(&fscache_n_store_calls);
/* find a page to store */
page = NULL;
@@ -817,7 +817,7 @@ static void fscache_write_op(struct fscache_operation *_op)
page = results[0];
_debug("gang %d [%lx]", n, page->index);
if (page->index > op->store_limit) {
- fscache_stat(&fscache_n_store_pages_over_limit);
+ fscache_stat_unchecked(&fscache_n_store_pages_over_limit);
goto superseded;
}
@@ -829,7 +829,7 @@ static void fscache_write_op(struct fscache_operation *_op)
spin_unlock(&cookie->stores_lock);
spin_unlock(&object->lock);
- fscache_stat(&fscache_n_store_pages);
+ fscache_stat_unchecked(&fscache_n_store_pages);
fscache_stat(&fscache_n_cop_write_page);
ret = object->cache->ops->write_page(op, page);
fscache_stat_d(&fscache_n_cop_write_page);
@@ -933,7 +933,7 @@ int __fscache_write_page(struct fscache_cookie *cookie,
ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
ASSERT(PageFsCache(page));
- fscache_stat(&fscache_n_stores);
+ fscache_stat_unchecked(&fscache_n_stores);
if (test_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) {
_leave(" = -ENOBUFS [invalidating]");
@@ -992,7 +992,7 @@ int __fscache_write_page(struct fscache_cookie *cookie,
spin_unlock(&cookie->stores_lock);
spin_unlock(&object->lock);
- op->op.debug_id = atomic_inc_return(&fscache_op_debug_id);
+ op->op.debug_id = atomic_inc_return_unchecked(&fscache_op_debug_id);
op->store_limit = object->store_limit;
__fscache_use_cookie(cookie);
@@ -1001,8 +1001,8 @@ int __fscache_write_page(struct fscache_cookie *cookie,
spin_unlock(&cookie->lock);
radix_tree_preload_end();
- fscache_stat(&fscache_n_store_ops);
- fscache_stat(&fscache_n_stores_ok);
+ fscache_stat_unchecked(&fscache_n_store_ops);
+ fscache_stat_unchecked(&fscache_n_stores_ok);
/* the work queue now carries its own ref on the object */
fscache_put_operation(&op->op);
@@ -1010,14 +1010,14 @@ int __fscache_write_page(struct fscache_cookie *cookie,
return 0;
already_queued:
- fscache_stat(&fscache_n_stores_again);
+ fscache_stat_unchecked(&fscache_n_stores_again);
already_pending:
spin_unlock(&cookie->stores_lock);
spin_unlock(&object->lock);
spin_unlock(&cookie->lock);
radix_tree_preload_end();
fscache_put_operation(&op->op);
- fscache_stat(&fscache_n_stores_ok);
+ fscache_stat_unchecked(&fscache_n_stores_ok);
_leave(" = 0");
return 0;
@@ -1039,14 +1039,14 @@ nobufs:
fscache_put_operation(&op->op);
if (wake_cookie)
__fscache_wake_unused_cookie(cookie);
- fscache_stat(&fscache_n_stores_nobufs);
+ fscache_stat_unchecked(&fscache_n_stores_nobufs);
_leave(" = -ENOBUFS");
return -ENOBUFS;
nomem_free:
fscache_put_operation(&op->op);
nomem:
- fscache_stat(&fscache_n_stores_oom);
+ fscache_stat_unchecked(&fscache_n_stores_oom);
_leave(" = -ENOMEM");
return -ENOMEM;
}
@@ -1064,7 +1064,7 @@ void __fscache_uncache_page(struct fscache_cookie *cookie, struct page *page)
ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
ASSERTCMP(page, !=, NULL);
- fscache_stat(&fscache_n_uncaches);
+ fscache_stat_unchecked(&fscache_n_uncaches);
/* cache withdrawal may beat us to it */
if (!PageFsCache(page))
@@ -1115,7 +1115,7 @@ void fscache_mark_page_cached(struct fscache_retrieval *op, struct page *page)
struct fscache_cookie *cookie = op->op.object->cookie;
#ifdef CONFIG_FSCACHE_STATS
- atomic_inc(&fscache_n_marks);
+ atomic_inc_unchecked(&fscache_n_marks);
#endif
_debug("- mark %p{%lx}", page, page->index);
diff --git a/fs/fscache/stats.c b/fs/fscache/stats.c
index 7cfa0aa..d5ef97b7 100644
--- a/fs/fscache/stats.c
+++ b/fs/fscache/stats.c
@@ -18,100 +18,100 @@
/*
* operation counters
*/
-atomic_t fscache_n_op_pend;
-atomic_t fscache_n_op_run;
-atomic_t fscache_n_op_enqueue;
-atomic_t fscache_n_op_requeue;
-atomic_t fscache_n_op_deferred_release;
-atomic_t fscache_n_op_initialised;
-atomic_t fscache_n_op_release;
-atomic_t fscache_n_op_gc;
-atomic_t fscache_n_op_cancelled;
-atomic_t fscache_n_op_rejected;
-
-atomic_t fscache_n_attr_changed;
-atomic_t fscache_n_attr_changed_ok;
-atomic_t fscache_n_attr_changed_nobufs;
-atomic_t fscache_n_attr_changed_nomem;
-atomic_t fscache_n_attr_changed_calls;
-
-atomic_t fscache_n_allocs;
-atomic_t fscache_n_allocs_ok;
-atomic_t fscache_n_allocs_wait;
-atomic_t fscache_n_allocs_nobufs;
-atomic_t fscache_n_allocs_intr;
-atomic_t fscache_n_allocs_object_dead;
-atomic_t fscache_n_alloc_ops;
-atomic_t fscache_n_alloc_op_waits;
-
-atomic_t fscache_n_retrievals;
-atomic_t fscache_n_retrievals_ok;
-atomic_t fscache_n_retrievals_wait;
-atomic_t fscache_n_retrievals_nodata;
-atomic_t fscache_n_retrievals_nobufs;
-atomic_t fscache_n_retrievals_intr;
-atomic_t fscache_n_retrievals_nomem;
-atomic_t fscache_n_retrievals_object_dead;
-atomic_t fscache_n_retrieval_ops;
-atomic_t fscache_n_retrieval_op_waits;
-
-atomic_t fscache_n_stores;
-atomic_t fscache_n_stores_ok;
-atomic_t fscache_n_stores_again;
-atomic_t fscache_n_stores_nobufs;
-atomic_t fscache_n_stores_oom;
-atomic_t fscache_n_store_ops;
-atomic_t fscache_n_store_calls;
-atomic_t fscache_n_store_pages;
-atomic_t fscache_n_store_radix_deletes;
-atomic_t fscache_n_store_pages_over_limit;
-
-atomic_t fscache_n_store_vmscan_not_storing;
-atomic_t fscache_n_store_vmscan_gone;
-atomic_t fscache_n_store_vmscan_busy;
-atomic_t fscache_n_store_vmscan_cancelled;
-atomic_t fscache_n_store_vmscan_wait;
-
-atomic_t fscache_n_marks;
-atomic_t fscache_n_uncaches;
-
-atomic_t fscache_n_acquires;
-atomic_t fscache_n_acquires_null;
-atomic_t fscache_n_acquires_no_cache;
-atomic_t fscache_n_acquires_ok;
-atomic_t fscache_n_acquires_nobufs;
-atomic_t fscache_n_acquires_oom;
-
-atomic_t fscache_n_invalidates;
-atomic_t fscache_n_invalidates_run;
-
-atomic_t fscache_n_updates;
-atomic_t fscache_n_updates_null;
-atomic_t fscache_n_updates_run;
-
-atomic_t fscache_n_relinquishes;
-atomic_t fscache_n_relinquishes_null;
-atomic_t fscache_n_relinquishes_waitcrt;
-atomic_t fscache_n_relinquishes_retire;
-
-atomic_t fscache_n_cookie_index;
-atomic_t fscache_n_cookie_data;
-atomic_t fscache_n_cookie_special;
-
-atomic_t fscache_n_object_alloc;
-atomic_t fscache_n_object_no_alloc;
-atomic_t fscache_n_object_lookups;
-atomic_t fscache_n_object_lookups_negative;
-atomic_t fscache_n_object_lookups_positive;
-atomic_t fscache_n_object_lookups_timed_out;
-atomic_t fscache_n_object_created;
-atomic_t fscache_n_object_avail;
-atomic_t fscache_n_object_dead;
-
-atomic_t fscache_n_checkaux_none;
-atomic_t fscache_n_checkaux_okay;
-atomic_t fscache_n_checkaux_update;
-atomic_t fscache_n_checkaux_obsolete;
+atomic_unchecked_t fscache_n_op_pend;
+atomic_unchecked_t fscache_n_op_run;
+atomic_unchecked_t fscache_n_op_enqueue;
+atomic_unchecked_t fscache_n_op_requeue;
+atomic_unchecked_t fscache_n_op_deferred_release;
+atomic_unchecked_t fscache_n_op_initialised;
+atomic_unchecked_t fscache_n_op_release;
+atomic_unchecked_t fscache_n_op_gc;
+atomic_unchecked_t fscache_n_op_cancelled;
+atomic_unchecked_t fscache_n_op_rejected;
+
+atomic_unchecked_t fscache_n_attr_changed;
+atomic_unchecked_t fscache_n_attr_changed_ok;
+atomic_unchecked_t fscache_n_attr_changed_nobufs;
+atomic_unchecked_t fscache_n_attr_changed_nomem;
+atomic_unchecked_t fscache_n_attr_changed_calls;
+
+atomic_unchecked_t fscache_n_allocs;
+atomic_unchecked_t fscache_n_allocs_ok;
+atomic_unchecked_t fscache_n_allocs_wait;
+atomic_unchecked_t fscache_n_allocs_nobufs;
+atomic_unchecked_t fscache_n_allocs_intr;
+atomic_unchecked_t fscache_n_allocs_object_dead;
+atomic_unchecked_t fscache_n_alloc_ops;
+atomic_unchecked_t fscache_n_alloc_op_waits;
+
+atomic_unchecked_t fscache_n_retrievals;
+atomic_unchecked_t fscache_n_retrievals_ok;
+atomic_unchecked_t fscache_n_retrievals_wait;
+atomic_unchecked_t fscache_n_retrievals_nodata;
+atomic_unchecked_t fscache_n_retrievals_nobufs;
+atomic_unchecked_t fscache_n_retrievals_intr;
+atomic_unchecked_t fscache_n_retrievals_nomem;
+atomic_unchecked_t fscache_n_retrievals_object_dead;
+atomic_unchecked_t fscache_n_retrieval_ops;
+atomic_unchecked_t fscache_n_retrieval_op_waits;
+
+atomic_unchecked_t fscache_n_stores;
+atomic_unchecked_t fscache_n_stores_ok;
+atomic_unchecked_t fscache_n_stores_again;
+atomic_unchecked_t fscache_n_stores_nobufs;
+atomic_unchecked_t fscache_n_stores_oom;
+atomic_unchecked_t fscache_n_store_ops;
+atomic_unchecked_t fscache_n_store_calls;
+atomic_unchecked_t fscache_n_store_pages;
+atomic_unchecked_t fscache_n_store_radix_deletes;
+atomic_unchecked_t fscache_n_store_pages_over_limit;
+
+atomic_unchecked_t fscache_n_store_vmscan_not_storing;
+atomic_unchecked_t fscache_n_store_vmscan_gone;
+atomic_unchecked_t fscache_n_store_vmscan_busy;
+atomic_unchecked_t fscache_n_store_vmscan_cancelled;
+atomic_unchecked_t fscache_n_store_vmscan_wait;
+
+atomic_unchecked_t fscache_n_marks;
+atomic_unchecked_t fscache_n_uncaches;
+
+atomic_unchecked_t fscache_n_acquires;
+atomic_unchecked_t fscache_n_acquires_null;
+atomic_unchecked_t fscache_n_acquires_no_cache;
+atomic_unchecked_t fscache_n_acquires_ok;
+atomic_unchecked_t fscache_n_acquires_nobufs;
+atomic_unchecked_t fscache_n_acquires_oom;
+
+atomic_unchecked_t fscache_n_invalidates;
+atomic_unchecked_t fscache_n_invalidates_run;
+
+atomic_unchecked_t fscache_n_updates;
+atomic_unchecked_t fscache_n_updates_null;
+atomic_unchecked_t fscache_n_updates_run;
+
+atomic_unchecked_t fscache_n_relinquishes;
+atomic_unchecked_t fscache_n_relinquishes_null;
+atomic_unchecked_t fscache_n_relinquishes_waitcrt;
+atomic_unchecked_t fscache_n_relinquishes_retire;
+
+atomic_unchecked_t fscache_n_cookie_index;
+atomic_unchecked_t fscache_n_cookie_data;
+atomic_unchecked_t fscache_n_cookie_special;
+
+atomic_unchecked_t fscache_n_object_alloc;
+atomic_unchecked_t fscache_n_object_no_alloc;
+atomic_unchecked_t fscache_n_object_lookups;
+atomic_unchecked_t fscache_n_object_lookups_negative;
+atomic_unchecked_t fscache_n_object_lookups_positive;
+atomic_unchecked_t fscache_n_object_lookups_timed_out;
+atomic_unchecked_t fscache_n_object_created;
+atomic_unchecked_t fscache_n_object_avail;
+atomic_unchecked_t fscache_n_object_dead;
+
+atomic_unchecked_t fscache_n_checkaux_none;
+atomic_unchecked_t fscache_n_checkaux_okay;
+atomic_unchecked_t fscache_n_checkaux_update;
+atomic_unchecked_t fscache_n_checkaux_obsolete;
atomic_t fscache_n_cop_alloc_object;
atomic_t fscache_n_cop_lookup_object;
@@ -144,119 +144,119 @@ static int fscache_stats_show(struct seq_file *m, void *v)
seq_puts(m, "FS-Cache statistics\n");
seq_printf(m, "Cookies: idx=%u dat=%u spc=%u\n",
- atomic_read(&fscache_n_cookie_index),
- atomic_read(&fscache_n_cookie_data),
- atomic_read(&fscache_n_cookie_special));
+ atomic_read_unchecked(&fscache_n_cookie_index),
+ atomic_read_unchecked(&fscache_n_cookie_data),
+ atomic_read_unchecked(&fscache_n_cookie_special));
seq_printf(m, "Objects: alc=%u nal=%u avl=%u ded=%u\n",
- atomic_read(&fscache_n_object_alloc),
- atomic_read(&fscache_n_object_no_alloc),
- atomic_read(&fscache_n_object_avail),
- atomic_read(&fscache_n_object_dead));
+ atomic_read_unchecked(&fscache_n_object_alloc),
+ atomic_read_unchecked(&fscache_n_object_no_alloc),
+ atomic_read_unchecked(&fscache_n_object_avail),
+ atomic_read_unchecked(&fscache_n_object_dead));
seq_printf(m, "ChkAux : non=%u ok=%u upd=%u obs=%u\n",
- atomic_read(&fscache_n_checkaux_none),
- atomic_read(&fscache_n_checkaux_okay),
- atomic_read(&fscache_n_checkaux_update),
- atomic_read(&fscache_n_checkaux_obsolete));
+ atomic_read_unchecked(&fscache_n_checkaux_none),
+ atomic_read_unchecked(&fscache_n_checkaux_okay),
+ atomic_read_unchecked(&fscache_n_checkaux_update),
+ atomic_read_unchecked(&fscache_n_checkaux_obsolete));
seq_printf(m, "Pages : mrk=%u unc=%u\n",
- atomic_read(&fscache_n_marks),
- atomic_read(&fscache_n_uncaches));
+ atomic_read_unchecked(&fscache_n_marks),
+ atomic_read_unchecked(&fscache_n_uncaches));
seq_printf(m, "Acquire: n=%u nul=%u noc=%u ok=%u nbf=%u"
" oom=%u\n",
- atomic_read(&fscache_n_acquires),
- atomic_read(&fscache_n_acquires_null),
- atomic_read(&fscache_n_acquires_no_cache),
- atomic_read(&fscache_n_acquires_ok),
- atomic_read(&fscache_n_acquires_nobufs),
- atomic_read(&fscache_n_acquires_oom));
+ atomic_read_unchecked(&fscache_n_acquires),
+ atomic_read_unchecked(&fscache_n_acquires_null),
+ atomic_read_unchecked(&fscache_n_acquires_no_cache),
+ atomic_read_unchecked(&fscache_n_acquires_ok),
+ atomic_read_unchecked(&fscache_n_acquires_nobufs),
+ atomic_read_unchecked(&fscache_n_acquires_oom));
seq_printf(m, "Lookups: n=%u neg=%u pos=%u crt=%u tmo=%u\n",
- atomic_read(&fscache_n_object_lookups),
- atomic_read(&fscache_n_object_lookups_negative),
- atomic_read(&fscache_n_object_lookups_positive),
- atomic_read(&fscache_n_object_created),
- atomic_read(&fscache_n_object_lookups_timed_out));
+ atomic_read_unchecked(&fscache_n_object_lookups),
+ atomic_read_unchecked(&fscache_n_object_lookups_negative),
+ atomic_read_unchecked(&fscache_n_object_lookups_positive),
+ atomic_read_unchecked(&fscache_n_object_created),
+ atomic_read_unchecked(&fscache_n_object_lookups_timed_out));
seq_printf(m, "Invals : n=%u run=%u\n",
- atomic_read(&fscache_n_invalidates),
- atomic_read(&fscache_n_invalidates_run));
+ atomic_read_unchecked(&fscache_n_invalidates),
+ atomic_read_unchecked(&fscache_n_invalidates_run));
seq_printf(m, "Updates: n=%u nul=%u run=%u\n",
- atomic_read(&fscache_n_updates),
- atomic_read(&fscache_n_updates_null),
- atomic_read(&fscache_n_updates_run));
+ atomic_read_unchecked(&fscache_n_updates),
+ atomic_read_unchecked(&fscache_n_updates_null),
+ atomic_read_unchecked(&fscache_n_updates_run));
seq_printf(m, "Relinqs: n=%u nul=%u wcr=%u rtr=%u\n",
- atomic_read(&fscache_n_relinquishes),
- atomic_read(&fscache_n_relinquishes_null),
- atomic_read(&fscache_n_relinquishes_waitcrt),
- atomic_read(&fscache_n_relinquishes_retire));
+ atomic_read_unchecked(&fscache_n_relinquishes),
+ atomic_read_unchecked(&fscache_n_relinquishes_null),
+ atomic_read_unchecked(&fscache_n_relinquishes_waitcrt),
+ atomic_read_unchecked(&fscache_n_relinquishes_retire));
seq_printf(m, "AttrChg: n=%u ok=%u nbf=%u oom=%u run=%u\n",
- atomic_read(&fscache_n_attr_changed),
- atomic_read(&fscache_n_attr_changed_ok),
- atomic_read(&fscache_n_attr_changed_nobufs),
- atomic_read(&fscache_n_attr_changed_nomem),
- atomic_read(&fscache_n_attr_changed_calls));
+ atomic_read_unchecked(&fscache_n_attr_changed),
+ atomic_read_unchecked(&fscache_n_attr_changed_ok),
+ atomic_read_unchecked(&fscache_n_attr_changed_nobufs),
+ atomic_read_unchecked(&fscache_n_attr_changed_nomem),
+ atomic_read_unchecked(&fscache_n_attr_changed_calls));
seq_printf(m, "Allocs : n=%u ok=%u wt=%u nbf=%u int=%u\n",
- atomic_read(&fscache_n_allocs),
- atomic_read(&fscache_n_allocs_ok),
- atomic_read(&fscache_n_allocs_wait),
- atomic_read(&fscache_n_allocs_nobufs),
- atomic_read(&fscache_n_allocs_intr));
+ atomic_read_unchecked(&fscache_n_allocs),
+ atomic_read_unchecked(&fscache_n_allocs_ok),
+ atomic_read_unchecked(&fscache_n_allocs_wait),
+ atomic_read_unchecked(&fscache_n_allocs_nobufs),
+ atomic_read_unchecked(&fscache_n_allocs_intr));
seq_printf(m, "Allocs : ops=%u owt=%u abt=%u\n",
- atomic_read(&fscache_n_alloc_ops),
- atomic_read(&fscache_n_alloc_op_waits),
- atomic_read(&fscache_n_allocs_object_dead));
+ atomic_read_unchecked(&fscache_n_alloc_ops),
+ atomic_read_unchecked(&fscache_n_alloc_op_waits),
+ atomic_read_unchecked(&fscache_n_allocs_object_dead));
seq_printf(m, "Retrvls: n=%u ok=%u wt=%u nod=%u nbf=%u"
" int=%u oom=%u\n",
- atomic_read(&fscache_n_retrievals),
- atomic_read(&fscache_n_retrievals_ok),
- atomic_read(&fscache_n_retrievals_wait),
- atomic_read(&fscache_n_retrievals_nodata),
- atomic_read(&fscache_n_retrievals_nobufs),
- atomic_read(&fscache_n_retrievals_intr),
- atomic_read(&fscache_n_retrievals_nomem));
+ atomic_read_unchecked(&fscache_n_retrievals),
+ atomic_read_unchecked(&fscache_n_retrievals_ok),
+ atomic_read_unchecked(&fscache_n_retrievals_wait),
+ atomic_read_unchecked(&fscache_n_retrievals_nodata),
+ atomic_read_unchecked(&fscache_n_retrievals_nobufs),
+ atomic_read_unchecked(&fscache_n_retrievals_intr),
+ atomic_read_unchecked(&fscache_n_retrievals_nomem));
seq_printf(m, "Retrvls: ops=%u owt=%u abt=%u\n",
- atomic_read(&fscache_n_retrieval_ops),
- atomic_read(&fscache_n_retrieval_op_waits),
- atomic_read(&fscache_n_retrievals_object_dead));
+ atomic_read_unchecked(&fscache_n_retrieval_ops),
+ atomic_read_unchecked(&fscache_n_retrieval_op_waits),
+ atomic_read_unchecked(&fscache_n_retrievals_object_dead));
seq_printf(m, "Stores : n=%u ok=%u agn=%u nbf=%u oom=%u\n",
- atomic_read(&fscache_n_stores),
- atomic_read(&fscache_n_stores_ok),
- atomic_read(&fscache_n_stores_again),
- atomic_read(&fscache_n_stores_nobufs),
- atomic_read(&fscache_n_stores_oom));
+ atomic_read_unchecked(&fscache_n_stores),
+ atomic_read_unchecked(&fscache_n_stores_ok),
+ atomic_read_unchecked(&fscache_n_stores_again),
+ atomic_read_unchecked(&fscache_n_stores_nobufs),
+ atomic_read_unchecked(&fscache_n_stores_oom));
seq_printf(m, "Stores : ops=%u run=%u pgs=%u rxd=%u olm=%u\n",
- atomic_read(&fscache_n_store_ops),
- atomic_read(&fscache_n_store_calls),
- atomic_read(&fscache_n_store_pages),
- atomic_read(&fscache_n_store_radix_deletes),
- atomic_read(&fscache_n_store_pages_over_limit));
+ atomic_read_unchecked(&fscache_n_store_ops),
+ atomic_read_unchecked(&fscache_n_store_calls),
+ atomic_read_unchecked(&fscache_n_store_pages),
+ atomic_read_unchecked(&fscache_n_store_radix_deletes),
+ atomic_read_unchecked(&fscache_n_store_pages_over_limit));
seq_printf(m, "VmScan : nos=%u gon=%u bsy=%u can=%u wt=%u\n",
- atomic_read(&fscache_n_store_vmscan_not_storing),
- atomic_read(&fscache_n_store_vmscan_gone),
- atomic_read(&fscache_n_store_vmscan_busy),
- atomic_read(&fscache_n_store_vmscan_cancelled),
- atomic_read(&fscache_n_store_vmscan_wait));
+ atomic_read_unchecked(&fscache_n_store_vmscan_not_storing),
+ atomic_read_unchecked(&fscache_n_store_vmscan_gone),
+ atomic_read_unchecked(&fscache_n_store_vmscan_busy),
+ atomic_read_unchecked(&fscache_n_store_vmscan_cancelled),
+ atomic_read_unchecked(&fscache_n_store_vmscan_wait));
seq_printf(m, "Ops : pend=%u run=%u enq=%u can=%u rej=%u\n",
- atomic_read(&fscache_n_op_pend),
- atomic_read(&fscache_n_op_run),
- atomic_read(&fscache_n_op_enqueue),
- atomic_read(&fscache_n_op_cancelled),
- atomic_read(&fscache_n_op_rejected));
+ atomic_read_unchecked(&fscache_n_op_pend),
+ atomic_read_unchecked(&fscache_n_op_run),
+ atomic_read_unchecked(&fscache_n_op_enqueue),
+ atomic_read_unchecked(&fscache_n_op_cancelled),
+ atomic_read_unchecked(&fscache_n_op_rejected));
seq_printf(m, "Ops : ini=%u dfr=%u rel=%u gc=%u\n",
- atomic_read(&fscache_n_op_initialised),
- atomic_read(&fscache_n_op_deferred_release),
- atomic_read(&fscache_n_op_release),
- atomic_read(&fscache_n_op_gc));
+ atomic_read_unchecked(&fscache_n_op_initialised),
+ atomic_read_unchecked(&fscache_n_op_deferred_release),
+ atomic_read_unchecked(&fscache_n_op_release),
+ atomic_read_unchecked(&fscache_n_op_gc));
seq_printf(m, "CacheOp: alo=%d luo=%d luc=%d gro=%d\n",
atomic_read(&fscache_n_cop_alloc_object),
diff --git a/fs/inode.c b/fs/inode.c
index d30640f..df4be97 100644
--- a/fs/inode.c
+++ b/fs/inode.c
@@ -834,8 +834,8 @@ unsigned int get_next_ino(void)
#ifdef CONFIG_SMP
if (unlikely((res & (LAST_INO_BATCH-1)) == 0)) {
- static atomic_t shared_last_ino;
- int next = atomic_add_return(LAST_INO_BATCH, &shared_last_ino);
+ static atomic_unchecked_t shared_last_ino;
+ int next = atomic_add_return_unchecked(LAST_INO_BATCH, &shared_last_ino);
res = next - LAST_INO_BATCH;
}
diff --git a/fs/kernfs/file.c b/fs/kernfs/file.c
index 7247252..9c58009 100644
--- a/fs/kernfs/file.c
+++ b/fs/kernfs/file.c
@@ -34,7 +34,7 @@ static DEFINE_MUTEX(kernfs_open_file_mutex);
struct kernfs_open_node {
atomic_t refcnt;
- atomic_t event;
+ atomic_unchecked_t event;
wait_queue_head_t poll;
struct list_head files; /* goes through kernfs_open_file.list */
};
@@ -163,7 +163,7 @@ static int kernfs_seq_show(struct seq_file *sf, void *v)
{
struct kernfs_open_file *of = sf->private;
- of->event = atomic_read(&of->kn->attr.open->event);
+ of->event = atomic_read_unchecked(&of->kn->attr.open->event);
return of->kn->attr.ops->seq_show(sf, v);
}
@@ -207,7 +207,7 @@ static ssize_t kernfs_file_direct_read(struct kernfs_open_file *of,
goto out_free;
}
- of->event = atomic_read(&of->kn->attr.open->event);
+ of->event = atomic_read_unchecked(&of->kn->attr.open->event);
ops = kernfs_ops(of->kn);
if (ops->read)
len = ops->read(of, buf, len, *ppos);
@@ -569,7 +569,7 @@ static int kernfs_get_open_node(struct kernfs_node *kn,
return -ENOMEM;
atomic_set(&new_on->refcnt, 0);
- atomic_set(&new_on->event, 1);
+ atomic_set_unchecked(&new_on->event, 1);
init_waitqueue_head(&new_on->poll);
INIT_LIST_HEAD(&new_on->files);
goto retry;
@@ -792,7 +792,7 @@ static unsigned int kernfs_fop_poll(struct file *filp, poll_table *wait)
kernfs_put_active(kn);
- if (of->event != atomic_read(&on->event))
+ if (of->event != atomic_read_unchecked(&on->event))
goto trigger;
return DEFAULT_POLLMASK;
@@ -823,7 +823,7 @@ repeat:
on = kn->attr.open;
if (on) {
- atomic_inc(&on->event);
+ atomic_inc_unchecked(&on->event);
wake_up_interruptible(&on->poll);
}
diff --git a/fs/lockd/clntproc.c b/fs/lockd/clntproc.c
index acd3947..1f896e2 100644
--- a/fs/lockd/clntproc.c
+++ b/fs/lockd/clntproc.c
@@ -36,11 +36,11 @@ static const struct rpc_call_ops nlmclnt_cancel_ops;
/*
* Cookie counter for NLM requests
*/
-static atomic_t nlm_cookie = ATOMIC_INIT(0x1234);
+static atomic_unchecked_t nlm_cookie = ATOMIC_INIT(0x1234);
void nlmclnt_next_cookie(struct nlm_cookie *c)
{
- u32 cookie = atomic_inc_return(&nlm_cookie);
+ u32 cookie = atomic_inc_return_unchecked(&nlm_cookie);
memcpy(c->data, &cookie, 4);
c->len=4;
diff --git a/fs/namespace.c b/fs/namespace.c
index 2b8aa15..4e94d1a 100644
--- a/fs/namespace.c
+++ b/fs/namespace.c
@@ -2743,7 +2743,7 @@ static void free_mnt_ns(struct mnt_namespace *ns)
* number incrementing at 10Ghz will take 12,427 years to wrap which
* is effectively never, so we can ignore the possibility.
*/
-static atomic64_t mnt_ns_seq = ATOMIC64_INIT(1);
+static atomic64_unchecked_t mnt_ns_seq = ATOMIC64_INIT(1);
static struct mnt_namespace *alloc_mnt_ns(struct user_namespace *user_ns)
{
@@ -2759,7 +2759,7 @@ static struct mnt_namespace *alloc_mnt_ns(struct user_namespace *user_ns)
return ERR_PTR(ret);
}
new_ns->ns.ops = &mntns_operations;
- new_ns->seq = atomic64_add_return(1, &mnt_ns_seq);
+ new_ns->seq = atomic64_add_return_unchecked(1, &mnt_ns_seq);
atomic_set(&new_ns->count, 1);
new_ns->root = NULL;
INIT_LIST_HEAD(&new_ns->list);
diff --git a/fs/nfs/inode.c b/fs/nfs/inode.c
index 4afbe13..a6a26ce 100644
--- a/fs/nfs/inode.c
+++ b/fs/nfs/inode.c
@@ -1273,16 +1273,16 @@ static int nfs_check_inode_attributes(struct inode *inode, struct nfs_fattr *fat
return 0;
}
-static atomic_long_t nfs_attr_generation_counter;
+static atomic_long_unchecked_t nfs_attr_generation_counter;
static unsigned long nfs_read_attr_generation_counter(void)
{
- return atomic_long_read(&nfs_attr_generation_counter);
+ return atomic_long_read_unchecked(&nfs_attr_generation_counter);
}
unsigned long nfs_inc_attr_generation_counter(void)
{
- return atomic_long_inc_return(&nfs_attr_generation_counter);
+ return atomic_long_inc_return_unchecked(&nfs_attr_generation_counter);
}
EXPORT_SYMBOL_GPL(nfs_inc_attr_generation_counter);
diff --git a/fs/notify/notification.c b/fs/notify/notification.c
index a95d8e0..a91a5fd 100644
--- a/fs/notify/notification.c
+++ b/fs/notify/notification.c
@@ -48,7 +48,7 @@
#include <linux/fsnotify_backend.h>
#include "fsnotify.h"
-static atomic_t fsnotify_sync_cookie = ATOMIC_INIT(0);
+static atomic_unchecked_t fsnotify_sync_cookie = ATOMIC_INIT(0);
/**
* fsnotify_get_cookie - return a unique cookie for use in synchronizing events.
@@ -56,7 +56,7 @@ static atomic_t fsnotify_sync_cookie = ATOMIC_INIT(0);
*/
u32 fsnotify_get_cookie(void)
{
- return atomic_inc_return(&fsnotify_sync_cookie);
+ return atomic_inc_return_unchecked(&fsnotify_sync_cookie);
}
EXPORT_SYMBOL_GPL(fsnotify_get_cookie);
diff --git a/fs/ocfs2/localalloc.c b/fs/ocfs2/localalloc.c
index 857bbbc..3c47d15 100644
--- a/fs/ocfs2/localalloc.c
+++ b/fs/ocfs2/localalloc.c
@@ -1320,7 +1320,7 @@ static int ocfs2_local_alloc_slide_window(struct ocfs2_super *osb,
goto bail;
}
- atomic_inc(&osb->alloc_stats.moves);
+ atomic_inc_unchecked(&osb->alloc_stats.moves);
bail:
if (handle)
diff --git a/fs/ocfs2/ocfs2.h b/fs/ocfs2/ocfs2.h
index 690ddc6..f2d4c4d 100644
--- a/fs/ocfs2/ocfs2.h
+++ b/fs/ocfs2/ocfs2.h
@@ -247,11 +247,11 @@ enum ocfs2_vol_state
struct ocfs2_alloc_stats
{
- atomic_t moves;
- atomic_t local_data;
- atomic_t bitmap_data;
- atomic_t bg_allocs;
- atomic_t bg_extends;
+ atomic_unchecked_t moves;
+ atomic_unchecked_t local_data;
+ atomic_unchecked_t bitmap_data;
+ atomic_unchecked_t bg_allocs;
+ atomic_unchecked_t bg_extends;
};
enum ocfs2_local_alloc_state
diff --git a/fs/ocfs2/suballoc.c b/fs/ocfs2/suballoc.c
index 4479029..5de740b 100644
--- a/fs/ocfs2/suballoc.c
+++ b/fs/ocfs2/suballoc.c
@@ -867,7 +867,7 @@ static int ocfs2_reserve_suballoc_bits(struct ocfs2_super *osb,
mlog_errno(status);
goto bail;
}
- atomic_inc(&osb->alloc_stats.bg_extends);
+ atomic_inc_unchecked(&osb->alloc_stats.bg_extends);
/* You should never ask for this much metadata */
BUG_ON(bits_wanted >
@@ -2014,7 +2014,7 @@ int ocfs2_claim_metadata(handle_t *handle,
mlog_errno(status);
goto bail;
}
- atomic_inc(&OCFS2_SB(ac->ac_inode->i_sb)->alloc_stats.bg_allocs);
+ atomic_inc_unchecked(&OCFS2_SB(ac->ac_inode->i_sb)->alloc_stats.bg_allocs);
*suballoc_loc = res.sr_bg_blkno;
*suballoc_bit_start = res.sr_bit_offset;
@@ -2180,7 +2180,7 @@ int ocfs2_claim_new_inode_at_loc(handle_t *handle,
trace_ocfs2_claim_new_inode_at_loc((unsigned long long)di_blkno,
res->sr_bits);
- atomic_inc(&OCFS2_SB(ac->ac_inode->i_sb)->alloc_stats.bg_allocs);
+ atomic_inc_unchecked(&OCFS2_SB(ac->ac_inode->i_sb)->alloc_stats.bg_allocs);
BUG_ON(res->sr_bits != 1);
@@ -2222,7 +2222,7 @@ int ocfs2_claim_new_inode(handle_t *handle,
mlog_errno(status);
goto bail;
}
- atomic_inc(&OCFS2_SB(ac->ac_inode->i_sb)->alloc_stats.bg_allocs);
+ atomic_inc_unchecked(&OCFS2_SB(ac->ac_inode->i_sb)->alloc_stats.bg_allocs);
BUG_ON(res.sr_bits != 1);
@@ -2326,7 +2326,7 @@ int __ocfs2_claim_clusters(handle_t *handle,
cluster_start,
num_clusters);
if (!status)
- atomic_inc(&osb->alloc_stats.local_data);
+ atomic_inc_unchecked(&osb->alloc_stats.local_data);
} else {
if (min_clusters > (osb->bitmap_cpg - 1)) {
/* The only paths asking for contiguousness
@@ -2352,7 +2352,7 @@ int __ocfs2_claim_clusters(handle_t *handle,
ocfs2_desc_bitmap_to_cluster_off(ac->ac_inode,
res.sr_bg_blkno,
res.sr_bit_offset);
- atomic_inc(&osb->alloc_stats.bitmap_data);
+ atomic_inc_unchecked(&osb->alloc_stats.bitmap_data);
*num_clusters = res.sr_bits;
}
}
diff --git a/fs/ocfs2/super.c b/fs/ocfs2/super.c
index a482e31..81b251d 100644
--- a/fs/ocfs2/super.c
+++ b/fs/ocfs2/super.c
@@ -308,11 +308,11 @@ static int ocfs2_osb_dump(struct ocfs2_super *osb, char *buf, int len)
"%10s => GlobalAllocs: %d LocalAllocs: %d "
"SubAllocs: %d LAWinMoves: %d SAExtends: %d\n",
"Stats",
- atomic_read(&osb->alloc_stats.bitmap_data),
- atomic_read(&osb->alloc_stats.local_data),
- atomic_read(&osb->alloc_stats.bg_allocs),
- atomic_read(&osb->alloc_stats.moves),
- atomic_read(&osb->alloc_stats.bg_extends));
+ atomic_read_unchecked(&osb->alloc_stats.bitmap_data),
+ atomic_read_unchecked(&osb->alloc_stats.local_data),
+ atomic_read_unchecked(&osb->alloc_stats.bg_allocs),
+ atomic_read_unchecked(&osb->alloc_stats.moves),
+ atomic_read_unchecked(&osb->alloc_stats.bg_extends));
out += snprintf(buf + out, len - out,
"%10s => State: %u Descriptor: %llu Size: %u bits "
@@ -2095,11 +2095,11 @@ static int ocfs2_initialize_super(struct super_block *sb,
mutex_init(&osb->system_file_mutex);
- atomic_set(&osb->alloc_stats.moves, 0);
- atomic_set(&osb->alloc_stats.local_data, 0);
- atomic_set(&osb->alloc_stats.bitmap_data, 0);
- atomic_set(&osb->alloc_stats.bg_allocs, 0);
- atomic_set(&osb->alloc_stats.bg_extends, 0);
+ atomic_set_unchecked(&osb->alloc_stats.moves, 0);
+ atomic_set_unchecked(&osb->alloc_stats.local_data, 0);
+ atomic_set_unchecked(&osb->alloc_stats.bitmap_data, 0);
+ atomic_set_unchecked(&osb->alloc_stats.bg_allocs, 0);
+ atomic_set_unchecked(&osb->alloc_stats.bg_extends, 0);
/* Copy the blockcheck stats from the superblock probe */
osb->osb_ecc_stats = *stats;
diff --git a/fs/proc/meminfo.c b/fs/proc/meminfo.c
index d3ebf2e..6ad42d1 100644
--- a/fs/proc/meminfo.c
+++ b/fs/proc/meminfo.c
@@ -194,7 +194,7 @@ static int meminfo_proc_show(struct seq_file *m, void *v)
vmi.used >> 10,
vmi.largest_chunk >> 10
#ifdef CONFIG_MEMORY_FAILURE
- , atomic_long_read(&num_poisoned_pages) << (PAGE_SHIFT - 10)
+ , atomic_long_read_unchecked(&num_poisoned_pages) << (PAGE_SHIFT - 10)
#endif
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
, K(global_page_state(NR_ANON_TRANSPARENT_HUGEPAGES) *
diff --git a/fs/quota/netlink.c b/fs/quota/netlink.c
index bb2869f..d34ada8 100644
--- a/fs/quota/netlink.c
+++ b/fs/quota/netlink.c
@@ -44,7 +44,7 @@ static struct genl_family quota_genl_family = {
void quota_send_warning(struct kqid qid, dev_t dev,
const char warntype)
{
- static atomic_t seq;
+ static atomic_unchecked_t seq;
struct sk_buff *skb;
void *msg_head;
int ret;
@@ -60,7 +60,7 @@ void quota_send_warning(struct kqid qid, dev_t dev,
"VFS: Not enough memory to send quota warning.\n");
return;
}
- msg_head = genlmsg_put(skb, 0, atomic_add_return(1, &seq),
+ msg_head = genlmsg_put(skb, 0, atomic_add_return_unchecked(1, &seq),
"a_genl_family, 0, QUOTA_NL_C_WARNING);
if (!msg_head) {
printk(KERN_ERR
diff --git a/fs/reiserfs/do_balan.c b/fs/reiserfs/do_balan.c
index 9c02d96..6562c10 100644
--- a/fs/reiserfs/do_balan.c
+++ b/fs/reiserfs/do_balan.c
@@ -1887,7 +1887,7 @@ void do_balance(struct tree_balance *tb, struct item_head *ih,
return;
}
- atomic_inc(&fs_generation(tb->tb_sb));
+ atomic_inc_unchecked(&fs_generation(tb->tb_sb));
do_balance_starts(tb);
/*
diff --git a/fs/reiserfs/procfs.c b/fs/reiserfs/procfs.c
index 621b9f3..af527fd 100644
--- a/fs/reiserfs/procfs.c
+++ b/fs/reiserfs/procfs.c
@@ -114,7 +114,7 @@ static int show_super(struct seq_file *m, void *unused)
"SMALL_TAILS " : "NO_TAILS ",
replay_only(sb) ? "REPLAY_ONLY " : "",
convert_reiserfs(sb) ? "CONV " : "",
- atomic_read(&r->s_generation_counter),
+ atomic_read_unchecked(&r->s_generation_counter),
SF(s_disk_reads), SF(s_disk_writes), SF(s_fix_nodes),
SF(s_do_balance), SF(s_unneeded_left_neighbor),
SF(s_good_search_by_key_reada), SF(s_bmaps),
diff --git a/fs/reiserfs/reiserfs.h b/fs/reiserfs/reiserfs.h
index 2adcde1..7d27bc8 100644
--- a/fs/reiserfs/reiserfs.h
+++ b/fs/reiserfs/reiserfs.h
@@ -580,7 +580,7 @@ struct reiserfs_sb_info {
/* Comment? -Hans */
wait_queue_head_t s_wait;
/* increased by one every time the tree gets re-balanced */
- atomic_t s_generation_counter;
+ atomic_unchecked_t s_generation_counter;
/* File system properties. Currently holds on-disk FS format */
unsigned long s_properties;
@@ -2300,7 +2300,7 @@ static inline loff_t max_reiserfs_offset(struct inode *inode)
#define REISERFS_USER_MEM 1 /* user memory mode */
#define fs_generation(s) (REISERFS_SB(s)->s_generation_counter)
-#define get_generation(s) atomic_read (&fs_generation(s))
+#define get_generation(s) atomic_read_unchecked (&fs_generation(s))
#define FILESYSTEM_CHANGED_TB(tb) (get_generation((tb)->tb_sb) != (tb)->fs_gen)
#define __fs_changed(gen,s) (gen != get_generation (s))
#define fs_changed(gen,s) \
diff --git a/include/linux/fscache-cache.h b/include/linux/fscache-cache.h
index 604e152..5954d0d 100644
--- a/include/linux/fscache-cache.h
+++ b/include/linux/fscache-cache.h
@@ -117,7 +117,7 @@ struct fscache_operation {
fscache_operation_release_t release;
};
-extern atomic_t fscache_op_debug_id;
+extern atomic_unchecked_t fscache_op_debug_id;
extern void fscache_op_work_func(struct work_struct *work);
extern void fscache_enqueue_operation(struct fscache_operation *);
--
2.5.0
^ permalink raw reply related [flat|nested] 32+ messages in thread* [kernel-hardening][RFC PATCH v2 06/12] net: opt out of PAX_REFCOUNT protection
2015-12-17 14:57 [kernel-hardening][RFC PATCH v2 00/12] Add PAX_REFCOUNT overflow protection David Windsor
` (4 preceding siblings ...)
2015-12-17 14:57 ` [kernel-hardening][RFC PATCH v2 05/12] fs: " David Windsor
@ 2015-12-17 14:57 ` David Windsor
2015-12-17 14:57 ` [kernel-hardening][RFC PATCH v2 07/12] security: " David Windsor
` (6 subsequent siblings)
12 siblings, 0 replies; 32+ messages in thread
From: David Windsor @ 2015-12-17 14:57 UTC (permalink / raw)
To: kernel-hardening; +Cc: David Windsor
Signed-off-by: David Windsor <dave@progbits.org>
---
include/linux/atmdev.h | 2 +-
include/linux/netdevice.h | 6 ++---
include/linux/sonet.h | 2 +-
include/linux/sunrpc/svc_rdma.h | 18 +++++++--------
include/net/bonding.h | 2 +-
include/net/caif/cfctrl.h | 4 ++--
include/net/flow.h | 2 +-
include/net/gro_cells.h | 2 +-
include/net/inetpeer.h | 2 +-
include/net/ip_fib.h | 2 +-
include/net/ip_vs.h | 4 ++--
include/net/iucv/af_iucv.h | 2 +-
include/net/net_namespace.h | 12 +++++-----
include/net/netns/ipv4.h | 4 ++--
include/net/netns/ipv6.h | 4 ++--
include/net/netns/xfrm.h | 2 +-
include/net/sock.h | 6 ++---
include/net/xfrm.h | 2 +-
net/atm/atm_misc.c | 8 +++----
net/atm/proc.c | 6 ++---
net/atm/resources.c | 4 ++--
net/batman-adv/bat_iv_ogm.c | 8 +++----
net/batman-adv/fragmentation.c | 2 +-
net/batman-adv/soft-interface.c | 6 ++---
net/batman-adv/types.h | 6 ++---
net/caif/cfctrl.c | 10 ++++-----
net/ceph/messenger.c | 4 ++--
net/core/datagram.c | 2 +-
net/core/dev.c | 12 +++++-----
net/core/flow.c | 6 ++---
net/core/net-sysfs.c | 2 +-
net/core/netpoll.c | 4 ++--
net/core/rtnetlink.c | 2 +-
net/core/sock.c | 12 +++++-----
net/core/sock_diag.c | 6 ++---
net/ipv4/devinet.c | 4 ++--
net/ipv4/fib_frontend.c | 6 ++---
net/ipv4/fib_semantics.c | 2 +-
net/ipv4/inet_connection_sock.c | 4 ++--
net/ipv4/inet_timewait_sock.c | 2 +-
net/ipv4/inetpeer.c | 2 +-
net/ipv4/ip_fragment.c | 2 +-
net/ipv4/ping.c | 2 +-
net/ipv4/raw.c | 4 ++--
net/ipv4/route.c | 10 ++++-----
net/ipv4/tcp_input.c | 2 +-
net/ipv4/udp.c | 10 ++++-----
net/ipv6/addrconf.c | 6 ++---
net/ipv6/af_inet6.c | 2 +-
net/ipv6/datagram.c | 2 +-
net/ipv6/ip6_fib.c | 4 ++--
net/ipv6/raw.c | 6 ++---
net/ipv6/udp.c | 6 ++---
net/iucv/af_iucv.c | 4 ++--
net/key/af_key.c | 4 ++--
net/l2tp/l2tp_eth.c | 38 ++++++++++++++++----------------
net/netfilter/ipvs/ip_vs_conn.c | 6 ++---
net/netfilter/ipvs/ip_vs_core.c | 4 ++--
net/netfilter/ipvs/ip_vs_ctl.c | 10 ++++-----
net/netfilter/ipvs/ip_vs_sync.c | 6 ++---
net/netfilter/ipvs/ip_vs_xmit.c | 4 ++--
net/netfilter/nfnetlink_log.c | 4 ++--
net/netfilter/xt_statistic.c | 8 +++----
net/netlink/af_netlink.c | 4 ++--
net/openvswitch/vport.c | 16 +++++++-------
net/openvswitch/vport.h | 8 +++----
net/packet/af_packet.c | 4 ++--
net/phonet/pep.c | 6 ++---
net/phonet/socket.c | 2 +-
net/rds/cong.c | 6 ++---
net/rds/ib.h | 2 +-
net/rds/ib_cm.c | 2 +-
net/rds/ib_recv.c | 4 ++--
net/rds/iw.h | 2 +-
net/rds/iw_cm.c | 2 +-
net/rds/iw_recv.c | 4 ++--
net/rxrpc/af_rxrpc.c | 2 +-
net/rxrpc/ar-ack.c | 14 ++++++------
net/rxrpc/ar-call.c | 2 +-
net/rxrpc/ar-connection.c | 2 +-
net/rxrpc/ar-connevent.c | 2 +-
net/rxrpc/ar-input.c | 4 ++--
net/rxrpc/ar-internal.h | 8 +++----
net/rxrpc/ar-local.c | 2 +-
net/rxrpc/ar-output.c | 4 ++--
net/rxrpc/ar-peer.c | 2 +-
net/rxrpc/ar-proc.c | 4 ++--
net/rxrpc/ar-transport.c | 2 +-
net/rxrpc/rxkad.c | 4 ++--
net/sched/sch_generic.c | 4 ++--
net/sunrpc/auth_gss/svcauth_gss.c | 4 ++--
net/sunrpc/sched.c | 4 ++--
net/sunrpc/xprtrdma/svc_rdma.c | 36 +++++++++++++++---------------
net/sunrpc/xprtrdma/svc_rdma_recvfrom.c | 8 +++----
net/sunrpc/xprtrdma/svc_rdma_sendto.c | 2 +-
net/sunrpc/xprtrdma/svc_rdma_transport.c | 10 ++++-----
net/xfrm/xfrm_policy.c | 10 ++++-----
net/xfrm/xfrm_state.c | 4 ++--
98 files changed, 269 insertions(+), 269 deletions(-)
diff --git a/include/linux/atmdev.h b/include/linux/atmdev.h
index c1da539..4db35ec 100644
--- a/include/linux/atmdev.h
+++ b/include/linux/atmdev.h
@@ -28,7 +28,7 @@ struct compat_atm_iobuf {
#endif
struct k_atm_aal_stats {
-#define __HANDLE_ITEM(i) atomic_t i
+#define __HANDLE_ITEM(i) atomic_unchecked_t i
__AAL_STAT_ITEMS
#undef __HANDLE_ITEM
};
diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index e20979d..d57cabd 100644
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -1519,7 +1519,7 @@ struct net_device {
unsigned long base_addr;
int irq;
- atomic_t carrier_changes;
+ atomic_unchecked_t carrier_changes;
/*
* Some hardware also needs these fields (state,dev_list,
@@ -1558,8 +1558,8 @@ struct net_device {
struct net_device_stats stats;
- atomic_long_t rx_dropped;
- atomic_long_t tx_dropped;
+ atomic_long_unchecked_t rx_dropped;
+ atomic_long_unchecked_t tx_dropped;
#ifdef CONFIG_WIRELESS_EXT
const struct iw_handler_def * wireless_handlers;
diff --git a/include/linux/sonet.h b/include/linux/sonet.h
index 680f9a3..f13aeb0 100644
--- a/include/linux/sonet.h
+++ b/include/linux/sonet.h
@@ -7,7 +7,7 @@
#include <uapi/linux/sonet.h>
struct k_sonet_stats {
-#define __HANDLE_ITEM(i) atomic_t i
+#define __HANDLE_ITEM(i) atomic_unchecked_t i
__SONET_ITEMS
#undef __HANDLE_ITEM
};
diff --git a/include/linux/sunrpc/svc_rdma.h b/include/linux/sunrpc/svc_rdma.h
index 4929a8a..b8f29e9 100644
--- a/include/linux/sunrpc/svc_rdma.h
+++ b/include/linux/sunrpc/svc_rdma.h
@@ -53,15 +53,15 @@ extern unsigned int svcrdma_ord;
extern unsigned int svcrdma_max_requests;
extern unsigned int svcrdma_max_req_size;
-extern atomic_t rdma_stat_recv;
-extern atomic_t rdma_stat_read;
-extern atomic_t rdma_stat_write;
-extern atomic_t rdma_stat_sq_starve;
-extern atomic_t rdma_stat_rq_starve;
-extern atomic_t rdma_stat_rq_poll;
-extern atomic_t rdma_stat_rq_prod;
-extern atomic_t rdma_stat_sq_poll;
-extern atomic_t rdma_stat_sq_prod;
+extern atomic_unchecked_t rdma_stat_recv;
+extern atomic_unchecked_t rdma_stat_read;
+extern atomic_unchecked_t rdma_stat_write;
+extern atomic_unchecked_t rdma_stat_sq_starve;
+extern atomic_unchecked_t rdma_stat_rq_starve;
+extern atomic_unchecked_t rdma_stat_rq_poll;
+extern atomic_unchecked_t rdma_stat_rq_prod;
+extern atomic_unchecked_t rdma_stat_sq_poll;
+extern atomic_unchecked_t rdma_stat_sq_prod;
/*
* Contexts are built when an RDMA request is created and are a
diff --git a/include/net/bonding.h b/include/net/bonding.h
index 20defc0..3072903 100644
--- a/include/net/bonding.h
+++ b/include/net/bonding.h
@@ -661,7 +661,7 @@ extern struct rtnl_link_ops bond_link_ops;
static inline void bond_tx_drop(struct net_device *dev, struct sk_buff *skb)
{
- atomic_long_inc(&dev->tx_dropped);
+ atomic_long_inc_unchecked(&dev->tx_dropped);
dev_kfree_skb_any(skb);
}
diff --git a/include/net/caif/cfctrl.h b/include/net/caif/cfctrl.h
index f2ae33d..57fcd31 100644
--- a/include/net/caif/cfctrl.h
+++ b/include/net/caif/cfctrl.h
@@ -101,8 +101,8 @@ struct cfctrl_request_info {
struct cfctrl {
struct cfsrvl serv;
struct cfctrl_rsp res;
- atomic_t req_seq_no;
- atomic_t rsp_seq_no;
+ atomic_unchecked_t req_seq_no;
+ atomic_unchecked_t rsp_seq_no;
struct list_head list;
/* Protects from simultaneous access to first_req list */
spinlock_t info_list_lock;
diff --git a/include/net/flow.h b/include/net/flow.h
index 8109a15..504466d 100644
--- a/include/net/flow.h
+++ b/include/net/flow.h
@@ -231,6 +231,6 @@ void flow_cache_fini(struct net *net);
void flow_cache_flush(struct net *net);
void flow_cache_flush_deferred(struct net *net);
-extern atomic_t flow_cache_genid;
+extern atomic_unchecked_t flow_cache_genid;
#endif
diff --git a/include/net/gro_cells.h b/include/net/gro_cells.h
index 0f712c0..cd762c4 100644
--- a/include/net/gro_cells.h
+++ b/include/net/gro_cells.h
@@ -27,7 +27,7 @@ static inline void gro_cells_receive(struct gro_cells *gcells, struct sk_buff *s
cell = this_cpu_ptr(gcells->cells);
if (skb_queue_len(&cell->napi_skbs) > netdev_max_backlog) {
- atomic_long_inc(&dev->rx_dropped);
+ atomic_long_inc_unchecked(&dev->rx_dropped);
kfree_skb(skb);
return;
}
diff --git a/include/net/inetpeer.h b/include/net/inetpeer.h
index d5332dd..10a5c3c 100644
--- a/include/net/inetpeer.h
+++ b/include/net/inetpeer.h
@@ -48,7 +48,7 @@ struct inet_peer {
*/
union {
struct {
- atomic_t rid; /* Frag reception counter */
+ atomic_unchecked_t rid; /* Frag reception counter */
};
struct rcu_head rcu;
struct inet_peer *gc_next;
diff --git a/include/net/ip_fib.h b/include/net/ip_fib.h
index 5fa643b..d871e20 100644
--- a/include/net/ip_fib.h
+++ b/include/net/ip_fib.h
@@ -170,7 +170,7 @@ __be32 fib_info_update_nh_saddr(struct net *net, struct fib_nh *nh);
#define FIB_RES_SADDR(net, res) \
((FIB_RES_NH(res).nh_saddr_genid == \
- atomic_read(&(net)->ipv4.dev_addr_genid)) ? \
+ atomic_read_unchecked(&(net)->ipv4.dev_addr_genid)) ? \
FIB_RES_NH(res).nh_saddr : \
fib_info_update_nh_saddr((net), &FIB_RES_NH(res)))
#define FIB_RES_GW(res) (FIB_RES_NH(res).nh_gw)
diff --git a/include/net/ip_vs.h b/include/net/ip_vs.h
index 4e3731e..9a0a688 100644
--- a/include/net/ip_vs.h
+++ b/include/net/ip_vs.h
@@ -551,7 +551,7 @@ struct ip_vs_conn {
struct ip_vs_conn *control; /* Master control connection */
atomic_t n_control; /* Number of controlled ones */
struct ip_vs_dest *dest; /* real server */
- atomic_t in_pkts; /* incoming packet counter */
+ atomic_unchecked_t in_pkts; /* incoming packet counter */
/* Packet transmitter for different forwarding methods. If it
* mangles the packet, it must return NF_DROP or better NF_STOLEN,
@@ -699,7 +699,7 @@ struct ip_vs_dest {
__be16 port; /* port number of the server */
union nf_inet_addr addr; /* IP address of the server */
volatile unsigned int flags; /* dest status flags */
- atomic_t conn_flags; /* flags to copy to conn */
+ atomic_unchecked_t conn_flags; /* flags to copy to conn */
atomic_t weight; /* server weight */
atomic_t refcnt; /* reference counter */
diff --git a/include/net/iucv/af_iucv.h b/include/net/iucv/af_iucv.h
index 714cc9a..ea05f3e 100644
--- a/include/net/iucv/af_iucv.h
+++ b/include/net/iucv/af_iucv.h
@@ -149,7 +149,7 @@ struct iucv_skb_cb {
struct iucv_sock_list {
struct hlist_head head;
rwlock_t lock;
- atomic_t autobind_name;
+ atomic_unchecked_t autobind_name;
};
unsigned int iucv_sock_poll(struct file *file, struct socket *sock,
diff --git a/include/net/net_namespace.h b/include/net/net_namespace.h
index e951453..8a28d2c 100644
--- a/include/net/net_namespace.h
+++ b/include/net/net_namespace.h
@@ -53,7 +53,7 @@ struct net {
*/
spinlock_t rules_mod_lock;
- atomic64_t cookie_gen;
+ atomic64_unchecked_t cookie_gen;
struct list_head list; /* list of network namespaces */
struct list_head cleanup_list; /* namespaces on death row */
@@ -135,7 +135,7 @@ struct net {
struct netns_mpls mpls;
#endif
struct sock *diag_nlsk;
- atomic_t fnhe_genid;
+ atomic_unchecked_t fnhe_genid;
};
#include <linux/seq_file_net.h>
@@ -334,12 +334,12 @@ static inline void unregister_net_sysctl_table(struct ctl_table_header *header)
static inline int rt_genid_ipv4(struct net *net)
{
- return atomic_read(&net->ipv4.rt_genid);
+ return atomic_read_unchecked(&net->ipv4.rt_genid);
}
static inline void rt_genid_bump_ipv4(struct net *net)
{
- atomic_inc(&net->ipv4.rt_genid);
+ atomic_inc_unchecked(&net->ipv4.rt_genid);
}
extern void (*__fib6_flush_trees)(struct net *net);
@@ -366,12 +366,12 @@ static inline void rt_genid_bump_all(struct net *net)
static inline int fnhe_genid(struct net *net)
{
- return atomic_read(&net->fnhe_genid);
+ return atomic_read_unchecked(&net->fnhe_genid);
}
static inline void fnhe_genid_bump(struct net *net)
{
- atomic_inc(&net->fnhe_genid);
+ atomic_inc_unchecked(&net->fnhe_genid);
}
#endif /* __NET_NET_NAMESPACE_H */
diff --git a/include/net/netns/ipv4.h b/include/net/netns/ipv4.h
index c68926b..106c147 100644
--- a/include/net/netns/ipv4.h
+++ b/include/net/netns/ipv4.h
@@ -93,7 +93,7 @@ struct netns_ipv4 {
struct ping_group_range ping_group_range;
- atomic_t dev_addr_genid;
+ atomic_unchecked_t dev_addr_genid;
#ifdef CONFIG_SYSCTL
unsigned long *sysctl_local_reserved_ports;
@@ -107,6 +107,6 @@ struct netns_ipv4 {
struct fib_rules_ops *mr_rules_ops;
#endif
#endif
- atomic_t rt_genid;
+ atomic_unchecked_t rt_genid;
};
#endif
diff --git a/include/net/netns/ipv6.h b/include/net/netns/ipv6.h
index 8d93544..05c3e89 100644
--- a/include/net/netns/ipv6.h
+++ b/include/net/netns/ipv6.h
@@ -79,8 +79,8 @@ struct netns_ipv6 {
struct fib_rules_ops *mr6_rules_ops;
#endif
#endif
- atomic_t dev_addr_genid;
- atomic_t fib6_sernum;
+ atomic_unchecked_t dev_addr_genid;
+ atomic_unchecked_t fib6_sernum;
};
#if IS_ENABLED(CONFIG_NF_DEFRAG_IPV6)
diff --git a/include/net/netns/xfrm.h b/include/net/netns/xfrm.h
index 730d82a..045f2c4 100644
--- a/include/net/netns/xfrm.h
+++ b/include/net/netns/xfrm.h
@@ -78,7 +78,7 @@ struct netns_xfrm {
/* flow cache part */
struct flow_cache flow_cache_global;
- atomic_t flow_cache_genid;
+ atomic_unchecked_t flow_cache_genid;
struct list_head flow_cache_gc_list;
spinlock_t flow_cache_gc_lock;
struct work_struct flow_cache_gc_work;
diff --git a/include/net/sock.h b/include/net/sock.h
index 4ca4c3f..010c67c 100644
--- a/include/net/sock.h
+++ b/include/net/sock.h
@@ -198,7 +198,7 @@ struct sock_common {
struct in6_addr skc_v6_rcv_saddr;
#endif
- atomic64_t skc_cookie;
+ atomic64_unchecked_t skc_cookie;
/*
* fields between dontcopy_begin/dontcopy_end
@@ -364,7 +364,7 @@ struct sock {
unsigned int sk_napi_id;
unsigned int sk_ll_usec;
#endif
- atomic_t sk_drops;
+ atomic_unchecked_t sk_drops;
int sk_rcvbuf;
struct sk_filter __rcu *sk_filter;
@@ -2107,7 +2107,7 @@ struct sock_skb_cb {
static inline void
sock_skb_set_dropcount(const struct sock *sk, struct sk_buff *skb)
{
- SOCK_SKB_CB(skb)->dropcount = atomic_read(&sk->sk_drops);
+ SOCK_SKB_CB(skb)->dropcount = atomic_read_unchecked(&sk->sk_drops);
}
void __sock_recv_timestamp(struct msghdr *msg, struct sock *sk,
diff --git a/include/net/xfrm.h b/include/net/xfrm.h
index f0ee97e..c82ef16 100644
--- a/include/net/xfrm.h
+++ b/include/net/xfrm.h
@@ -531,7 +531,7 @@ struct xfrm_policy {
struct timer_list timer;
struct flow_cache_object flo;
- atomic_t genid;
+ atomic_unchecked_t genid;
u32 priority;
u32 index;
struct xfrm_mark mark;
diff --git a/net/atm/atm_misc.c b/net/atm/atm_misc.c
index 876fbe8..8bbea9f 100644
--- a/net/atm/atm_misc.c
+++ b/net/atm/atm_misc.c
@@ -17,7 +17,7 @@ int atm_charge(struct atm_vcc *vcc, int truesize)
if (atomic_read(&sk_atm(vcc)->sk_rmem_alloc) <= sk_atm(vcc)->sk_rcvbuf)
return 1;
atm_return(vcc, truesize);
- atomic_inc(&vcc->stats->rx_drop);
+ atomic_inc_unchecked(&vcc->stats->rx_drop);
return 0;
}
EXPORT_SYMBOL(atm_charge);
@@ -39,7 +39,7 @@ struct sk_buff *atm_alloc_charge(struct atm_vcc *vcc, int pdu_size,
}
}
atm_return(vcc, guess);
- atomic_inc(&vcc->stats->rx_drop);
+ atomic_inc_unchecked(&vcc->stats->rx_drop);
return NULL;
}
EXPORT_SYMBOL(atm_alloc_charge);
@@ -86,7 +86,7 @@ EXPORT_SYMBOL(atm_pcr_goal);
void sonet_copy_stats(struct k_sonet_stats *from, struct sonet_stats *to)
{
-#define __HANDLE_ITEM(i) to->i = atomic_read(&from->i)
+#define __HANDLE_ITEM(i) to->i = atomic_read_unchecked(&from->i)
__SONET_ITEMS
#undef __HANDLE_ITEM
}
@@ -94,7 +94,7 @@ EXPORT_SYMBOL(sonet_copy_stats);
void sonet_subtract_stats(struct k_sonet_stats *from, struct sonet_stats *to)
{
-#define __HANDLE_ITEM(i) atomic_sub(to->i, &from->i)
+#define __HANDLE_ITEM(i) atomic_sub_unchecked(to->i,&from->i)
__SONET_ITEMS
#undef __HANDLE_ITEM
}
diff --git a/net/atm/proc.c b/net/atm/proc.c
index bbb6461..cf04016 100644
--- a/net/atm/proc.c
+++ b/net/atm/proc.c
@@ -45,9 +45,9 @@ static void add_stats(struct seq_file *seq, const char *aal,
const struct k_atm_aal_stats *stats)
{
seq_printf(seq, "%s ( %d %d %d %d %d )", aal,
- atomic_read(&stats->tx), atomic_read(&stats->tx_err),
- atomic_read(&stats->rx), atomic_read(&stats->rx_err),
- atomic_read(&stats->rx_drop));
+ atomic_read_unchecked(&stats->tx),atomic_read_unchecked(&stats->tx_err),
+ atomic_read_unchecked(&stats->rx),atomic_read_unchecked(&stats->rx_err),
+ atomic_read_unchecked(&stats->rx_drop));
}
static void atm_dev_info(struct seq_file *seq, const struct atm_dev *dev)
diff --git a/net/atm/resources.c b/net/atm/resources.c
index 0447d5d..3cf4728 100644
--- a/net/atm/resources.c
+++ b/net/atm/resources.c
@@ -160,7 +160,7 @@ EXPORT_SYMBOL(atm_dev_deregister);
static void copy_aal_stats(struct k_atm_aal_stats *from,
struct atm_aal_stats *to)
{
-#define __HANDLE_ITEM(i) to->i = atomic_read(&from->i)
+#define __HANDLE_ITEM(i) to->i = atomic_read_unchecked(&from->i)
__AAL_STAT_ITEMS
#undef __HANDLE_ITEM
}
@@ -168,7 +168,7 @@ static void copy_aal_stats(struct k_atm_aal_stats *from,
static void subtract_aal_stats(struct k_atm_aal_stats *from,
struct atm_aal_stats *to)
{
-#define __HANDLE_ITEM(i) atomic_sub(to->i, &from->i)
+#define __HANDLE_ITEM(i) atomic_sub_unchecked(to->i, &from->i)
__AAL_STAT_ITEMS
#undef __HANDLE_ITEM
}
diff --git a/net/batman-adv/bat_iv_ogm.c b/net/batman-adv/bat_iv_ogm.c
index 753383c..32d12d9 100644
--- a/net/batman-adv/bat_iv_ogm.c
+++ b/net/batman-adv/bat_iv_ogm.c
@@ -343,7 +343,7 @@ static int batadv_iv_ogm_iface_enable(struct batadv_hard_iface *hard_iface)
/* randomize initial seqno to avoid collision */
get_random_bytes(&random_seqno, sizeof(random_seqno));
- atomic_set(&hard_iface->bat_iv.ogm_seqno, random_seqno);
+ atomic_set_unchecked(&hard_iface->bat_iv.ogm_seqno, random_seqno);
hard_iface->bat_iv.ogm_buff_len = BATADV_OGM_HLEN;
ogm_buff = kmalloc(hard_iface->bat_iv.ogm_buff_len, GFP_ATOMIC);
@@ -947,9 +947,9 @@ static void batadv_iv_ogm_schedule(struct batadv_hard_iface *hard_iface)
batadv_ogm_packet->tvlv_len = htons(tvlv_len);
/* change sequence number to network order */
- seqno = (uint32_t)atomic_read(&hard_iface->bat_iv.ogm_seqno);
+ seqno = (uint32_t)atomic_read_unchecked(&hard_iface->bat_iv.ogm_seqno);
batadv_ogm_packet->seqno = htonl(seqno);
- atomic_inc(&hard_iface->bat_iv.ogm_seqno);
+ atomic_inc_unchecked(&hard_iface->bat_iv.ogm_seqno);
batadv_iv_ogm_slide_own_bcast_window(hard_iface);
@@ -1626,7 +1626,7 @@ static void batadv_iv_ogm_process(const struct sk_buff *skb, int ogm_offset,
return;
/* could be changed by schedule_own_packet() */
- if_incoming_seqno = atomic_read(&if_incoming->bat_iv.ogm_seqno);
+ if_incoming_seqno = atomic_read_unchecked(&if_incoming->bat_iv.ogm_seqno);
if (ogm_packet->flags & BATADV_DIRECTLINK)
has_directlink_flag = true;
diff --git a/net/batman-adv/fragmentation.c b/net/batman-adv/fragmentation.c
index c0f0d01..725928a 100644
--- a/net/batman-adv/fragmentation.c
+++ b/net/batman-adv/fragmentation.c
@@ -465,7 +465,7 @@ bool batadv_frag_send_packet(struct sk_buff *skb,
frag_header.packet_type = BATADV_UNICAST_FRAG;
frag_header.version = BATADV_COMPAT_VERSION;
frag_header.ttl = BATADV_TTL;
- frag_header.seqno = htons(atomic_inc_return(&bat_priv->frag_seqno));
+ frag_header.seqno = htons(atomic_inc_return_unchecked(&bat_priv->frag_seqno));
frag_header.reserved = 0;
frag_header.no = 0;
frag_header.total_size = htons(skb->len);
diff --git a/net/batman-adv/soft-interface.c b/net/batman-adv/soft-interface.c
index 51cda3a..f5cbc7e 100644
--- a/net/batman-adv/soft-interface.c
+++ b/net/batman-adv/soft-interface.c
@@ -330,7 +330,7 @@ send:
primary_if->net_dev->dev_addr);
/* set broadcast sequence number */
- seqno = atomic_inc_return(&bat_priv->bcast_seqno);
+ seqno = atomic_inc_return_unchecked(&bat_priv->bcast_seqno);
bcast_packet->seqno = htonl(seqno);
batadv_add_bcast_packet_to_list(bat_priv, skb, brd_delay);
@@ -798,7 +798,7 @@ static int batadv_softif_init_late(struct net_device *dev)
atomic_set(&bat_priv->batman_queue_left, BATADV_BATMAN_QUEUE_LEN);
atomic_set(&bat_priv->mesh_state, BATADV_MESH_INACTIVE);
- atomic_set(&bat_priv->bcast_seqno, 1);
+ atomic_set_unchecked(&bat_priv->bcast_seqno, 1);
atomic_set(&bat_priv->tt.vn, 0);
atomic_set(&bat_priv->tt.local_changes, 0);
atomic_set(&bat_priv->tt.ogm_append_cnt, 0);
@@ -812,7 +812,7 @@ static int batadv_softif_init_late(struct net_device *dev)
/* randomize initial seqno to avoid collision */
get_random_bytes(&random_seqno, sizeof(random_seqno));
- atomic_set(&bat_priv->frag_seqno, random_seqno);
+ atomic_set_unchecked(&bat_priv->frag_seqno, random_seqno);
bat_priv->primary_if = NULL;
bat_priv->num_ifaces = 0;
diff --git a/net/batman-adv/types.h b/net/batman-adv/types.h
index 55610a8..aba2ae8 100644
--- a/net/batman-adv/types.h
+++ b/net/batman-adv/types.h
@@ -81,7 +81,7 @@ enum batadv_dhcp_recipient {
struct batadv_hard_iface_bat_iv {
unsigned char *ogm_buff;
int ogm_buff_len;
- atomic_t ogm_seqno;
+ atomic_unchecked_t ogm_seqno;
};
/**
@@ -786,7 +786,7 @@ struct batadv_priv {
atomic_t bonding;
atomic_t fragmentation;
atomic_t packet_size_max;
- atomic_t frag_seqno;
+ atomic_unchecked_t frag_seqno;
#ifdef CONFIG_BATMAN_ADV_BLA
atomic_t bridge_loop_avoidance;
#endif
@@ -805,7 +805,7 @@ struct batadv_priv {
#endif
uint32_t isolation_mark;
uint32_t isolation_mark_mask;
- atomic_t bcast_seqno;
+ atomic_unchecked_t bcast_seqno;
atomic_t bcast_queue_left;
atomic_t batman_queue_left;
char num_ifaces;
diff --git a/net/caif/cfctrl.c b/net/caif/cfctrl.c
index f5afda1..74e77ce 100644
--- a/net/caif/cfctrl.c
+++ b/net/caif/cfctrl.c
@@ -43,8 +43,8 @@ struct cflayer *cfctrl_create(void)
memset(&dev_info, 0, sizeof(dev_info));
dev_info.id = 0xff;
cfsrvl_init(&this->serv, 0, &dev_info, false);
- atomic_set(&this->req_seq_no, 1);
- atomic_set(&this->rsp_seq_no, 1);
+ atomic_set_unchecked(&this->req_seq_no, 1);
+ atomic_set_unchecked(&this->rsp_seq_no, 1);
this->serv.layer.receive = cfctrl_recv;
sprintf(this->serv.layer.name, "ctrl");
this->serv.layer.ctrlcmd = cfctrl_ctrlcmd;
@@ -130,8 +130,8 @@ static void cfctrl_insert_req(struct cfctrl *ctrl,
struct cfctrl_request_info *req)
{
spin_lock_bh(&ctrl->info_list_lock);
- atomic_inc(&ctrl->req_seq_no);
- req->sequence_no = atomic_read(&ctrl->req_seq_no);
+ atomic_inc_unchecked(&ctrl->req_seq_no);
+ req->sequence_no = atomic_read_unchecked(&ctrl->req_seq_no);
list_add_tail(&req->list, &ctrl->list);
spin_unlock_bh(&ctrl->info_list_lock);
}
@@ -149,7 +149,7 @@ static struct cfctrl_request_info *cfctrl_remove_req(struct cfctrl *ctrl,
if (p != first)
pr_warn("Requests are not received in order\n");
- atomic_set(&ctrl->rsp_seq_no,
+ atomic_set_unchecked(&ctrl->rsp_seq_no,
p->sequence_no);
list_del(&p->list);
goto out;
diff --git a/net/ceph/messenger.c b/net/ceph/messenger.c
index e3be1d2..254c555 100644
--- a/net/ceph/messenger.c
+++ b/net/ceph/messenger.c
@@ -189,7 +189,7 @@ static void con_fault(struct ceph_connection *con);
#define MAX_ADDR_STR_LEN 64 /* 54 is enough */
static char addr_str[ADDR_STR_COUNT][MAX_ADDR_STR_LEN];
-static atomic_t addr_str_seq = ATOMIC_INIT(0);
+static atomic_unchecked_t addr_str_seq = ATOMIC_INIT(0);
static struct page *zero_page; /* used in certain error cases */
@@ -200,7 +200,7 @@ const char *ceph_pr_addr(const struct sockaddr_storage *ss)
struct sockaddr_in *in4 = (struct sockaddr_in *) ss;
struct sockaddr_in6 *in6 = (struct sockaddr_in6 *) ss;
- i = atomic_inc_return(&addr_str_seq) & ADDR_STR_COUNT_MASK;
+ i = atomic_inc_return_unchecked(&addr_str_seq) & ADDR_STR_COUNT_MASK;
s = addr_str[i];
switch (ss->ss_family) {
diff --git a/net/core/datagram.c b/net/core/datagram.c
index 617088a..0364f4f 100644
--- a/net/core/datagram.c
+++ b/net/core/datagram.c
@@ -338,7 +338,7 @@ int skb_kill_datagram(struct sock *sk, struct sk_buff *skb, unsigned int flags)
}
kfree_skb(skb);
- atomic_inc(&sk->sk_drops);
+ atomic_inc_unchecked(&sk->sk_drops);
sk_mem_reclaim_partial(sk);
return err;
diff --git a/net/core/dev.c b/net/core/dev.c
index a8e4dd4..8419b44 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -1721,7 +1721,7 @@ int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
{
if (skb_orphan_frags(skb, GFP_ATOMIC) ||
unlikely(!is_skb_forwardable(dev, skb))) {
- atomic_long_inc(&dev->rx_dropped);
+ atomic_long_inc_unchecked(&dev->rx_dropped);
kfree_skb(skb);
return NET_RX_DROP;
}
@@ -3125,7 +3125,7 @@ recursion_alert:
drop:
rcu_read_unlock_bh();
- atomic_long_inc(&dev->tx_dropped);
+ atomic_long_inc_unchecked(&dev->tx_dropped);
kfree_skb_list(skb);
return rc;
out:
@@ -3477,7 +3477,7 @@ drop:
local_irq_restore(flags);
- atomic_long_inc(&skb->dev->rx_dropped);
+ atomic_long_inc_unchecked(&skb->dev->rx_dropped);
kfree_skb(skb);
return NET_RX_DROP;
}
@@ -3892,7 +3892,7 @@ ncls:
ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
} else {
drop:
- atomic_long_inc(&skb->dev->rx_dropped);
+ atomic_long_inc_unchecked(&skb->dev->rx_dropped);
kfree_skb(skb);
/* Jamal, now you will not able to escape explaining
* me how you were going to use this. :-)
@@ -6843,8 +6843,8 @@ struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
} else {
netdev_stats_to_stats64(storage, &dev->stats);
}
- storage->rx_dropped += atomic_long_read(&dev->rx_dropped);
- storage->tx_dropped += atomic_long_read(&dev->tx_dropped);
+ storage->rx_dropped += atomic_long_read_unchecked(&dev->rx_dropped);
+ storage->tx_dropped += atomic_long_read_unchecked(&dev->tx_dropped);
return storage;
}
EXPORT_SYMBOL(dev_get_stats);
diff --git a/net/core/flow.c b/net/core/flow.c
index 1033725..340f65d 100644
--- a/net/core/flow.c
+++ b/net/core/flow.c
@@ -65,7 +65,7 @@ static void flow_cache_new_hashrnd(unsigned long arg)
static int flow_entry_valid(struct flow_cache_entry *fle,
struct netns_xfrm *xfrm)
{
- if (atomic_read(&xfrm->flow_cache_genid) != fle->genid)
+ if (atomic_read_unchecked(&xfrm->flow_cache_genid) != fle->genid)
return 0;
if (fle->object && !fle->object->ops->check(fle->object))
return 0;
@@ -242,7 +242,7 @@ flow_cache_lookup(struct net *net, const struct flowi *key, u16 family, u8 dir,
hlist_add_head(&fle->u.hlist, &fcp->hash_table[hash]);
fcp->hash_count++;
}
- } else if (likely(fle->genid == atomic_read(&net->xfrm.flow_cache_genid))) {
+ } else if (likely(fle->genid == atomic_read_unchecked(&net->xfrm.flow_cache_genid))) {
flo = fle->object;
if (!flo)
goto ret_object;
@@ -263,7 +263,7 @@ nocache:
}
flo = resolver(net, key, family, dir, flo, ctx);
if (fle) {
- fle->genid = atomic_read(&net->xfrm.flow_cache_genid);
+ fle->genid = atomic_read_unchecked(&net->xfrm.flow_cache_genid);
if (!IS_ERR(flo))
fle->object = flo;
else
diff --git a/net/core/net-sysfs.c b/net/core/net-sysfs.c
index 18b34d7..faecc1d 100644
--- a/net/core/net-sysfs.c
+++ b/net/core/net-sysfs.c
@@ -288,7 +288,7 @@ static ssize_t carrier_changes_show(struct device *dev,
{
struct net_device *netdev = to_net_dev(dev);
return sprintf(buf, fmt_dec,
- atomic_read(&netdev->carrier_changes));
+ atomic_read_unchecked(&netdev->carrier_changes));
}
static DEVICE_ATTR_RO(carrier_changes);
diff --git a/net/core/netpoll.c b/net/core/netpoll.c
index c126a87..10ad89d 100644
--- a/net/core/netpoll.c
+++ b/net/core/netpoll.c
@@ -377,7 +377,7 @@ void netpoll_send_udp(struct netpoll *np, const char *msg, int len)
struct udphdr *udph;
struct iphdr *iph;
struct ethhdr *eth;
- static atomic_t ip_ident;
+ static atomic_unchecked_t ip_ident;
struct ipv6hdr *ip6h;
udp_len = len + sizeof(*udph);
@@ -448,7 +448,7 @@ void netpoll_send_udp(struct netpoll *np, const char *msg, int len)
put_unaligned(0x45, (unsigned char *)iph);
iph->tos = 0;
put_unaligned(htons(ip_len), &(iph->tot_len));
- iph->id = htons(atomic_inc_return(&ip_ident));
+ iph->id = htons(atomic_inc_return_unchecked(&ip_ident));
iph->frag_off = 0;
iph->ttl = 64;
iph->protocol = IPPROTO_UDP;
diff --git a/net/core/rtnetlink.c b/net/core/rtnetlink.c
index 0861018..4a60a1e 100644
--- a/net/core/rtnetlink.c
+++ b/net/core/rtnetlink.c
@@ -1082,7 +1082,7 @@ static int rtnl_fill_ifinfo(struct sk_buff *skb, struct net_device *dev,
(dev->ifalias &&
nla_put_string(skb, IFLA_IFALIAS, dev->ifalias)) ||
nla_put_u32(skb, IFLA_CARRIER_CHANGES,
- atomic_read(&dev->carrier_changes)))
+ atomic_read_unchecked(&dev->carrier_changes)))
goto nla_put_failure;
if (1) {
diff --git a/net/core/sock.c b/net/core/sock.c
index 193901d..a5b2488 100644
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -441,7 +441,7 @@ int sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
struct sk_buff_head *list = &sk->sk_receive_queue;
if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf) {
- atomic_inc(&sk->sk_drops);
+ atomic_inc_unchecked(&sk->sk_drops);
trace_sock_rcvqueue_full(sk, skb);
return -ENOMEM;
}
@@ -451,7 +451,7 @@ int sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
return err;
if (!sk_rmem_schedule(sk, skb, skb->truesize)) {
- atomic_inc(&sk->sk_drops);
+ atomic_inc_unchecked(&sk->sk_drops);
return -ENOBUFS;
}
@@ -484,7 +484,7 @@ int sk_receive_skb(struct sock *sk, struct sk_buff *skb, const int nested)
skb->dev = NULL;
if (sk_rcvqueues_full(sk, sk->sk_rcvbuf)) {
- atomic_inc(&sk->sk_drops);
+ atomic_inc_unchecked(&sk->sk_drops);
goto discard_and_relse;
}
if (nested)
@@ -502,7 +502,7 @@ int sk_receive_skb(struct sock *sk, struct sk_buff *skb, const int nested)
mutex_release(&sk->sk_lock.dep_map, 1, _RET_IP_);
} else if (sk_add_backlog(sk, skb, sk->sk_rcvbuf)) {
bh_unlock_sock(sk);
- atomic_inc(&sk->sk_drops);
+ atomic_inc_unchecked(&sk->sk_drops);
goto discard_and_relse;
}
@@ -1550,7 +1550,7 @@ struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority)
newsk->sk_err = 0;
newsk->sk_priority = 0;
newsk->sk_incoming_cpu = raw_smp_processor_id();
- atomic64_set(&newsk->sk_cookie, 0);
+ atomic64_set_unchecked(&newsk->sk_cookie, 0);
/*
* Before updating sk_refcnt, we must commit prior changes to memory
* (Documentation/RCU/rculist_nulls.txt for details)
@@ -2359,7 +2359,7 @@ void sock_init_data(struct socket *sock, struct sock *sk)
*/
smp_wmb();
atomic_set(&sk->sk_refcnt, 1);
- atomic_set(&sk->sk_drops, 0);
+ atomic_set_unchecked(&sk->sk_drops, 0);
}
EXPORT_SYMBOL(sock_init_data);
diff --git a/net/core/sock_diag.c b/net/core/sock_diag.c
index 817622f..8846e2b 100644
--- a/net/core/sock_diag.c
+++ b/net/core/sock_diag.c
@@ -20,12 +20,12 @@ static struct workqueue_struct *broadcast_wq;
static u64 sock_gen_cookie(struct sock *sk)
{
while (1) {
- u64 res = atomic64_read(&sk->sk_cookie);
+ u64 res = atomic64_read_unchecked(&sk->sk_cookie);
if (res)
return res;
- res = atomic64_inc_return(&sock_net(sk)->cookie_gen);
- atomic64_cmpxchg(&sk->sk_cookie, 0, res);
+ res = atomic64_inc_return_unchecked(&sock_net(sk)->cookie_gen);
+ atomic64_cmpxchg_unchecked(&sk->sk_cookie, 0, res);
}
}
diff --git a/net/ipv4/devinet.c b/net/ipv4/devinet.c
index 2d9cb17..fa6f6f8 100644
--- a/net/ipv4/devinet.c
+++ b/net/ipv4/devinet.c
@@ -1579,7 +1579,7 @@ static int inet_dump_ifaddr(struct sk_buff *skb, struct netlink_callback *cb)
idx = 0;
head = &net->dev_index_head[h];
rcu_read_lock();
- cb->seq = atomic_read(&net->ipv4.dev_addr_genid) ^
+ cb->seq = atomic_read_unchecked(&net->ipv4.dev_addr_genid) ^
net->dev_base_seq;
hlist_for_each_entry_rcu(dev, head, index_hlist) {
if (idx < s_idx)
@@ -1905,7 +1905,7 @@ static int inet_netconf_dump_devconf(struct sk_buff *skb,
idx = 0;
head = &net->dev_index_head[h];
rcu_read_lock();
- cb->seq = atomic_read(&net->ipv4.dev_addr_genid) ^
+ cb->seq = atomic_read_unchecked(&net->ipv4.dev_addr_genid) ^
net->dev_base_seq;
hlist_for_each_entry_rcu(dev, head, index_hlist) {
if (idx < s_idx)
diff --git a/net/ipv4/fib_frontend.c b/net/ipv4/fib_frontend.c
index 6bbc549..28d7495 100644
--- a/net/ipv4/fib_frontend.c
+++ b/net/ipv4/fib_frontend.c
@@ -1083,12 +1083,12 @@ static int fib_inetaddr_event(struct notifier_block *this, unsigned long event,
#ifdef CONFIG_IP_ROUTE_MULTIPATH
fib_sync_up(dev, RTNH_F_DEAD);
#endif
- atomic_inc(&net->ipv4.dev_addr_genid);
+ atomic_inc_unchecked(&net->ipv4.dev_addr_genid);
rt_cache_flush(dev_net(dev));
break;
case NETDEV_DOWN:
fib_del_ifaddr(ifa, NULL);
- atomic_inc(&net->ipv4.dev_addr_genid);
+ atomic_inc_unchecked(&net->ipv4.dev_addr_genid);
if (!ifa->ifa_dev->ifa_list) {
/* Last address was deleted from this interface.
* Disable IP.
@@ -1127,7 +1127,7 @@ static int fib_netdev_event(struct notifier_block *this, unsigned long event, vo
#ifdef CONFIG_IP_ROUTE_MULTIPATH
fib_sync_up(dev, RTNH_F_DEAD);
#endif
- atomic_inc(&net->ipv4.dev_addr_genid);
+ atomic_inc_unchecked(&net->ipv4.dev_addr_genid);
rt_cache_flush(net);
break;
case NETDEV_DOWN:
diff --git a/net/ipv4/fib_semantics.c b/net/ipv4/fib_semantics.c
index 3a06586..1020c5b 100644
--- a/net/ipv4/fib_semantics.c
+++ b/net/ipv4/fib_semantics.c
@@ -755,7 +755,7 @@ __be32 fib_info_update_nh_saddr(struct net *net, struct fib_nh *nh)
nh->nh_saddr = inet_select_addr(nh->nh_dev,
nh->nh_gw,
nh->nh_parent->fib_scope);
- nh->nh_saddr_genid = atomic_read(&net->ipv4.dev_addr_genid);
+ nh->nh_saddr_genid = atomic_read_unchecked(&net->ipv4.dev_addr_genid);
return nh->nh_saddr;
}
diff --git a/net/ipv4/inet_connection_sock.c b/net/ipv4/inet_connection_sock.c
index 61b45a1..2970363 100644
--- a/net/ipv4/inet_connection_sock.c
+++ b/net/ipv4/inet_connection_sock.c
@@ -729,8 +729,8 @@ struct sock *inet_csk_clone_lock(const struct sock *sk,
newsk->sk_write_space = sk_stream_write_space;
newsk->sk_mark = inet_rsk(req)->ir_mark;
- atomic64_set(&newsk->sk_cookie,
- atomic64_read(&inet_rsk(req)->ir_cookie));
+ atomic64_set_unchecked(&newsk->sk_cookie,
+ atomic64_read_unchecked(&inet_rsk(req)->ir_cookie));
newicsk->icsk_retransmits = 0;
newicsk->icsk_backoff = 0;
diff --git a/net/ipv4/inet_timewait_sock.c b/net/ipv4/inet_timewait_sock.c
index 2ffbd16..6e94995 100644
--- a/net/ipv4/inet_timewait_sock.c
+++ b/net/ipv4/inet_timewait_sock.c
@@ -214,7 +214,7 @@ struct inet_timewait_sock *inet_twsk_alloc(const struct sock *sk,
tw->tw_ipv6only = 0;
tw->tw_transparent = inet->transparent;
tw->tw_prot = sk->sk_prot_creator;
- atomic64_set(&tw->tw_cookie, atomic64_read(&sk->sk_cookie));
+ atomic64_set_unchecked(&tw->tw_cookie, atomic64_read_unchecked(&sk->sk_cookie));
twsk_net_set(tw, sock_net(sk));
setup_timer(&tw->tw_timer, tw_timer_handler, (unsigned long)tw);
/*
diff --git a/net/ipv4/inetpeer.c b/net/ipv4/inetpeer.c
index 241afd7..31b95d5 100644
--- a/net/ipv4/inetpeer.c
+++ b/net/ipv4/inetpeer.c
@@ -461,7 +461,7 @@ relookup:
if (p) {
p->daddr = *daddr;
atomic_set(&p->refcnt, 1);
- atomic_set(&p->rid, 0);
+ atomic_set_unchecked(&p->rid, 0);
p->metrics[RTAX_LOCK-1] = INETPEER_METRICS_NEW;
p->rate_tokens = 0;
/* 60*HZ is arbitrary, but chosen enough high so that the first
diff --git a/net/ipv4/ip_fragment.c b/net/ipv4/ip_fragment.c
index 921138f..327173b 100644
--- a/net/ipv4/ip_fragment.c
+++ b/net/ipv4/ip_fragment.c
@@ -276,7 +276,7 @@ static int ip_frag_too_far(struct ipq *qp)
return 0;
start = qp->rid;
- end = atomic_inc_return(&peer->rid);
+ end = atomic_inc_return_unchecked(&peer->rid);
qp->rid = end;
rc = qp->q.fragments && (end - start) > max;
diff --git a/net/ipv4/ping.c b/net/ipv4/ping.c
index 05ff44b..8e685ea 100644
--- a/net/ipv4/ping.c
+++ b/net/ipv4/ping.c
@@ -1116,7 +1116,7 @@ static void ping_v4_format_sock(struct sock *sp, struct seq_file *f,
from_kuid_munged(seq_user_ns(f), sock_i_uid(sp)),
0, sock_i_ino(sp),
atomic_read(&sp->sk_refcnt), sp,
- atomic_read(&sp->sk_drops));
+ atomic_read_unchecked(&sp->sk_drops));
}
static int ping_v4_seq_show(struct seq_file *seq, void *v)
diff --git a/net/ipv4/raw.c b/net/ipv4/raw.c
index 561cd4b..7e7104d 100644
--- a/net/ipv4/raw.c
+++ b/net/ipv4/raw.c
@@ -323,7 +323,7 @@ static int raw_rcv_skb(struct sock *sk, struct sk_buff *skb)
int raw_rcv(struct sock *sk, struct sk_buff *skb)
{
if (!xfrm4_policy_check(sk, XFRM_POLICY_IN, skb)) {
- atomic_inc(&sk->sk_drops);
+ atomic_inc_unchecked(&sk->sk_drops);
kfree_skb(skb);
return NET_RX_DROP;
}
@@ -1020,7 +1020,7 @@ static void raw_sock_seq_show(struct seq_file *seq, struct sock *sp, int i)
0, 0L, 0,
from_kuid_munged(seq_user_ns(seq), sock_i_uid(sp)),
0, sock_i_ino(sp),
- atomic_read(&sp->sk_refcnt), sp, atomic_read(&sp->sk_drops));
+ atomic_read(&sp->sk_refcnt), sp, atomic_read_unchecked(&sp->sk_drops));
}
static int raw_seq_show(struct seq_file *seq, void *v)
diff --git a/net/ipv4/route.c b/net/ipv4/route.c
index e681b85..ef2796f 100644
--- a/net/ipv4/route.c
+++ b/net/ipv4/route.c
@@ -458,7 +458,7 @@ static struct neighbour *ipv4_neigh_lookup(const struct dst_entry *dst,
#define IP_IDENTS_SZ 2048u
-static atomic_t *ip_idents __read_mostly;
+static atomic_unchecked_t *ip_idents __read_mostly;
static u32 *ip_tstamps __read_mostly;
/* In order to protect privacy, we add a perturbation to identifiers
@@ -468,7 +468,7 @@ static u32 *ip_tstamps __read_mostly;
u32 ip_idents_reserve(u32 hash, int segs)
{
u32 *p_tstamp = ip_tstamps + hash % IP_IDENTS_SZ;
- atomic_t *p_id = ip_idents + hash % IP_IDENTS_SZ;
+ atomic_unchecked_t *p_id = ip_idents + hash % IP_IDENTS_SZ;
u32 old = ACCESS_ONCE(*p_tstamp);
u32 now = (u32)jiffies;
u32 delta = 0;
@@ -476,7 +476,7 @@ u32 ip_idents_reserve(u32 hash, int segs)
if (old != now && cmpxchg(p_tstamp, old, now) == old)
delta = prandom_u32_max(now - old);
- return atomic_add_return(segs + delta, p_id) - segs;
+ return atomic_add_return_unchecked(segs + delta, p_id) - segs;
}
EXPORT_SYMBOL(ip_idents_reserve);
@@ -2690,8 +2690,8 @@ static __net_initdata struct pernet_operations sysctl_route_ops = {
static __net_init int rt_genid_init(struct net *net)
{
- atomic_set(&net->ipv4.rt_genid, 0);
- atomic_set(&net->fnhe_genid, 0);
+ atomic_set_unchecked(&net->ipv4.rt_genid, 0);
+ atomic_set_unchecked(&net->fnhe_genid, 0);
get_random_bytes(&net->ipv4.dev_addr_genid,
sizeof(net->ipv4.dev_addr_genid));
return 0;
diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c
index 728f5b3..a95e744 100644
--- a/net/ipv4/tcp_input.c
+++ b/net/ipv4/tcp_input.c
@@ -6026,7 +6026,7 @@ struct request_sock *inet_reqsk_alloc(const struct request_sock_ops *ops,
kmemcheck_annotate_bitfield(ireq, flags);
ireq->opt = NULL;
- atomic64_set(&ireq->ir_cookie, 0);
+ atomic64_set_unchecked(&ireq->ir_cookie, 0);
ireq->ireq_state = TCP_NEW_SYN_RECV;
write_pnet(&ireq->ireq_net, sock_net(sk_listener));
ireq->ireq_family = sk_listener->sk_family;
diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c
index 1b8c5ba..b7b9745 100644
--- a/net/ipv4/udp.c
+++ b/net/ipv4/udp.c
@@ -1193,7 +1193,7 @@ static unsigned int first_packet_length(struct sock *sk)
IS_UDPLITE(sk));
UDP_INC_STATS_BH(sock_net(sk), UDP_MIB_INERRORS,
IS_UDPLITE(sk));
- atomic_inc(&sk->sk_drops);
+ atomic_inc_unchecked(&sk->sk_drops);
__skb_unlink(skb, rcvq);
__skb_queue_tail(&list_kill, skb);
}
@@ -1305,7 +1305,7 @@ try_again:
if (unlikely(err)) {
trace_kfree_skb(skb, udp_recvmsg);
if (!peeked) {
- atomic_inc(&sk->sk_drops);
+ atomic_inc_unchecked(&sk->sk_drops);
UDP_INC_STATS_USER(sock_net(sk),
UDP_MIB_INERRORS, is_udplite);
}
@@ -1599,7 +1599,7 @@ csum_error:
UDP_INC_STATS_BH(sock_net(sk), UDP_MIB_CSUMERRORS, is_udplite);
drop:
UDP_INC_STATS_BH(sock_net(sk), UDP_MIB_INERRORS, is_udplite);
- atomic_inc(&sk->sk_drops);
+ atomic_inc_unchecked(&sk->sk_drops);
kfree_skb(skb);
return -1;
}
@@ -1617,7 +1617,7 @@ static void flush_stack(struct sock **stack, unsigned int count,
skb1 = (i == final) ? skb : skb_clone(skb, GFP_ATOMIC);
if (!skb1) {
- atomic_inc(&sk->sk_drops);
+ atomic_inc_unchecked(&sk->sk_drops);
UDP_INC_STATS_BH(sock_net(sk), UDP_MIB_RCVBUFERRORS,
IS_UDPLITE(sk));
UDP_INC_STATS_BH(sock_net(sk), UDP_MIB_INERRORS,
@@ -2427,7 +2427,7 @@ static void udp4_format_sock(struct sock *sp, struct seq_file *f,
from_kuid_munged(seq_user_ns(f), sock_i_uid(sp)),
0, sock_i_ino(sp),
atomic_read(&sp->sk_refcnt), sp,
- atomic_read(&sp->sk_drops));
+ atomic_read_unchecked(&sp->sk_drops));
}
int udp4_seq_show(struct seq_file *seq, void *v)
diff --git a/net/ipv6/addrconf.c b/net/ipv6/addrconf.c
index 21c2c81..80267dd 100644
--- a/net/ipv6/addrconf.c
+++ b/net/ipv6/addrconf.c
@@ -620,7 +620,7 @@ static int inet6_netconf_dump_devconf(struct sk_buff *skb,
idx = 0;
head = &net->dev_index_head[h];
rcu_read_lock();
- cb->seq = atomic_read(&net->ipv6.dev_addr_genid) ^
+ cb->seq = atomic_read_unchecked(&net->ipv6.dev_addr_genid) ^
net->dev_base_seq;
hlist_for_each_entry_rcu(dev, head, index_hlist) {
if (idx < s_idx)
@@ -4402,7 +4402,7 @@ static int inet6_dump_addr(struct sk_buff *skb, struct netlink_callback *cb,
s_ip_idx = ip_idx = cb->args[2];
rcu_read_lock();
- cb->seq = atomic_read(&net->ipv6.dev_addr_genid) ^ net->dev_base_seq;
+ cb->seq = atomic_read_unchecked(&net->ipv6.dev_addr_genid) ^ net->dev_base_seq;
for (h = s_h; h < NETDEV_HASHENTRIES; h++, s_idx = 0) {
idx = 0;
head = &net->dev_index_head[h];
@@ -5059,7 +5059,7 @@ static void __ipv6_ifa_notify(int event, struct inet6_ifaddr *ifp)
rt_genid_bump_ipv6(net);
break;
}
- atomic_inc(&net->ipv6.dev_addr_genid);
+ atomic_inc_unchecked(&net->ipv6.dev_addr_genid);
}
static void ipv6_ifa_notify(int event, struct inet6_ifaddr *ifp)
diff --git a/net/ipv6/af_inet6.c b/net/ipv6/af_inet6.c
index 7de52b6..ce7fb94 100644
--- a/net/ipv6/af_inet6.c
+++ b/net/ipv6/af_inet6.c
@@ -770,7 +770,7 @@ static int __net_init inet6_net_init(struct net *net)
net->ipv6.sysctl.idgen_retries = 3;
net->ipv6.sysctl.idgen_delay = 1 * HZ;
net->ipv6.sysctl.flowlabel_state_ranges = 1;
- atomic_set(&net->ipv6.fib6_sernum, 1);
+ atomic_set_unchecked(&net->ipv6.fib6_sernum, 1);
err = ipv6_init_mibs(net);
if (err)
diff --git a/net/ipv6/datagram.c b/net/ipv6/datagram.c
index b10a889..e881e1f 100644
--- a/net/ipv6/datagram.c
+++ b/net/ipv6/datagram.c
@@ -977,5 +977,5 @@ void ip6_dgram_sock_seq_show(struct seq_file *seq, struct sock *sp,
0,
sock_i_ino(sp),
atomic_read(&sp->sk_refcnt), sp,
- atomic_read(&sp->sk_drops));
+ atomic_read_unchecked(&sp->sk_drops));
}
diff --git a/net/ipv6/ip6_fib.c b/net/ipv6/ip6_fib.c
index 548c623..bc8ec4f 100644
--- a/net/ipv6/ip6_fib.c
+++ b/net/ipv6/ip6_fib.c
@@ -99,9 +99,9 @@ static int fib6_new_sernum(struct net *net)
int new, old;
do {
- old = atomic_read(&net->ipv6.fib6_sernum);
+ old = atomic_read_unchecked(&net->ipv6.fib6_sernum);
new = old < INT_MAX ? old + 1 : 1;
- } while (atomic_cmpxchg(&net->ipv6.fib6_sernum,
+ } while (atomic_cmpxchg_unchecked(&net->ipv6.fib6_sernum,
old, new) != old);
return new;
}
diff --git a/net/ipv6/raw.c b/net/ipv6/raw.c
index ca4700c..f20fe21 100644
--- a/net/ipv6/raw.c
+++ b/net/ipv6/raw.c
@@ -388,7 +388,7 @@ static inline int rawv6_rcv_skb(struct sock *sk, struct sk_buff *skb)
{
if ((raw6_sk(sk)->checksum || rcu_access_pointer(sk->sk_filter)) &&
skb_checksum_complete(skb)) {
- atomic_inc(&sk->sk_drops);
+ atomic_inc_unchecked(&sk->sk_drops);
kfree_skb(skb);
return NET_RX_DROP;
}
@@ -416,7 +416,7 @@ int rawv6_rcv(struct sock *sk, struct sk_buff *skb)
struct raw6_sock *rp = raw6_sk(sk);
if (!xfrm6_policy_check(sk, XFRM_POLICY_IN, skb)) {
- atomic_inc(&sk->sk_drops);
+ atomic_inc_unchecked(&sk->sk_drops);
kfree_skb(skb);
return NET_RX_DROP;
}
@@ -440,7 +440,7 @@ int rawv6_rcv(struct sock *sk, struct sk_buff *skb)
if (inet->hdrincl) {
if (skb_checksum_complete(skb)) {
- atomic_inc(&sk->sk_drops);
+ atomic_inc_unchecked(&sk->sk_drops);
kfree_skb(skb);
return NET_RX_DROP;
}
diff --git a/net/ipv6/udp.c b/net/ipv6/udp.c
index e51fc3e..5b274f6 100644
--- a/net/ipv6/udp.c
+++ b/net/ipv6/udp.c
@@ -445,7 +445,7 @@ try_again:
if (unlikely(err)) {
trace_kfree_skb(skb, udpv6_recvmsg);
if (!peeked) {
- atomic_inc(&sk->sk_drops);
+ atomic_inc_unchecked(&sk->sk_drops);
if (is_udp4)
UDP_INC_STATS_USER(sock_net(sk),
UDP_MIB_INERRORS,
@@ -709,7 +709,7 @@ csum_error:
UDP6_INC_STATS_BH(sock_net(sk), UDP_MIB_CSUMERRORS, is_udplite);
drop:
UDP6_INC_STATS_BH(sock_net(sk), UDP_MIB_INERRORS, is_udplite);
- atomic_inc(&sk->sk_drops);
+ atomic_inc_unchecked(&sk->sk_drops);
kfree_skb(skb);
return -1;
}
@@ -750,7 +750,7 @@ static void flush_stack(struct sock **stack, unsigned int count,
if (likely(!skb1))
skb1 = (i == final) ? skb : skb_clone(skb, GFP_ATOMIC);
if (!skb1) {
- atomic_inc(&sk->sk_drops);
+ atomic_inc_unchecked(&sk->sk_drops);
UDP6_INC_STATS_BH(sock_net(sk), UDP_MIB_RCVBUFERRORS,
IS_UDPLITE(sk));
UDP6_INC_STATS_BH(sock_net(sk), UDP_MIB_INERRORS,
diff --git a/net/iucv/af_iucv.c b/net/iucv/af_iucv.c
index 918151c..5bbe95a 100644
--- a/net/iucv/af_iucv.c
+++ b/net/iucv/af_iucv.c
@@ -686,10 +686,10 @@ static void __iucv_auto_name(struct iucv_sock *iucv)
{
char name[12];
- sprintf(name, "%08x", atomic_inc_return(&iucv_sk_list.autobind_name));
+ sprintf(name, "%08x", atomic_inc_return_unchecked(&iucv_sk_list.autobind_name));
while (__iucv_get_sock_by_name(name)) {
sprintf(name, "%08x",
- atomic_inc_return(&iucv_sk_list.autobind_name));
+ atomic_inc_return_unchecked(&iucv_sk_list.autobind_name));
}
memcpy(iucv->src_name, name, 8);
}
diff --git a/net/key/af_key.c b/net/key/af_key.c
index 83a7068..facf2f0 100644
--- a/net/key/af_key.c
+++ b/net/key/af_key.c
@@ -3050,10 +3050,10 @@ static int pfkey_send_policy_notify(struct xfrm_policy *xp, int dir, const struc
static u32 get_acqseq(void)
{
u32 res;
- static atomic_t acqseq;
+ static atomic_unchecked_t acqseq;
do {
- res = atomic_inc_return(&acqseq);
+ res = atomic_inc_return_unchecked(&acqseq);
} while (!res);
return res;
}
diff --git a/net/l2tp/l2tp_eth.c b/net/l2tp/l2tp_eth.c
index 4b55287..bd247f7 100644
--- a/net/l2tp/l2tp_eth.c
+++ b/net/l2tp/l2tp_eth.c
@@ -42,12 +42,12 @@ struct l2tp_eth {
struct sock *tunnel_sock;
struct l2tp_session *session;
struct list_head list;
- atomic_long_t tx_bytes;
- atomic_long_t tx_packets;
- atomic_long_t tx_dropped;
- atomic_long_t rx_bytes;
- atomic_long_t rx_packets;
- atomic_long_t rx_errors;
+ atomic_long_unchecked_t tx_bytes;
+ atomic_long_unchecked_t tx_packets;
+ atomic_long_unchecked_t tx_dropped;
+ atomic_long_unchecked_t rx_bytes;
+ atomic_long_unchecked_t rx_packets;
+ atomic_long_unchecked_t rx_errors;
};
/* via l2tp_session_priv() */
@@ -98,10 +98,10 @@ static int l2tp_eth_dev_xmit(struct sk_buff *skb, struct net_device *dev)
int ret = l2tp_xmit_skb(session, skb, session->hdr_len);
if (likely(ret == NET_XMIT_SUCCESS)) {
- atomic_long_add(len, &priv->tx_bytes);
- atomic_long_inc(&priv->tx_packets);
+ atomic_long_add_unchecked(len, &priv->tx_bytes);
+ atomic_long_inc_unchecked(&priv->tx_packets);
} else {
- atomic_long_inc(&priv->tx_dropped);
+ atomic_long_inc_unchecked(&priv->tx_dropped);
}
return NETDEV_TX_OK;
}
@@ -111,12 +111,12 @@ static struct rtnl_link_stats64 *l2tp_eth_get_stats64(struct net_device *dev,
{
struct l2tp_eth *priv = netdev_priv(dev);
- stats->tx_bytes = atomic_long_read(&priv->tx_bytes);
- stats->tx_packets = atomic_long_read(&priv->tx_packets);
- stats->tx_dropped = atomic_long_read(&priv->tx_dropped);
- stats->rx_bytes = atomic_long_read(&priv->rx_bytes);
- stats->rx_packets = atomic_long_read(&priv->rx_packets);
- stats->rx_errors = atomic_long_read(&priv->rx_errors);
+ stats->tx_bytes = atomic_long_read_unchecked(&priv->tx_bytes);
+ stats->tx_packets = atomic_long_read_unchecked(&priv->tx_packets);
+ stats->tx_dropped = atomic_long_read_unchecked(&priv->tx_dropped);
+ stats->rx_bytes = atomic_long_read_unchecked(&priv->rx_bytes);
+ stats->rx_packets = atomic_long_read_unchecked(&priv->rx_packets);
+ stats->rx_errors = atomic_long_read_unchecked(&priv->rx_errors);
return stats;
}
@@ -167,15 +167,15 @@ static void l2tp_eth_dev_recv(struct l2tp_session *session, struct sk_buff *skb,
nf_reset(skb);
if (dev_forward_skb(dev, skb) == NET_RX_SUCCESS) {
- atomic_long_inc(&priv->rx_packets);
- atomic_long_add(data_len, &priv->rx_bytes);
+ atomic_long_inc_unchecked(&priv->rx_packets);
+ atomic_long_add_unchecked(data_len, &priv->rx_bytes);
} else {
- atomic_long_inc(&priv->rx_errors);
+ atomic_long_inc_unchecked(&priv->rx_errors);
}
return;
error:
- atomic_long_inc(&priv->rx_errors);
+ atomic_long_inc_unchecked(&priv->rx_errors);
kfree_skb(skb);
}
diff --git a/net/netfilter/ipvs/ip_vs_conn.c b/net/netfilter/ipvs/ip_vs_conn.c
index b0f7b62..0541842 100644
--- a/net/netfilter/ipvs/ip_vs_conn.c
+++ b/net/netfilter/ipvs/ip_vs_conn.c
@@ -572,7 +572,7 @@ ip_vs_bind_dest(struct ip_vs_conn *cp, struct ip_vs_dest *dest)
/* Increase the refcnt counter of the dest */
ip_vs_dest_hold(dest);
- conn_flags = atomic_read(&dest->conn_flags);
+ conn_flags = atomic_read_unchecked(&dest->conn_flags);
if (cp->protocol != IPPROTO_UDP)
conn_flags &= ~IP_VS_CONN_F_ONE_PACKET;
flags = cp->flags;
@@ -922,7 +922,7 @@ ip_vs_conn_new(const struct ip_vs_conn_param *p, int dest_af,
cp->control = NULL;
atomic_set(&cp->n_control, 0);
- atomic_set(&cp->in_pkts, 0);
+ atomic_set_unchecked(&cp->in_pkts, 0);
cp->packet_xmit = NULL;
cp->app = NULL;
@@ -1229,7 +1229,7 @@ static inline int todrop_entry(struct ip_vs_conn *cp)
/* Don't drop the entry if its number of incoming packets is not
located in [0, 8] */
- i = atomic_read(&cp->in_pkts);
+ i = atomic_read_unchecked(&cp->in_pkts);
if (i > 8 || i < 0) return 0;
if (!todrop_rate[i]) return 0;
diff --git a/net/netfilter/ipvs/ip_vs_core.c b/net/netfilter/ipvs/ip_vs_core.c
index 38fbc19..4272cb4 100644
--- a/net/netfilter/ipvs/ip_vs_core.c
+++ b/net/netfilter/ipvs/ip_vs_core.c
@@ -586,7 +586,7 @@ int ip_vs_leave(struct ip_vs_service *svc, struct sk_buff *skb,
ret = cp->packet_xmit(skb, cp, pd->pp, iph);
/* do not touch skb anymore */
- atomic_inc(&cp->in_pkts);
+ atomic_inc_unchecked(&cp->in_pkts);
ip_vs_conn_put(cp);
return ret;
}
@@ -1762,7 +1762,7 @@ ip_vs_in(unsigned int hooknum, struct sk_buff *skb, int af)
if (cp->flags & IP_VS_CONN_F_ONE_PACKET)
pkts = sysctl_sync_threshold(ipvs);
else
- pkts = atomic_add_return(1, &cp->in_pkts);
+ pkts = atomic_add_return_unchecked(1, &cp->in_pkts);
if (ipvs->sync_state & IP_VS_STATE_MASTER)
ip_vs_sync_conn(net, cp, pkts);
diff --git a/net/netfilter/ipvs/ip_vs_ctl.c b/net/netfilter/ipvs/ip_vs_ctl.c
index 24c5542..d4c95f9 100644
--- a/net/netfilter/ipvs/ip_vs_ctl.c
+++ b/net/netfilter/ipvs/ip_vs_ctl.c
@@ -814,7 +814,7 @@ __ip_vs_update_dest(struct ip_vs_service *svc, struct ip_vs_dest *dest,
*/
ip_vs_rs_hash(ipvs, dest);
}
- atomic_set(&dest->conn_flags, conn_flags);
+ atomic_set_unchecked(&dest->conn_flags, conn_flags);
/* bind the service */
old_svc = rcu_dereference_protected(dest->svc, 1);
@@ -2036,7 +2036,7 @@ static int ip_vs_info_seq_show(struct seq_file *seq, void *v)
" %-7s %-6d %-10d %-10d\n",
&dest->addr.in6,
ntohs(dest->port),
- ip_vs_fwd_name(atomic_read(&dest->conn_flags)),
+ ip_vs_fwd_name(atomic_read_unchecked(&dest->conn_flags)),
atomic_read(&dest->weight),
atomic_read(&dest->activeconns),
atomic_read(&dest->inactconns));
@@ -2047,7 +2047,7 @@ static int ip_vs_info_seq_show(struct seq_file *seq, void *v)
"%-7s %-6d %-10d %-10d\n",
ntohl(dest->addr.ip),
ntohs(dest->port),
- ip_vs_fwd_name(atomic_read(&dest->conn_flags)),
+ ip_vs_fwd_name(atomic_read_unchecked(&dest->conn_flags)),
atomic_read(&dest->weight),
atomic_read(&dest->activeconns),
atomic_read(&dest->inactconns));
@@ -2546,7 +2546,7 @@ __ip_vs_get_dest_entries(struct net *net, const struct ip_vs_get_dests *get,
entry.addr = dest->addr.ip;
entry.port = dest->port;
- entry.conn_flags = atomic_read(&dest->conn_flags);
+ entry.conn_flags = atomic_read_unchecked(&dest->conn_flags);
entry.weight = atomic_read(&dest->weight);
entry.u_threshold = dest->u_threshold;
entry.l_threshold = dest->l_threshold;
@@ -3121,7 +3121,7 @@ static int ip_vs_genl_fill_dest(struct sk_buff *skb, struct ip_vs_dest *dest)
if (nla_put(skb, IPVS_DEST_ATTR_ADDR, sizeof(dest->addr), &dest->addr) ||
nla_put_be16(skb, IPVS_DEST_ATTR_PORT, dest->port) ||
nla_put_u32(skb, IPVS_DEST_ATTR_FWD_METHOD,
- (atomic_read(&dest->conn_flags) &
+ (atomic_read_unchecked(&dest->conn_flags) &
IP_VS_CONN_F_FWD_MASK)) ||
nla_put_u32(skb, IPVS_DEST_ATTR_WEIGHT,
atomic_read(&dest->weight)) ||
diff --git a/net/netfilter/ipvs/ip_vs_sync.c b/net/netfilter/ipvs/ip_vs_sync.c
index d99ad93..09bd6dc 100644
--- a/net/netfilter/ipvs/ip_vs_sync.c
+++ b/net/netfilter/ipvs/ip_vs_sync.c
@@ -609,7 +609,7 @@ static void ip_vs_sync_conn_v0(struct net *net, struct ip_vs_conn *cp,
cp = cp->control;
if (cp) {
if (cp->flags & IP_VS_CONN_F_TEMPLATE)
- pkts = atomic_add_return(1, &cp->in_pkts);
+ pkts = atomic_add_return_unchecked(1, &cp->in_pkts);
else
pkts = sysctl_sync_threshold(ipvs);
ip_vs_sync_conn(net, cp, pkts);
@@ -771,7 +771,7 @@ control:
if (!cp)
return;
if (cp->flags & IP_VS_CONN_F_TEMPLATE)
- pkts = atomic_add_return(1, &cp->in_pkts);
+ pkts = atomic_add_return_unchecked(1, &cp->in_pkts);
else
pkts = sysctl_sync_threshold(ipvs);
goto sloop;
@@ -919,7 +919,7 @@ static void ip_vs_proc_conn(struct net *net, struct ip_vs_conn_param *param,
if (opt)
memcpy(&cp->in_seq, opt, sizeof(*opt));
- atomic_set(&cp->in_pkts, sysctl_sync_threshold(ipvs));
+ atomic_set_unchecked(&cp->in_pkts, sysctl_sync_threshold(ipvs));
cp->state = state;
cp->old_state = cp->state;
/*
diff --git a/net/netfilter/ipvs/ip_vs_xmit.c b/net/netfilter/ipvs/ip_vs_xmit.c
index 258a0b0..2082f50 100644
--- a/net/netfilter/ipvs/ip_vs_xmit.c
+++ b/net/netfilter/ipvs/ip_vs_xmit.c
@@ -1259,7 +1259,7 @@ ip_vs_icmp_xmit(struct sk_buff *skb, struct ip_vs_conn *cp,
else
rc = NF_ACCEPT;
/* do not touch skb anymore */
- atomic_inc(&cp->in_pkts);
+ atomic_inc_unchecked(&cp->in_pkts);
goto out;
}
@@ -1352,7 +1352,7 @@ ip_vs_icmp_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp,
else
rc = NF_ACCEPT;
/* do not touch skb anymore */
- atomic_inc(&cp->in_pkts);
+ atomic_inc_unchecked(&cp->in_pkts);
goto out;
}
diff --git a/net/netfilter/nfnetlink_log.c b/net/netfilter/nfnetlink_log.c
index 4670821..a6c3c47d 100644
--- a/net/netfilter/nfnetlink_log.c
+++ b/net/netfilter/nfnetlink_log.c
@@ -84,7 +84,7 @@ static int nfnl_log_net_id __read_mostly;
struct nfnl_log_net {
spinlock_t instances_lock;
struct hlist_head instance_table[INSTANCE_BUCKETS];
- atomic_t global_seq;
+ atomic_unchecked_t global_seq;
};
static struct nfnl_log_net *nfnl_log_pernet(struct net *net)
@@ -572,7 +572,7 @@ __build_packet_message(struct nfnl_log_net *log,
/* global sequence number */
if ((inst->flags & NFULNL_CFG_F_SEQ_GLOBAL) &&
nla_put_be32(inst->skb, NFULA_SEQ_GLOBAL,
- htonl(atomic_inc_return(&log->global_seq))))
+ htonl(atomic_inc_return_unchecked(&log->global_seq))))
goto nla_put_failure;
if (data_len) {
diff --git a/net/netfilter/xt_statistic.c b/net/netfilter/xt_statistic.c
index 11de55e..f25e448 100644
--- a/net/netfilter/xt_statistic.c
+++ b/net/netfilter/xt_statistic.c
@@ -19,7 +19,7 @@
#include <linux/module.h>
struct xt_statistic_priv {
- atomic_t count;
+ atomic_unchecked_t count;
} ____cacheline_aligned_in_smp;
MODULE_LICENSE("GPL");
@@ -42,9 +42,9 @@ statistic_mt(const struct sk_buff *skb, struct xt_action_param *par)
break;
case XT_STATISTIC_MODE_NTH:
do {
- oval = atomic_read(&info->master->count);
+ oval = atomic_read_unchecked(&info->master->count);
nval = (oval == info->u.nth.every) ? 0 : oval + 1;
- } while (atomic_cmpxchg(&info->master->count, oval, nval) != oval);
+ } while (atomic_cmpxchg_unchecked(&info->master->count, oval, nval) != oval);
if (nval == 0)
ret = !ret;
break;
@@ -64,7 +64,7 @@ static int statistic_mt_check(const struct xt_mtchk_param *par)
info->master = kzalloc(sizeof(*info->master), GFP_KERNEL);
if (info->master == NULL)
return -ENOMEM;
- atomic_set(&info->master->count, info->u.nth.count);
+ atomic_set_unchecked(&info->master->count, info->u.nth.count);
return 0;
}
diff --git a/net/netlink/af_netlink.c b/net/netlink/af_netlink.c
index a133d16..fafee70 100644
--- a/net/netlink/af_netlink.c
+++ b/net/netlink/af_netlink.c
@@ -286,7 +286,7 @@ static void netlink_overrun(struct sock *sk)
sk->sk_error_report(sk);
}
}
- atomic_inc(&sk->sk_drops);
+ atomic_inc_unchecked(&sk->sk_drops);
}
static void netlink_rcv_wake(struct sock *sk)
@@ -3145,7 +3145,7 @@ static int netlink_seq_show(struct seq_file *seq, void *v)
sk_wmem_alloc_get(s),
nlk->cb_running,
atomic_read(&s->sk_refcnt),
- atomic_read(&s->sk_drops),
+ atomic_read_unchecked(&s->sk_drops),
sock_i_ino(s)
);
diff --git a/net/openvswitch/vport.c b/net/openvswitch/vport.c
index 067a3ff..9ecc05c 100644
--- a/net/openvswitch/vport.c
+++ b/net/openvswitch/vport.c
@@ -303,10 +303,10 @@ void ovs_vport_get_stats(struct vport *vport, struct ovs_vport_stats *stats)
* netdev-stats can be directly read over netlink-ioctl.
*/
- stats->rx_errors = atomic_long_read(&vport->err_stats.rx_errors);
- stats->tx_errors = atomic_long_read(&vport->err_stats.tx_errors);
- stats->tx_dropped = atomic_long_read(&vport->err_stats.tx_dropped);
- stats->rx_dropped = atomic_long_read(&vport->err_stats.rx_dropped);
+ stats->rx_errors = atomic_long_read_unchecked(&vport->err_stats.rx_errors);
+ stats->tx_errors = atomic_long_read_unchecked(&vport->err_stats.tx_errors);
+ stats->tx_dropped = atomic_long_read_unchecked(&vport->err_stats.tx_dropped);
+ stats->rx_dropped = atomic_long_read_unchecked(&vport->err_stats.rx_dropped);
for_each_possible_cpu(i) {
const struct pcpu_sw_netstats *percpu_stats;
@@ -538,19 +538,19 @@ static void ovs_vport_record_error(struct vport *vport,
{
switch (err_type) {
case VPORT_E_RX_DROPPED:
- atomic_long_inc(&vport->err_stats.rx_dropped);
+ atomic_long_inc_unchecked(&vport->err_stats.rx_dropped);
break;
case VPORT_E_RX_ERROR:
- atomic_long_inc(&vport->err_stats.rx_errors);
+ atomic_long_inc_unchecked(&vport->err_stats.rx_errors);
break;
case VPORT_E_TX_DROPPED:
- atomic_long_inc(&vport->err_stats.tx_dropped);
+ atomic_long_inc_unchecked(&vport->err_stats.tx_dropped);
break;
case VPORT_E_TX_ERROR:
- atomic_long_inc(&vport->err_stats.tx_errors);
+ atomic_long_inc_unchecked(&vport->err_stats.tx_errors);
break;
}
diff --git a/net/openvswitch/vport.h b/net/openvswitch/vport.h
index bc85331..0d3dce0 100644
--- a/net/openvswitch/vport.h
+++ b/net/openvswitch/vport.h
@@ -71,10 +71,10 @@ int ovs_vport_get_egress_tun_info(struct vport *vport, struct sk_buff *skb,
/* The following definitions are for implementers of vport devices: */
struct vport_err_stats {
- atomic_long_t rx_dropped;
- atomic_long_t rx_errors;
- atomic_long_t tx_dropped;
- atomic_long_t tx_errors;
+ atomic_long_unchecked_t rx_dropped;
+ atomic_long_unchecked_t rx_errors;
+ atomic_long_unchecked_t tx_dropped;
+ atomic_long_unchecked_t tx_errors;
};
/**
* struct vport_portids - array of netlink portids of a vport.
diff --git a/net/packet/af_packet.c b/net/packet/af_packet.c
index 7851b12..67ecf51 100644
--- a/net/packet/af_packet.c
+++ b/net/packet/af_packet.c
@@ -277,7 +277,7 @@ static int packet_direct_xmit(struct sk_buff *skb)
return ret;
drop:
- atomic_long_inc(&dev->tx_dropped);
+ atomic_long_inc_unchecked(&dev->tx_dropped);
kfree_skb(skb);
return NET_XMIT_DROP;
}
@@ -1946,7 +1946,7 @@ static int packet_rcv(struct sk_buff *skb, struct net_device *dev,
drop_n_acct:
spin_lock(&sk->sk_receive_queue.lock);
po->stats.stats1.tp_drops++;
- atomic_inc(&sk->sk_drops);
+ atomic_inc_unchecked(&sk->sk_drops);
spin_unlock(&sk->sk_receive_queue.lock);
drop_n_restore:
diff --git a/net/phonet/pep.c b/net/phonet/pep.c
index 850a86c..8884a37 100644
--- a/net/phonet/pep.c
+++ b/net/phonet/pep.c
@@ -388,7 +388,7 @@ static int pipe_do_rcv(struct sock *sk, struct sk_buff *skb)
case PNS_PEP_CTRL_REQ:
if (skb_queue_len(&pn->ctrlreq_queue) >= PNPIPE_CTRLREQ_MAX) {
- atomic_inc(&sk->sk_drops);
+ atomic_inc_unchecked(&sk->sk_drops);
break;
}
__skb_pull(skb, 4);
@@ -409,7 +409,7 @@ static int pipe_do_rcv(struct sock *sk, struct sk_buff *skb)
}
if (pn->rx_credits == 0) {
- atomic_inc(&sk->sk_drops);
+ atomic_inc_unchecked(&sk->sk_drops);
err = -ENOBUFS;
break;
}
@@ -579,7 +579,7 @@ static int pipe_handler_do_rcv(struct sock *sk, struct sk_buff *skb)
}
if (pn->rx_credits == 0) {
- atomic_inc(&sk->sk_drops);
+ atomic_inc_unchecked(&sk->sk_drops);
err = NET_RX_DROP;
break;
}
diff --git a/net/phonet/socket.c b/net/phonet/socket.c
index d575ef4..1dedc31 100644
--- a/net/phonet/socket.c
+++ b/net/phonet/socket.c
@@ -611,7 +611,7 @@ static int pn_sock_seq_show(struct seq_file *seq, void *v)
from_kuid_munged(seq_user_ns(seq), sock_i_uid(sk)),
sock_i_ino(sk),
atomic_read(&sk->sk_refcnt), sk,
- atomic_read(&sk->sk_drops));
+ atomic_read_unchecked(&sk->sk_drops));
}
seq_pad(seq, '\n');
return 0;
diff --git a/net/rds/cong.c b/net/rds/cong.c
index e6144b8..4f9fda6 100644
--- a/net/rds/cong.c
+++ b/net/rds/cong.c
@@ -78,7 +78,7 @@
* finds that the saved generation number is smaller than the global generation
* number, it wakes up the process.
*/
-static atomic_t rds_cong_generation = ATOMIC_INIT(0);
+static atomic_unchecked_t rds_cong_generation = ATOMIC_INIT(0);
/*
* Congestion monitoring
@@ -247,7 +247,7 @@ void rds_cong_map_updated(struct rds_cong_map *map, uint64_t portmask)
rdsdebug("waking map %p for %pI4\n",
map, &map->m_addr);
rds_stats_inc(s_cong_update_received);
- atomic_inc(&rds_cong_generation);
+ atomic_inc_unchecked(&rds_cong_generation);
if (waitqueue_active(&map->m_waitq))
wake_up(&map->m_waitq);
if (waitqueue_active(&rds_poll_waitq))
@@ -273,7 +273,7 @@ EXPORT_SYMBOL_GPL(rds_cong_map_updated);
int rds_cong_updated_since(unsigned long *recent)
{
- unsigned long gen = atomic_read(&rds_cong_generation);
+ unsigned long gen = atomic_read_unchecked(&rds_cong_generation);
if (likely(*recent == gen))
return 0;
diff --git a/net/rds/ib.h b/net/rds/ib.h
index 86d88ec..b5461c2 100644
--- a/net/rds/ib.h
+++ b/net/rds/ib.h
@@ -128,7 +128,7 @@ struct rds_ib_connection {
/* sending acks */
unsigned long i_ack_flags;
#ifdef KERNEL_HAS_ATOMIC64
- atomic64_t i_ack_next; /* next ACK to send */
+ atomic64_unchecked_t i_ack_next; /* next ACK to send */
#else
spinlock_t i_ack_lock; /* protect i_ack_next */
u64 i_ack_next; /* next ACK to send */
diff --git a/net/rds/ib_cm.c b/net/rds/ib_cm.c
index 0da2a45..388ae0b 100644
--- a/net/rds/ib_cm.c
+++ b/net/rds/ib_cm.c
@@ -699,7 +699,7 @@ void rds_ib_conn_shutdown(struct rds_connection *conn)
/* Clear the ACK state */
clear_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags);
#ifdef KERNEL_HAS_ATOMIC64
- atomic64_set(&ic->i_ack_next, 0);
+ atomic64_set_unchecked(&ic->i_ack_next, 0);
#else
ic->i_ack_next = 0;
#endif
diff --git a/net/rds/ib_recv.c b/net/rds/ib_recv.c
index cac5b45..9a6f263 100644
--- a/net/rds/ib_recv.c
+++ b/net/rds/ib_recv.c
@@ -581,7 +581,7 @@ static u64 rds_ib_get_ack(struct rds_ib_connection *ic)
static void rds_ib_set_ack(struct rds_ib_connection *ic, u64 seq,
int ack_required)
{
- atomic64_set(&ic->i_ack_next, seq);
+ atomic64_set_unchecked(&ic->i_ack_next, seq);
if (ack_required) {
smp_mb__before_atomic();
set_bit(IB_ACK_REQUESTED, &ic->i_ack_flags);
@@ -593,7 +593,7 @@ static u64 rds_ib_get_ack(struct rds_ib_connection *ic)
clear_bit(IB_ACK_REQUESTED, &ic->i_ack_flags);
smp_mb__after_atomic();
- return atomic64_read(&ic->i_ack_next);
+ return atomic64_read_unchecked(&ic->i_ack_next);
}
#endif
diff --git a/net/rds/iw.h b/net/rds/iw.h
index cbe6674..dc9eb89 100644
--- a/net/rds/iw.h
+++ b/net/rds/iw.h
@@ -134,7 +134,7 @@ struct rds_iw_connection {
/* sending acks */
unsigned long i_ack_flags;
#ifdef KERNEL_HAS_ATOMIC64
- atomic64_t i_ack_next; /* next ACK to send */
+ atomic64_unchecked_t i_ack_next; /* next ACK to send */
#else
spinlock_t i_ack_lock; /* protect i_ack_next */
u64 i_ack_next; /* next ACK to send */
diff --git a/net/rds/iw_cm.c b/net/rds/iw_cm.c
index 8f486fa..7849d8c 100644
--- a/net/rds/iw_cm.c
+++ b/net/rds/iw_cm.c
@@ -666,7 +666,7 @@ void rds_iw_conn_shutdown(struct rds_connection *conn)
/* Clear the ACK state */
clear_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags);
#ifdef KERNEL_HAS_ATOMIC64
- atomic64_set(&ic->i_ack_next, 0);
+ atomic64_set_unchecked(&ic->i_ack_next, 0);
#else
ic->i_ack_next = 0;
#endif
diff --git a/net/rds/iw_recv.c b/net/rds/iw_recv.c
index a66d179..cf1e258 100644
--- a/net/rds/iw_recv.c
+++ b/net/rds/iw_recv.c
@@ -412,7 +412,7 @@ static u64 rds_iw_get_ack(struct rds_iw_connection *ic)
static void rds_iw_set_ack(struct rds_iw_connection *ic, u64 seq,
int ack_required)
{
- atomic64_set(&ic->i_ack_next, seq);
+ atomic64_set_unchecked(&ic->i_ack_next, seq);
if (ack_required) {
smp_mb__before_atomic();
set_bit(IB_ACK_REQUESTED, &ic->i_ack_flags);
@@ -424,7 +424,7 @@ static u64 rds_iw_get_ack(struct rds_iw_connection *ic)
clear_bit(IB_ACK_REQUESTED, &ic->i_ack_flags);
smp_mb__after_atomic();
- return atomic64_read(&ic->i_ack_next);
+ return atomic64_read_unchecked(&ic->i_ack_next);
}
#endif
diff --git a/net/rxrpc/af_rxrpc.c b/net/rxrpc/af_rxrpc.c
index 25d60ed..ec9ae2c 100644
--- a/net/rxrpc/af_rxrpc.c
+++ b/net/rxrpc/af_rxrpc.c
@@ -40,7 +40,7 @@ static const struct proto_ops rxrpc_rpc_ops;
__be32 rxrpc_epoch;
/* current debugging ID */
-atomic_t rxrpc_debug_id;
+atomic_unchecked_t rxrpc_debug_id;
/* count of skbs currently in use */
atomic_t rxrpc_n_skbs;
diff --git a/net/rxrpc/ar-ack.c b/net/rxrpc/ar-ack.c
index e0547f5..a171f7a 100644
--- a/net/rxrpc/ar-ack.c
+++ b/net/rxrpc/ar-ack.c
@@ -224,7 +224,7 @@ static void rxrpc_resend(struct rxrpc_call *call)
_enter("{%d,%d,%d,%d},",
call->acks_hard, call->acks_unacked,
- atomic_read(&call->sequence),
+ atomic_read_unchecked(&call->sequence),
CIRC_CNT(call->acks_head, call->acks_tail, call->acks_winsz));
stop = false;
@@ -248,7 +248,7 @@ static void rxrpc_resend(struct rxrpc_call *call)
/* each Tx packet has a new serial number */
sp->hdr.serial =
- htonl(atomic_inc_return(&call->conn->serial));
+ htonl(atomic_inc_return_unchecked(&call->conn->serial));
hdr = (struct rxrpc_header *) txb->head;
hdr->serial = sp->hdr.serial;
@@ -452,7 +452,7 @@ static void rxrpc_rotate_tx_window(struct rxrpc_call *call, u32 hard)
*/
static void rxrpc_clear_tx_window(struct rxrpc_call *call)
{
- rxrpc_rotate_tx_window(call, atomic_read(&call->sequence));
+ rxrpc_rotate_tx_window(call, atomic_read_unchecked(&call->sequence));
}
/*
@@ -678,7 +678,7 @@ process_further:
latest = ntohl(sp->hdr.serial);
hard = ntohl(ack.firstPacket);
- tx = atomic_read(&call->sequence);
+ tx = atomic_read_unchecked(&call->sequence);
_proto("Rx ACK %%%u { m=%hu f=#%u p=#%u s=%%%u r=%s n=%u }",
latest,
@@ -1210,7 +1210,7 @@ void rxrpc_process_call(struct work_struct *work)
goto maybe_reschedule;
send_ACK_with_skew:
- ack.maxSkew = htons(atomic_read(&call->conn->hi_serial) -
+ ack.maxSkew = htons(atomic_read_unchecked(&call->conn->hi_serial) -
ntohl(ack.serial));
send_ACK:
mtu = call->conn->trans->peer->if_mtu;
@@ -1222,7 +1222,7 @@ send_ACK:
ackinfo.rxMTU = htonl(rxrpc_rx_mtu);
ackinfo.jumbo_max = htonl(rxrpc_rx_jumbo_max);
- hdr.serial = htonl(atomic_inc_return(&call->conn->serial));
+ hdr.serial = htonl(atomic_inc_return_unchecked(&call->conn->serial));
_proto("Tx ACK %%%u { m=%hu f=#%u p=#%u s=%%%u r=%s n=%u }",
ntohl(hdr.serial),
ntohs(ack.maxSkew),
@@ -1240,7 +1240,7 @@ send_ACK:
send_message:
_debug("send message");
- hdr.serial = htonl(atomic_inc_return(&call->conn->serial));
+ hdr.serial = htonl(atomic_inc_return_unchecked(&call->conn->serial));
_proto("Tx %s %%%u", rxrpc_pkts[hdr.type], ntohl(hdr.serial));
send_message_2:
diff --git a/net/rxrpc/ar-call.c b/net/rxrpc/ar-call.c
index a9e05db..194e793 100644
--- a/net/rxrpc/ar-call.c
+++ b/net/rxrpc/ar-call.c
@@ -232,7 +232,7 @@ static struct rxrpc_call *rxrpc_alloc_call(gfp_t gfp)
spin_lock_init(&call->lock);
rwlock_init(&call->state_lock);
atomic_set(&call->usage, 1);
- call->debug_id = atomic_inc_return(&rxrpc_debug_id);
+ call->debug_id = atomic_inc_return_unchecked(&rxrpc_debug_id);
call->state = RXRPC_CALL_CLIENT_SEND_REQUEST;
memset(&call->sock_node, 0xed, sizeof(call->sock_node));
diff --git a/net/rxrpc/ar-connection.c b/net/rxrpc/ar-connection.c
index 6631f4f..bfdf056 100644
--- a/net/rxrpc/ar-connection.c
+++ b/net/rxrpc/ar-connection.c
@@ -210,7 +210,7 @@ static struct rxrpc_connection *rxrpc_alloc_connection(gfp_t gfp)
rwlock_init(&conn->lock);
spin_lock_init(&conn->state_lock);
atomic_set(&conn->usage, 1);
- conn->debug_id = atomic_inc_return(&rxrpc_debug_id);
+ conn->debug_id = atomic_inc_return_unchecked(&rxrpc_debug_id);
conn->avail_calls = RXRPC_MAXCALLS;
conn->size_align = 4;
conn->header_size = sizeof(struct rxrpc_header);
diff --git a/net/rxrpc/ar-connevent.c b/net/rxrpc/ar-connevent.c
index e7ed43a..6afa140 100644
--- a/net/rxrpc/ar-connevent.c
+++ b/net/rxrpc/ar-connevent.c
@@ -109,7 +109,7 @@ static int rxrpc_abort_connection(struct rxrpc_connection *conn,
len = iov[0].iov_len + iov[1].iov_len;
- hdr.serial = htonl(atomic_inc_return(&conn->serial));
+ hdr.serial = htonl(atomic_inc_return_unchecked(&conn->serial));
_proto("Tx CONN ABORT %%%u { %d }", ntohl(hdr.serial), abort_code);
ret = kernel_sendmsg(conn->trans->local->socket, &msg, iov, 2, len);
diff --git a/net/rxrpc/ar-input.c b/net/rxrpc/ar-input.c
index 4505a69..fe73096 100644
--- a/net/rxrpc/ar-input.c
+++ b/net/rxrpc/ar-input.c
@@ -331,9 +331,9 @@ void rxrpc_fast_process_packet(struct rxrpc_call *call, struct sk_buff *skb)
/* track the latest serial number on this connection for ACK packet
* information */
serial = ntohl(sp->hdr.serial);
- hi_serial = atomic_read(&call->conn->hi_serial);
+ hi_serial = atomic_read_unchecked(&call->conn->hi_serial);
while (serial > hi_serial)
- hi_serial = atomic_cmpxchg(&call->conn->hi_serial, hi_serial,
+ hi_serial = atomic_cmpxchg_unchecked(&call->conn->hi_serial, hi_serial,
serial);
/* request ACK generation for any ACK or DATA packet that requests
diff --git a/net/rxrpc/ar-internal.h b/net/rxrpc/ar-internal.h
index aef1bd2..97736c3 100644
--- a/net/rxrpc/ar-internal.h
+++ b/net/rxrpc/ar-internal.h
@@ -274,8 +274,8 @@ struct rxrpc_connection {
int error; /* error code for local abort */
int debug_id; /* debug ID for printks */
unsigned int call_counter; /* call ID counter */
- atomic_t serial; /* packet serial number counter */
- atomic_t hi_serial; /* highest serial number received */
+ atomic_unchecked_t serial; /* packet serial number counter */
+ atomic_unchecked_t hi_serial; /* highest serial number received */
u8 avail_calls; /* number of calls available */
u8 size_align; /* data size alignment (for security) */
u8 header_size; /* rxrpc + security header size */
@@ -348,7 +348,7 @@ struct rxrpc_call {
spinlock_t lock;
rwlock_t state_lock; /* lock for state transition */
atomic_t usage;
- atomic_t sequence; /* Tx data packet sequence counter */
+ atomic_unchecked_t sequence; /* Tx data packet sequence counter */
u32 abort_code; /* local/remote abort code */
enum { /* current state of call */
RXRPC_CALL_CLIENT_SEND_REQUEST, /* - client sending request phase */
@@ -433,7 +433,7 @@ static inline void rxrpc_abort_call(struct rxrpc_call *call, u32 abort_code)
*/
extern atomic_t rxrpc_n_skbs;
extern __be32 rxrpc_epoch;
-extern atomic_t rxrpc_debug_id;
+extern atomic_unchecked_t rxrpc_debug_id;
extern struct workqueue_struct *rxrpc_workqueue;
/*
diff --git a/net/rxrpc/ar-local.c b/net/rxrpc/ar-local.c
index 78483b4..b1c0e1e 100644
--- a/net/rxrpc/ar-local.c
+++ b/net/rxrpc/ar-local.c
@@ -53,7 +53,7 @@ struct rxrpc_local *rxrpc_alloc_local(struct sockaddr_rxrpc *srx)
spin_lock_init(&local->lock);
rwlock_init(&local->services_lock);
atomic_set(&local->usage, 1);
- local->debug_id = atomic_inc_return(&rxrpc_debug_id);
+ local->debug_id = atomic_inc_return_unchecked(&rxrpc_debug_id);
memcpy(&local->srx, srx, sizeof(*srx));
}
diff --git a/net/rxrpc/ar-output.c b/net/rxrpc/ar-output.c
index c004280..ab1ed5a 100644
--- a/net/rxrpc/ar-output.c
+++ b/net/rxrpc/ar-output.c
@@ -648,14 +648,14 @@ static int rxrpc_send_data(struct rxrpc_sock *rx,
memset(skb_put(skb, pad), 0, pad);
}
- seq = atomic_inc_return(&call->sequence);
+ seq = atomic_inc_return_unchecked(&call->sequence);
sp->hdr.epoch = conn->epoch;
sp->hdr.cid = call->cid;
sp->hdr.callNumber = call->call_id;
sp->hdr.seq = htonl(seq);
sp->hdr.serial =
- htonl(atomic_inc_return(&conn->serial));
+ htonl(atomic_inc_return_unchecked(&conn->serial));
sp->hdr.type = RXRPC_PACKET_TYPE_DATA;
sp->hdr.userStatus = 0;
sp->hdr.securityIndex = conn->security_ix;
diff --git a/net/rxrpc/ar-peer.c b/net/rxrpc/ar-peer.c
index bebaa43..2644591 100644
--- a/net/rxrpc/ar-peer.c
+++ b/net/rxrpc/ar-peer.c
@@ -72,7 +72,7 @@ static struct rxrpc_peer *rxrpc_alloc_peer(struct sockaddr_rxrpc *srx,
INIT_LIST_HEAD(&peer->error_targets);
spin_lock_init(&peer->lock);
atomic_set(&peer->usage, 1);
- peer->debug_id = atomic_inc_return(&rxrpc_debug_id);
+ peer->debug_id = atomic_inc_return_unchecked(&rxrpc_debug_id);
memcpy(&peer->srx, srx, sizeof(*srx));
rxrpc_assess_MTU_size(peer);
diff --git a/net/rxrpc/ar-proc.c b/net/rxrpc/ar-proc.c
index 38047f7..9f48511 100644
--- a/net/rxrpc/ar-proc.c
+++ b/net/rxrpc/ar-proc.c
@@ -164,8 +164,8 @@ static int rxrpc_connection_seq_show(struct seq_file *seq, void *v)
atomic_read(&conn->usage),
rxrpc_conn_states[conn->state],
key_serial(conn->key),
- atomic_read(&conn->serial),
- atomic_read(&conn->hi_serial));
+ atomic_read_unchecked(&conn->serial),
+ atomic_read_unchecked(&conn->hi_serial));
return 0;
}
diff --git a/net/rxrpc/ar-transport.c b/net/rxrpc/ar-transport.c
index 1976dec..aa70be5 100644
--- a/net/rxrpc/ar-transport.c
+++ b/net/rxrpc/ar-transport.c
@@ -51,7 +51,7 @@ static struct rxrpc_transport *rxrpc_alloc_transport(struct rxrpc_local *local,
spin_lock_init(&trans->client_lock);
rwlock_init(&trans->conn_lock);
atomic_set(&trans->usage, 1);
- trans->debug_id = atomic_inc_return(&rxrpc_debug_id);
+ trans->debug_id = atomic_inc_return_unchecked(&rxrpc_debug_id);
if (peer->srx.transport.family == AF_INET) {
switch (peer->srx.transport_type) {
diff --git a/net/rxrpc/rxkad.c b/net/rxrpc/rxkad.c
index f226709..0e735a8 100644
--- a/net/rxrpc/rxkad.c
+++ b/net/rxrpc/rxkad.c
@@ -610,7 +610,7 @@ static int rxkad_issue_challenge(struct rxrpc_connection *conn)
len = iov[0].iov_len + iov[1].iov_len;
- hdr.serial = htonl(atomic_inc_return(&conn->serial));
+ hdr.serial = htonl(atomic_inc_return_unchecked(&conn->serial));
_proto("Tx CHALLENGE %%%u", ntohl(hdr.serial));
ret = kernel_sendmsg(conn->trans->local->socket, &msg, iov, 2, len);
@@ -660,7 +660,7 @@ static int rxkad_send_response(struct rxrpc_connection *conn,
len = iov[0].iov_len + iov[1].iov_len + iov[2].iov_len;
- hdr->serial = htonl(atomic_inc_return(&conn->serial));
+ hdr->serial = htonl(atomic_inc_return_unchecked(&conn->serial));
_proto("Tx RESPONSE %%%u", ntohl(hdr->serial));
ret = kernel_sendmsg(conn->trans->local->socket, &msg, iov, 3, len);
diff --git a/net/sched/sch_generic.c b/net/sched/sch_generic.c
index 6efca30..1259f82 100644
--- a/net/sched/sch_generic.c
+++ b/net/sched/sch_generic.c
@@ -349,7 +349,7 @@ void netif_carrier_on(struct net_device *dev)
if (test_and_clear_bit(__LINK_STATE_NOCARRIER, &dev->state)) {
if (dev->reg_state == NETREG_UNINITIALIZED)
return;
- atomic_inc(&dev->carrier_changes);
+ atomic_inc_unchecked(&dev->carrier_changes);
linkwatch_fire_event(dev);
if (netif_running(dev))
__netdev_watchdog_up(dev);
@@ -368,7 +368,7 @@ void netif_carrier_off(struct net_device *dev)
if (!test_and_set_bit(__LINK_STATE_NOCARRIER, &dev->state)) {
if (dev->reg_state == NETREG_UNINITIALIZED)
return;
- atomic_inc(&dev->carrier_changes);
+ atomic_inc_unchecked(&dev->carrier_changes);
linkwatch_fire_event(dev);
}
}
diff --git a/net/sunrpc/auth_gss/svcauth_gss.c b/net/sunrpc/auth_gss/svcauth_gss.c
index 1095be9..815d777 100644
--- a/net/sunrpc/auth_gss/svcauth_gss.c
+++ b/net/sunrpc/auth_gss/svcauth_gss.c
@@ -1142,7 +1142,7 @@ static int gss_proxy_save_rsc(struct cache_detail *cd,
uint64_t *handle)
{
struct rsc rsci, *rscp = NULL;
- static atomic64_t ctxhctr;
+ static atomic64_unchecked_t ctxhctr = ATOMIC64_INIT(0);
long long ctxh;
struct gss_api_mech *gm = NULL;
time_t expiry;
@@ -1153,7 +1153,7 @@ static int gss_proxy_save_rsc(struct cache_detail *cd,
status = -ENOMEM;
/* the handle needs to be just a unique id,
* use a static counter */
- ctxh = atomic64_inc_return(&ctxhctr);
+ ctxh = atomic64_inc_return_unchecked(&ctxhctr);
/* make a copy for the caller */
*handle = ctxh;
diff --git a/net/sunrpc/sched.c b/net/sunrpc/sched.c
index 337ca85..42feae8 100644
--- a/net/sunrpc/sched.c
+++ b/net/sunrpc/sched.c
@@ -261,9 +261,9 @@ static int rpc_wait_bit_killable(struct wait_bit_key *key)
#if IS_ENABLED(CONFIG_SUNRPC_DEBUG) || IS_ENABLED(CONFIG_TRACEPOINTS)
static void rpc_task_set_debuginfo(struct rpc_task *task)
{
- static atomic_t rpc_pid;
+ static atomic_unchecked_t rpc_pid;
- task->tk_pid = atomic_inc_return(&rpc_pid);
+ task->tk_pid = atomic_inc_return_unchecked(&rpc_pid);
}
#else
static inline void rpc_task_set_debuginfo(struct rpc_task *task)
diff --git a/net/sunrpc/xprtrdma/svc_rdma.c b/net/sunrpc/xprtrdma/svc_rdma.c
index 2cd252f..f606982 100644
--- a/net/sunrpc/xprtrdma/svc_rdma.c
+++ b/net/sunrpc/xprtrdma/svc_rdma.c
@@ -61,15 +61,15 @@ unsigned int svcrdma_max_req_size = RPCRDMA_MAX_REQ_SIZE;
static unsigned int min_max_inline = 4096;
static unsigned int max_max_inline = 65536;
-atomic_t rdma_stat_recv;
-atomic_t rdma_stat_read;
-atomic_t rdma_stat_write;
-atomic_t rdma_stat_sq_starve;
-atomic_t rdma_stat_rq_starve;
-atomic_t rdma_stat_rq_poll;
-atomic_t rdma_stat_rq_prod;
-atomic_t rdma_stat_sq_poll;
-atomic_t rdma_stat_sq_prod;
+atomic_unchecked_t rdma_stat_recv;
+atomic_unchecked_t rdma_stat_read;
+atomic_unchecked_t rdma_stat_write;
+atomic_unchecked_t rdma_stat_sq_starve;
+atomic_unchecked_t rdma_stat_rq_starve;
+atomic_unchecked_t rdma_stat_rq_poll;
+atomic_unchecked_t rdma_stat_rq_prod;
+atomic_unchecked_t rdma_stat_sq_poll;
+atomic_unchecked_t rdma_stat_sq_prod;
/* Temporary NFS request map and context caches */
struct kmem_cache *svc_rdma_map_cachep;
@@ -150,63 +150,63 @@ static struct ctl_table svcrdma_parm_table[] = {
{
.procname = "rdma_stat_read",
.data = &rdma_stat_read,
- .maxlen = sizeof(atomic_t),
+ .maxlen = sizeof(atomic_unchecked_t),
.mode = 0644,
.proc_handler = read_reset_stat,
},
{
.procname = "rdma_stat_recv",
.data = &rdma_stat_recv,
- .maxlen = sizeof(atomic_t),
+ .maxlen = sizeof(atomic_unchecked_t),
.mode = 0644,
.proc_handler = read_reset_stat,
},
{
.procname = "rdma_stat_write",
.data = &rdma_stat_write,
- .maxlen = sizeof(atomic_t),
+ .maxlen = sizeof(atomic_unchecked_t),
.mode = 0644,
.proc_handler = read_reset_stat,
},
{
.procname = "rdma_stat_sq_starve",
.data = &rdma_stat_sq_starve,
- .maxlen = sizeof(atomic_t),
+ .maxlen = sizeof(atomic_unchecked_t),
.mode = 0644,
.proc_handler = read_reset_stat,
},
{
.procname = "rdma_stat_rq_starve",
.data = &rdma_stat_rq_starve,
- .maxlen = sizeof(atomic_t),
+ .maxlen = sizeof(atomic_unchecked_t),
.mode = 0644,
.proc_handler = read_reset_stat,
},
{
.procname = "rdma_stat_rq_poll",
.data = &rdma_stat_rq_poll,
- .maxlen = sizeof(atomic_t),
+ .maxlen = sizeof(atomic_unchecked_t),
.mode = 0644,
.proc_handler = read_reset_stat,
},
{
.procname = "rdma_stat_rq_prod",
.data = &rdma_stat_rq_prod,
- .maxlen = sizeof(atomic_t),
+ .maxlen = sizeof(atomic_unchecked_t),
.mode = 0644,
.proc_handler = read_reset_stat,
},
{
.procname = "rdma_stat_sq_poll",
.data = &rdma_stat_sq_poll,
- .maxlen = sizeof(atomic_t),
+ .maxlen = sizeof(atomic_unchecked_t),
.mode = 0644,
.proc_handler = read_reset_stat,
},
{
.procname = "rdma_stat_sq_prod",
.data = &rdma_stat_sq_prod,
- .maxlen = sizeof(atomic_t),
+ .maxlen = sizeof(atomic_unchecked_t),
.mode = 0644,
.proc_handler = read_reset_stat,
},
diff --git a/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c b/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c
index 96d886a..35a2137 100644
--- a/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c
+++ b/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c
@@ -210,7 +210,7 @@ int rdma_read_chunk_lcl(struct svcxprt_rdma *xprt,
*page_no = pg_no;
*page_offset = pg_off;
ret = read;
- atomic_inc(&rdma_stat_read);
+ atomic_inc_unchecked(&rdma_stat_read);
return ret;
err:
svc_rdma_unmap_dma(ctxt);
@@ -347,7 +347,7 @@ int rdma_read_chunk_frmr(struct svcxprt_rdma *xprt,
*page_no = pg_no;
*page_offset = pg_off;
ret = read;
- atomic_inc(&rdma_stat_read);
+ atomic_inc_unchecked(&rdma_stat_read);
return ret;
err:
svc_rdma_unmap_dma(ctxt);
@@ -601,7 +601,7 @@ int svc_rdma_recvfrom(struct svc_rqst *rqstp)
dto_q);
list_del_init(&ctxt->dto_q);
} else {
- atomic_inc(&rdma_stat_rq_starve);
+ atomic_inc_unchecked(&rdma_stat_rq_starve);
clear_bit(XPT_DATA, &xprt->xpt_flags);
ctxt = NULL;
}
@@ -619,7 +619,7 @@ int svc_rdma_recvfrom(struct svc_rqst *rqstp)
}
dprintk("svcrdma: processing ctxt=%p on xprt=%p, rqstp=%p, status=%d\n",
ctxt, rdma_xprt, rqstp, ctxt->wc_status);
- atomic_inc(&rdma_stat_recv);
+ atomic_inc_unchecked(&rdma_stat_recv);
/* Build up the XDR from the receive buffers. */
rdma_build_arg_xdr(rqstp, ctxt, ctxt->byte_len);
diff --git a/net/sunrpc/xprtrdma/svc_rdma_sendto.c b/net/sunrpc/xprtrdma/svc_rdma_sendto.c
index 95412ab..29e8f37 100644
--- a/net/sunrpc/xprtrdma/svc_rdma_sendto.c
+++ b/net/sunrpc/xprtrdma/svc_rdma_sendto.c
@@ -218,7 +218,7 @@ static int send_write(struct svcxprt_rdma *xprt, struct svc_rqst *rqstp,
write_wr.wr.rdma.remote_addr = to;
/* Post It */
- atomic_inc(&rdma_stat_write);
+ atomic_inc_unchecked(&rdma_stat_write);
if (svc_rdma_send(xprt, &write_wr))
goto err;
return write_len - bc;
diff --git a/net/sunrpc/xprtrdma/svc_rdma_transport.c b/net/sunrpc/xprtrdma/svc_rdma_transport.c
index 48f6de9..e89176b 100644
--- a/net/sunrpc/xprtrdma/svc_rdma_transport.c
+++ b/net/sunrpc/xprtrdma/svc_rdma_transport.c
@@ -288,7 +288,7 @@ static void rq_cq_reap(struct svcxprt_rdma *xprt)
return;
ib_req_notify_cq(xprt->sc_rq_cq, IB_CQ_NEXT_COMP);
- atomic_inc(&rdma_stat_rq_poll);
+ atomic_inc_unchecked(&rdma_stat_rq_poll);
while ((ret = ib_poll_cq(xprt->sc_rq_cq, 1, &wc)) > 0) {
ctxt = (struct svc_rdma_op_ctxt *)(unsigned long)wc.wr_id;
@@ -310,7 +310,7 @@ static void rq_cq_reap(struct svcxprt_rdma *xprt)
}
if (ctxt)
- atomic_inc(&rdma_stat_rq_prod);
+ atomic_inc_unchecked(&rdma_stat_rq_prod);
set_bit(XPT_DATA, &xprt->sc_xprt.xpt_flags);
/*
@@ -389,7 +389,7 @@ static void sq_cq_reap(struct svcxprt_rdma *xprt)
return;
ib_req_notify_cq(xprt->sc_sq_cq, IB_CQ_NEXT_COMP);
- atomic_inc(&rdma_stat_sq_poll);
+ atomic_inc_unchecked(&rdma_stat_sq_poll);
while ((ret = ib_poll_cq(cq, ARRAY_SIZE(wc_a), wc_a)) > 0) {
int i;
@@ -418,7 +418,7 @@ static void sq_cq_reap(struct svcxprt_rdma *xprt)
}
if (ctxt)
- atomic_inc(&rdma_stat_sq_prod);
+ atomic_inc_unchecked(&rdma_stat_sq_prod);
}
static void sq_comp_handler(struct ib_cq *cq, void *cq_context)
@@ -1254,7 +1254,7 @@ int svc_rdma_send(struct svcxprt_rdma *xprt, struct ib_send_wr *wr)
spin_lock_bh(&xprt->sc_lock);
if (xprt->sc_sq_depth < atomic_read(&xprt->sc_sq_count) + wr_count) {
spin_unlock_bh(&xprt->sc_lock);
- atomic_inc(&rdma_stat_sq_starve);
+ atomic_inc_unchecked(&rdma_stat_sq_starve);
/* See if we can opportunistically reap SQ WR to make room */
sq_cq_reap(xprt);
diff --git a/net/xfrm/xfrm_policy.c b/net/xfrm/xfrm_policy.c
index 18cead7..3ecb199 100644
--- a/net/xfrm/xfrm_policy.c
+++ b/net/xfrm/xfrm_policy.c
@@ -323,7 +323,7 @@ static void xfrm_policy_kill(struct xfrm_policy *policy)
{
policy->walk.dead = 1;
- atomic_inc(&policy->genid);
+ atomic_inc_unchecked(&policy->genid);
if (del_timer(&policy->polq.hold_timer))
xfrm_pol_put(policy);
@@ -773,7 +773,7 @@ int xfrm_policy_insert(int dir, struct xfrm_policy *policy, int excl)
else
hlist_add_head(&policy->bydst, chain);
__xfrm_policy_link(policy, dir);
- atomic_inc(&net->xfrm.flow_cache_genid);
+ atomic_inc_unchecked(&net->xfrm.flow_cache_genid);
/* After previous checking, family can either be AF_INET or AF_INET6 */
if (policy->family == AF_INET)
@@ -1874,7 +1874,7 @@ xfrm_resolve_and_create_bundle(struct xfrm_policy **pols, int num_pols,
xdst->num_pols = num_pols;
memcpy(xdst->pols, pols, sizeof(struct xfrm_policy *) * num_pols);
- xdst->policy_genid = atomic_read(&pols[0]->genid);
+ xdst->policy_genid = atomic_read_unchecked(&pols[0]->genid);
return xdst;
}
@@ -2740,7 +2740,7 @@ static int xfrm_bundle_ok(struct xfrm_dst *first)
if (xdst->xfrm_genid != dst->xfrm->genid)
return 0;
if (xdst->num_pols > 0 &&
- xdst->policy_genid != atomic_read(&xdst->pols[0]->genid))
+ xdst->policy_genid != atomic_read_unchecked(&xdst->pols[0]->genid))
return 0;
mtu = dst_mtu(dst->child);
@@ -3281,7 +3281,7 @@ static int xfrm_policy_migrate(struct xfrm_policy *pol,
sizeof(pol->xfrm_vec[i].saddr));
pol->xfrm_vec[i].encap_family = mp->new_family;
/* flush bundles */
- atomic_inc(&pol->genid);
+ atomic_inc_unchecked(&pol->genid);
}
}
diff --git a/net/xfrm/xfrm_state.c b/net/xfrm/xfrm_state.c
index 9895a8c..2908c63 100644
--- a/net/xfrm/xfrm_state.c
+++ b/net/xfrm/xfrm_state.c
@@ -1505,10 +1505,10 @@ EXPORT_SYMBOL(xfrm_find_acq_byseq);
u32 xfrm_get_acqseq(void)
{
u32 res;
- static atomic_t acqseq;
+ static atomic_unchecked_t acqseq;
do {
- res = atomic_inc_return(&acqseq);
+ res = atomic_inc_return_unchecked(&acqseq);
} while (!res);
return res;
--
2.5.0
^ permalink raw reply related [flat|nested] 32+ messages in thread* [kernel-hardening][RFC PATCH v2 07/12] security: opt out of PAX_REFCOUNT protection
2015-12-17 14:57 [kernel-hardening][RFC PATCH v2 00/12] Add PAX_REFCOUNT overflow protection David Windsor
` (5 preceding siblings ...)
2015-12-17 14:57 ` [kernel-hardening][RFC PATCH v2 06/12] net: " David Windsor
@ 2015-12-17 14:57 ` David Windsor
2015-12-17 14:57 ` [kernel-hardening][RFC PATCH v2 08/12] drivers: opt out of PAX_REFCOUNT protection (1 of 2) David Windsor
` (5 subsequent siblings)
12 siblings, 0 replies; 32+ messages in thread
From: David Windsor @ 2015-12-17 14:57 UTC (permalink / raw)
To: kernel-hardening; +Cc: David Windsor
Signed-off-by: David Windsor <dave@progbits.org>
---
security/integrity/ima/ima.h | 4 ++--
security/integrity/ima/ima_api.c | 2 +-
security/integrity/ima/ima_fs.c | 4 ++--
security/integrity/ima/ima_queue.c | 2 +-
security/selinux/avc.c | 6 +++---
security/selinux/include/xfrm.h | 2 +-
6 files changed, 10 insertions(+), 10 deletions(-)
diff --git a/security/integrity/ima/ima.h b/security/integrity/ima/ima.h
index e2a60c3..2e41dd3 100644
--- a/security/integrity/ima/ima.h
+++ b/security/integrity/ima/ima.h
@@ -125,8 +125,8 @@ int ima_init_template(void);
extern spinlock_t ima_queue_lock;
struct ima_h_table {
- atomic_long_t len; /* number of stored measurements in the list */
- atomic_long_t violations;
+ atomic_long_unchecked_t len; /* number of stored measurements in the list */
+ atomic_long_unchecked_t violations;
struct hlist_head queue[IMA_MEASURE_HTABLE_SIZE];
};
extern struct ima_h_table ima_htable;
diff --git a/security/integrity/ima/ima_api.c b/security/integrity/ima/ima_api.c
index 1d950fb..a8f4eab 100644
--- a/security/integrity/ima/ima_api.c
+++ b/security/integrity/ima/ima_api.c
@@ -137,7 +137,7 @@ void ima_add_violation(struct file *file, const unsigned char *filename,
int result;
/* can overflow, only indicator */
- atomic_long_inc(&ima_htable.violations);
+ atomic_long_inc_unchecked(&ima_htable.violations);
result = ima_alloc_init_template(&event_data, &entry);
if (result < 0) {
diff --git a/security/integrity/ima/ima_fs.c b/security/integrity/ima/ima_fs.c
index 816d175..20079ae 100644
--- a/security/integrity/ima/ima_fs.c
+++ b/security/integrity/ima/ima_fs.c
@@ -28,12 +28,12 @@
static int valid_policy = 1;
#define TMPBUFLEN 12
static ssize_t ima_show_htable_value(char __user *buf, size_t count,
- loff_t *ppos, atomic_long_t *val)
+ loff_t *ppos, atomic_long_unchecked_t *val)
{
char tmpbuf[TMPBUFLEN];
ssize_t len;
- len = scnprintf(tmpbuf, TMPBUFLEN, "%li\n", atomic_long_read(val));
+ len = scnprintf(tmpbuf, TMPBUFLEN, "%li\n", atomic_long_read_unchecked(val));
return simple_read_from_buffer(buf, count, ppos, tmpbuf, len);
}
diff --git a/security/integrity/ima/ima_queue.c b/security/integrity/ima/ima_queue.c
index 552705d..9920f4fb 100644
--- a/security/integrity/ima/ima_queue.c
+++ b/security/integrity/ima/ima_queue.c
@@ -83,7 +83,7 @@ static int ima_add_digest_entry(struct ima_template_entry *entry)
INIT_LIST_HEAD(&qe->later);
list_add_tail_rcu(&qe->later, &ima_measurements);
- atomic_long_inc(&ima_htable.len);
+ atomic_long_inc_unchecked(&ima_htable.len);
key = ima_hash_key(entry->digest);
hlist_add_head_rcu(&qe->hnext, &ima_htable.queue[key]);
return 0;
diff --git a/security/selinux/avc.c b/security/selinux/avc.c
index 0b122b1..d9f9568 100644
--- a/security/selinux/avc.c
+++ b/security/selinux/avc.c
@@ -59,7 +59,7 @@ struct avc_node {
struct avc_cache {
struct hlist_head slots[AVC_CACHE_SLOTS]; /* head for avc_node->list */
spinlock_t slots_lock[AVC_CACHE_SLOTS]; /* lock for writes */
- atomic_t lru_hint; /* LRU hint for reclaim scan */
+ atomic_unchecked_t lru_hint; /* LRU hint for reclaim scan */
atomic_t active_nodes;
u32 latest_notif; /* latest revocation notification */
};
@@ -167,7 +167,7 @@ void __init avc_init(void)
spin_lock_init(&avc_cache.slots_lock[i]);
}
atomic_set(&avc_cache.active_nodes, 0);
- atomic_set(&avc_cache.lru_hint, 0);
+ atomic_set_unchecked(&avc_cache.lru_hint, 0);
avc_node_cachep = kmem_cache_create("avc_node", sizeof(struct avc_node),
0, SLAB_PANIC, NULL);
@@ -242,7 +242,7 @@ static inline int avc_reclaim_node(void)
spinlock_t *lock;
for (try = 0, ecx = 0; try < AVC_CACHE_SLOTS; try++) {
- hvalue = atomic_inc_return(&avc_cache.lru_hint) & (AVC_CACHE_SLOTS - 1);
+ hvalue = atomic_inc_return_unchecked(&avc_cache.lru_hint) & (AVC_CACHE_SLOTS - 1);
head = &avc_cache.slots[hvalue];
lock = &avc_cache.slots_lock[hvalue];
diff --git a/security/selinux/include/xfrm.h b/security/selinux/include/xfrm.h
index 1450f85..a91e0bc 100644
--- a/security/selinux/include/xfrm.h
+++ b/security/selinux/include/xfrm.h
@@ -48,7 +48,7 @@ static inline void selinux_xfrm_notify_policyload(void)
rtnl_lock();
for_each_net(net) {
- atomic_inc(&net->xfrm.flow_cache_genid);
+ atomic_inc_unchecked(&net->xfrm.flow_cache_genid);
rt_genid_bump_all(net);
}
rtnl_unlock();
--
2.5.0
^ permalink raw reply related [flat|nested] 32+ messages in thread* [kernel-hardening][RFC PATCH v2 08/12] drivers: opt out of PAX_REFCOUNT protection (1 of 2)
2015-12-17 14:57 [kernel-hardening][RFC PATCH v2 00/12] Add PAX_REFCOUNT overflow protection David Windsor
` (6 preceding siblings ...)
2015-12-17 14:57 ` [kernel-hardening][RFC PATCH v2 07/12] security: " David Windsor
@ 2015-12-17 14:57 ` David Windsor
2015-12-17 14:57 ` [kernel-hardening][RFC PATCH v2 09/12] drivers: opt out of PAX_REFCOUNT protection (2 " David Windsor
` (4 subsequent siblings)
12 siblings, 0 replies; 32+ messages in thread
From: David Windsor @ 2015-12-17 14:57 UTC (permalink / raw)
To: kernel-hardening; +Cc: David Windsor
Signed-off-by: David Windsor <dave@progbits.org>
---
drivers/acpi/apei/ghes.c | 4 +-
drivers/ata/libata-core.c | 4 +-
drivers/ata/libata-scsi.c | 2 +-
drivers/ata/libata.h | 2 +-
drivers/atm/adummy.c | 2 +-
drivers/atm/ambassador.c | 8 +-
drivers/atm/atmtcp.c | 14 +--
drivers/atm/eni.c | 10 +-
drivers/atm/firestream.c | 8 +-
drivers/atm/fore200e.c | 14 +--
drivers/atm/he.c | 18 ++--
drivers/atm/horizon.c | 4 +-
drivers/atm/idt77252.c | 36 +++----
drivers/atm/iphase.c | 34 +++----
drivers/atm/lanai.c | 12 +--
drivers/atm/nicstar.c | 46 ++++-----
drivers/atm/solos-pci.c | 4 +-
drivers/atm/suni.c | 4 +-
drivers/atm/uPD98402.c | 16 +--
drivers/atm/zatm.c | 6 +-
drivers/base/power/wakeup.c | 8 +-
drivers/block/drbd/drbd_bitmap.c | 2 +-
drivers/block/drbd/drbd_int.h | 8 +-
drivers/block/drbd/drbd_main.c | 12 +--
drivers/block/drbd/drbd_nl.c | 4 +-
drivers/block/drbd/drbd_receiver.c | 30 +++---
drivers/block/drbd/drbd_worker.c | 8 +-
drivers/char/ipmi/ipmi_msghandler.c | 8 +-
drivers/char/ipmi/ipmi_si_intf.c | 8 +-
drivers/crypto/hifn_795x.c | 4 +-
drivers/edac/edac_device.c | 4 +-
drivers/edac/edac_pci.c | 4 +-
drivers/edac/edac_pci_sysfs.c | 20 ++--
drivers/firewire/core-card.c | 4 +-
drivers/firmware/efi/cper.c | 8 +-
drivers/gpio/gpio-vr41xx.c | 2 +-
drivers/gpu/drm/i810/i810_drv.h | 4 +-
drivers/gpu/drm/mga/mga_drv.h | 4 +-
drivers/gpu/drm/mga/mga_irq.c | 8 +-
drivers/gpu/drm/qxl/qxl_cmd.c | 12 +--
drivers/gpu/drm/qxl/qxl_debugfs.c | 8 +-
drivers/gpu/drm/qxl/qxl_drv.h | 8 +-
drivers/gpu/drm/qxl/qxl_irq.c | 16 +--
drivers/gpu/drm/r128/r128_cce.c | 2 +-
drivers/gpu/drm/r128/r128_drv.h | 4 +-
drivers/gpu/drm/r128/r128_irq.c | 4 +-
drivers/gpu/drm/r128/r128_state.c | 4 +-
drivers/gpu/drm/radeon/radeon_drv.h | 2 +-
drivers/gpu/drm/radeon/radeon_irq.c | 6 +-
drivers/gpu/drm/via/via_drv.h | 4 +-
drivers/gpu/drm/via/via_irq.c | 18 ++--
drivers/gpu/drm/vmwgfx/vmwgfx_drv.h | 2 +-
drivers/gpu/drm/vmwgfx/vmwgfx_fifo.c | 6 +-
drivers/gpu/drm/vmwgfx/vmwgfx_irq.c | 4 +-
drivers/gpu/drm/vmwgfx/vmwgfx_marker.c | 2 +-
drivers/hid/hid-core.c | 4 +-
drivers/hv/channel.c | 2 +-
drivers/hv/hv_balloon.c | 18 ++--
drivers/hv/hyperv_vmbus.h | 2 +-
drivers/hwmon/sht15.c | 12 +--
drivers/infiniband/core/cm.c | 32 +++---
drivers/infiniband/core/fmr_pool.c | 20 ++--
drivers/infiniband/hw/cxgb4/mem.c | 4 +-
drivers/infiniband/hw/ipath/ipath_rc.c | 6 +-
drivers/infiniband/hw/ipath/ipath_ruc.c | 6 +-
drivers/infiniband/hw/mlx4/mad.c | 2 +-
drivers/infiniband/hw/mlx4/mcg.c | 2 +-
drivers/infiniband/hw/mlx4/mlx4_ib.h | 2 +-
drivers/infiniband/hw/nes/nes.c | 4 +-
drivers/infiniband/hw/nes/nes.h | 40 ++++----
drivers/infiniband/hw/nes/nes_cm.c | 62 ++++++------
drivers/infiniband/hw/nes/nes_mgt.c | 8 +-
drivers/infiniband/hw/nes/nes_nic.c | 40 ++++----
drivers/infiniband/hw/nes/nes_verbs.c | 10 +-
drivers/input/gameport/gameport.c | 4 +-
drivers/input/input.c | 4 +-
drivers/input/joystick/xpad.c | 4 +-
drivers/input/misc/ims-pcu.c | 4 +-
drivers/input/serio/serio.c | 4 +-
drivers/input/serio/serio_raw.c | 4 +-
drivers/isdn/capi/capi.c | 10 +-
drivers/md/dm-raid1.c | 18 ++--
drivers/md/dm-stripe.c | 10 +-
drivers/md/dm.c | 16 +--
drivers/md/md.c | 26 ++---
drivers/md/md.h | 6 +-
drivers/md/raid1.c | 4 +-
drivers/md/raid10.c | 16 +--
drivers/md/raid5.c | 10 +-
drivers/media/pci/ivtv/ivtv-driver.c | 2 +-
drivers/media/pci/solo6x10/solo6x10-p2m.c | 2 +-
drivers/media/pci/solo6x10/solo6x10.h | 2 +-
drivers/media/pci/tw68/tw68-core.c | 2 +-
drivers/media/radio/radio-maxiradio.c | 2 +-
drivers/media/radio/radio-shark.c | 2 +-
drivers/media/radio/radio-shark2.c | 2 +-
drivers/media/radio/radio-si476x.c | 2 +-
drivers/media/v4l2-core/v4l2-device.c | 4 +-
drivers/misc/lis3lv02d/lis3lv02d.c | 8 +-
drivers/misc/lis3lv02d/lis3lv02d.h | 2 +-
drivers/misc/sgi-gru/gruhandles.c | 4 +-
drivers/misc/sgi-gru/gruprocfs.c | 8 +-
drivers/misc/sgi-gru/grutables.h | 158 +++++++++++++++---------------
drivers/net/hyperv/hyperv_net.h | 2 +-
drivers/net/hyperv/rndis_filter.c | 4 +-
105 files changed, 561 insertions(+), 561 deletions(-)
diff --git a/drivers/acpi/apei/ghes.c b/drivers/acpi/apei/ghes.c
index 2bfd53c..391e9a4 100644
--- a/drivers/acpi/apei/ghes.c
+++ b/drivers/acpi/apei/ghes.c
@@ -478,7 +478,7 @@ static void __ghes_print_estatus(const char *pfx,
const struct acpi_hest_generic *generic,
const struct acpi_hest_generic_status *estatus)
{
- static atomic_t seqno;
+ static atomic_unchecked_t seqno;
unsigned int curr_seqno;
char pfx_seq[64];
@@ -489,7 +489,7 @@ static void __ghes_print_estatus(const char *pfx,
else
pfx = KERN_ERR;
}
- curr_seqno = atomic_inc_return(&seqno);
+ curr_seqno = atomic_inc_return_unchecked(&seqno);
snprintf(pfx_seq, sizeof(pfx_seq), "%s{%u}" HW_ERR, pfx, curr_seqno);
printk("%s""Hardware error from APEI Generic Hardware Error Source: %d\n",
pfx_seq, generic->header.source_id);
diff --git a/drivers/ata/libata-core.c b/drivers/ata/libata-core.c
index 790e0de..24718d0 100644
--- a/drivers/ata/libata-core.c
+++ b/drivers/ata/libata-core.c
@@ -102,7 +102,7 @@ static unsigned int ata_dev_set_xfermode(struct ata_device *dev);
static void ata_dev_xfermask(struct ata_device *dev);
static unsigned long ata_dev_blacklisted(const struct ata_device *dev);
-atomic_t ata_print_id = ATOMIC_INIT(0);
+atomic_unchecked_t ata_print_id = ATOMIC_INIT(0);
struct ata_force_param {
const char *name;
@@ -6134,7 +6134,7 @@ int ata_host_register(struct ata_host *host, struct scsi_host_template *sht)
/* give ports names and add SCSI hosts */
for (i = 0; i < host->n_ports; i++) {
- host->ports[i]->print_id = atomic_inc_return(&ata_print_id);
+ host->ports[i]->print_id = atomic_inc_return_unchecked(&ata_print_id);
host->ports[i]->local_port_no = i + 1;
}
diff --git a/drivers/ata/libata-scsi.c b/drivers/ata/libata-scsi.c
index 0d7f0da..bc20aa6 100644
--- a/drivers/ata/libata-scsi.c
+++ b/drivers/ata/libata-scsi.c
@@ -4193,7 +4193,7 @@ int ata_sas_port_init(struct ata_port *ap)
if (rc)
return rc;
- ap->print_id = atomic_inc_return(&ata_print_id);
+ ap->print_id = atomic_inc_return_unchecked(&ata_print_id);
return 0;
}
EXPORT_SYMBOL_GPL(ata_sas_port_init);
diff --git a/drivers/ata/libata.h b/drivers/ata/libata.h
index f840ca1..edd6ef3 100644
--- a/drivers/ata/libata.h
+++ b/drivers/ata/libata.h
@@ -53,7 +53,7 @@ enum {
ATA_DNXFER_QUIET = (1 << 31),
};
-extern atomic_t ata_print_id;
+extern atomic_unchecked_t ata_print_id;
extern int atapi_passthru16;
extern int libata_fua;
extern int libata_noacpi;
diff --git a/drivers/atm/adummy.c b/drivers/atm/adummy.c
index f9b983a..887b9d8 100644
--- a/drivers/atm/adummy.c
+++ b/drivers/atm/adummy.c
@@ -114,7 +114,7 @@ adummy_send(struct atm_vcc *vcc, struct sk_buff *skb)
vcc->pop(vcc, skb);
else
dev_kfree_skb_any(skb);
- atomic_inc(&vcc->stats->tx);
+ atomic_inc_unchecked(&vcc->stats->tx);
return 0;
}
diff --git a/drivers/atm/ambassador.c b/drivers/atm/ambassador.c
index f1a9198..f466a4a 100644
--- a/drivers/atm/ambassador.c
+++ b/drivers/atm/ambassador.c
@@ -454,7 +454,7 @@ static void tx_complete (amb_dev * dev, tx_out * tx) {
PRINTD (DBG_FLOW|DBG_TX, "tx_complete %p %p", dev, tx);
// VC layer stats
- atomic_inc(&ATM_SKB(skb)->vcc->stats->tx);
+ atomic_inc_unchecked(&ATM_SKB(skb)->vcc->stats->tx);
// free the descriptor
kfree (tx_descr);
@@ -495,7 +495,7 @@ static void rx_complete (amb_dev * dev, rx_out * rx) {
dump_skb ("<<<", vc, skb);
// VC layer stats
- atomic_inc(&atm_vcc->stats->rx);
+ atomic_inc_unchecked(&atm_vcc->stats->rx);
__net_timestamp(skb);
// end of our responsibility
atm_vcc->push (atm_vcc, skb);
@@ -510,7 +510,7 @@ static void rx_complete (amb_dev * dev, rx_out * rx) {
} else {
PRINTK (KERN_INFO, "dropped over-size frame");
// should we count this?
- atomic_inc(&atm_vcc->stats->rx_drop);
+ atomic_inc_unchecked(&atm_vcc->stats->rx_drop);
}
} else {
@@ -1338,7 +1338,7 @@ static int amb_send (struct atm_vcc * atm_vcc, struct sk_buff * skb) {
}
if (check_area (skb->data, skb->len)) {
- atomic_inc(&atm_vcc->stats->tx_err);
+ atomic_inc_unchecked(&atm_vcc->stats->tx_err);
return -ENOMEM; // ?
}
diff --git a/drivers/atm/atmtcp.c b/drivers/atm/atmtcp.c
index 480fa6f..947067c 100644
--- a/drivers/atm/atmtcp.c
+++ b/drivers/atm/atmtcp.c
@@ -206,7 +206,7 @@ static int atmtcp_v_send(struct atm_vcc *vcc,struct sk_buff *skb)
if (vcc->pop) vcc->pop(vcc,skb);
else dev_kfree_skb(skb);
if (dev_data) return 0;
- atomic_inc(&vcc->stats->tx_err);
+ atomic_inc_unchecked(&vcc->stats->tx_err);
return -ENOLINK;
}
size = skb->len+sizeof(struct atmtcp_hdr);
@@ -214,7 +214,7 @@ static int atmtcp_v_send(struct atm_vcc *vcc,struct sk_buff *skb)
if (!new_skb) {
if (vcc->pop) vcc->pop(vcc,skb);
else dev_kfree_skb(skb);
- atomic_inc(&vcc->stats->tx_err);
+ atomic_inc_unchecked(&vcc->stats->tx_err);
return -ENOBUFS;
}
hdr = (void *) skb_put(new_skb,sizeof(struct atmtcp_hdr));
@@ -225,8 +225,8 @@ static int atmtcp_v_send(struct atm_vcc *vcc,struct sk_buff *skb)
if (vcc->pop) vcc->pop(vcc,skb);
else dev_kfree_skb(skb);
out_vcc->push(out_vcc,new_skb);
- atomic_inc(&vcc->stats->tx);
- atomic_inc(&out_vcc->stats->rx);
+ atomic_inc_unchecked(&vcc->stats->tx);
+ atomic_inc_unchecked(&out_vcc->stats->rx);
return 0;
}
@@ -300,7 +300,7 @@ static int atmtcp_c_send(struct atm_vcc *vcc,struct sk_buff *skb)
read_unlock(&vcc_sklist_lock);
if (!out_vcc) {
result = -EUNATCH;
- atomic_inc(&vcc->stats->tx_err);
+ atomic_inc_unchecked(&vcc->stats->tx_err);
goto done;
}
skb_pull(skb,sizeof(struct atmtcp_hdr));
@@ -312,8 +312,8 @@ static int atmtcp_c_send(struct atm_vcc *vcc,struct sk_buff *skb)
__net_timestamp(new_skb);
skb_copy_from_linear_data(skb, skb_put(new_skb, skb->len), skb->len);
out_vcc->push(out_vcc,new_skb);
- atomic_inc(&vcc->stats->tx);
- atomic_inc(&out_vcc->stats->rx);
+ atomic_inc_unchecked(&vcc->stats->tx);
+ atomic_inc_unchecked(&out_vcc->stats->rx);
done:
if (vcc->pop) vcc->pop(vcc,skb);
else dev_kfree_skb(skb);
diff --git a/drivers/atm/eni.c b/drivers/atm/eni.c
index 6339efd..2b441d5 100644
--- a/drivers/atm/eni.c
+++ b/drivers/atm/eni.c
@@ -525,7 +525,7 @@ static int rx_aal0(struct atm_vcc *vcc)
DPRINTK(DEV_LABEL "(itf %d): trashing empty cell\n",
vcc->dev->number);
length = 0;
- atomic_inc(&vcc->stats->rx_err);
+ atomic_inc_unchecked(&vcc->stats->rx_err);
}
else {
length = ATM_CELL_SIZE-1; /* no HEC */
@@ -580,7 +580,7 @@ static int rx_aal5(struct atm_vcc *vcc)
size);
}
eff = length = 0;
- atomic_inc(&vcc->stats->rx_err);
+ atomic_inc_unchecked(&vcc->stats->rx_err);
}
else {
size = (descr & MID_RED_COUNT)*(ATM_CELL_PAYLOAD >> 2);
@@ -597,7 +597,7 @@ static int rx_aal5(struct atm_vcc *vcc)
"(VCI=%d,length=%ld,size=%ld (descr 0x%lx))\n",
vcc->dev->number,vcc->vci,length,size << 2,descr);
length = eff = 0;
- atomic_inc(&vcc->stats->rx_err);
+ atomic_inc_unchecked(&vcc->stats->rx_err);
}
}
skb = eff ? atm_alloc_charge(vcc,eff << 2,GFP_ATOMIC) : NULL;
@@ -770,7 +770,7 @@ rx_dequeued++;
vcc->push(vcc,skb);
pushed++;
}
- atomic_inc(&vcc->stats->rx);
+ atomic_inc_unchecked(&vcc->stats->rx);
}
wake_up(&eni_dev->rx_wait);
}
@@ -1230,7 +1230,7 @@ static void dequeue_tx(struct atm_dev *dev)
DMA_TO_DEVICE);
if (vcc->pop) vcc->pop(vcc,skb);
else dev_kfree_skb_irq(skb);
- atomic_inc(&vcc->stats->tx);
+ atomic_inc_unchecked(&vcc->stats->tx);
wake_up(&eni_dev->tx_wait);
dma_complete++;
}
diff --git a/drivers/atm/firestream.c b/drivers/atm/firestream.c
index 82f2ae0..f205c02 100644
--- a/drivers/atm/firestream.c
+++ b/drivers/atm/firestream.c
@@ -749,7 +749,7 @@ static void process_txdone_queue (struct fs_dev *dev, struct queue *q)
}
}
- atomic_inc(&ATM_SKB(skb)->vcc->stats->tx);
+ atomic_inc_unchecked(&ATM_SKB(skb)->vcc->stats->tx);
fs_dprintk (FS_DEBUG_TXMEM, "i");
fs_dprintk (FS_DEBUG_ALLOC, "Free t-skb: %p\n", skb);
@@ -816,7 +816,7 @@ static void process_incoming (struct fs_dev *dev, struct queue *q)
#endif
skb_put (skb, qe->p1 & 0xffff);
ATM_SKB(skb)->vcc = atm_vcc;
- atomic_inc(&atm_vcc->stats->rx);
+ atomic_inc_unchecked(&atm_vcc->stats->rx);
__net_timestamp(skb);
fs_dprintk (FS_DEBUG_ALLOC, "Free rec-skb: %p (pushed)\n", skb);
atm_vcc->push (atm_vcc, skb);
@@ -837,12 +837,12 @@ static void process_incoming (struct fs_dev *dev, struct queue *q)
kfree (pe);
}
if (atm_vcc)
- atomic_inc(&atm_vcc->stats->rx_drop);
+ atomic_inc_unchecked(&atm_vcc->stats->rx_drop);
break;
case 0x1f: /* Reassembly abort: no buffers. */
/* Silently increment error counter. */
if (atm_vcc)
- atomic_inc(&atm_vcc->stats->rx_drop);
+ atomic_inc_unchecked(&atm_vcc->stats->rx_drop);
break;
default: /* Hmm. Haven't written the code to handle the others yet... -- REW */
printk (KERN_WARNING "Don't know what to do with RX status %x: %s.\n",
diff --git a/drivers/atm/fore200e.c b/drivers/atm/fore200e.c
index 75dde90..4309ead 100644
--- a/drivers/atm/fore200e.c
+++ b/drivers/atm/fore200e.c
@@ -932,9 +932,9 @@ fore200e_tx_irq(struct fore200e* fore200e)
#endif
/* check error condition */
if (*entry->status & STATUS_ERROR)
- atomic_inc(&vcc->stats->tx_err);
+ atomic_inc_unchecked(&vcc->stats->tx_err);
else
- atomic_inc(&vcc->stats->tx);
+ atomic_inc_unchecked(&vcc->stats->tx);
}
}
@@ -1083,7 +1083,7 @@ fore200e_push_rpd(struct fore200e* fore200e, struct atm_vcc* vcc, struct rpd* rp
if (skb == NULL) {
DPRINTK(2, "unable to alloc new skb, rx PDU length = %d\n", pdu_len);
- atomic_inc(&vcc->stats->rx_drop);
+ atomic_inc_unchecked(&vcc->stats->rx_drop);
return -ENOMEM;
}
@@ -1126,14 +1126,14 @@ fore200e_push_rpd(struct fore200e* fore200e, struct atm_vcc* vcc, struct rpd* rp
dev_kfree_skb_any(skb);
- atomic_inc(&vcc->stats->rx_drop);
+ atomic_inc_unchecked(&vcc->stats->rx_drop);
return -ENOMEM;
}
ASSERT(atomic_read(&sk_atm(vcc)->sk_wmem_alloc) >= 0);
vcc->push(vcc, skb);
- atomic_inc(&vcc->stats->rx);
+ atomic_inc_unchecked(&vcc->stats->rx);
ASSERT(atomic_read(&sk_atm(vcc)->sk_wmem_alloc) >= 0);
@@ -1211,7 +1211,7 @@ fore200e_rx_irq(struct fore200e* fore200e)
DPRINTK(2, "damaged PDU on %d.%d.%d\n",
fore200e->atm_dev->number,
entry->rpd->atm_header.vpi, entry->rpd->atm_header.vci);
- atomic_inc(&vcc->stats->rx_err);
+ atomic_inc_unchecked(&vcc->stats->rx_err);
}
}
@@ -1656,7 +1656,7 @@ fore200e_send(struct atm_vcc *vcc, struct sk_buff *skb)
goto retry_here;
}
- atomic_inc(&vcc->stats->tx_err);
+ atomic_inc_unchecked(&vcc->stats->tx_err);
fore200e->tx_sat++;
DPRINTK(2, "tx queue of device %s is saturated, PDU dropped - heartbeat is %08x\n",
diff --git a/drivers/atm/he.c b/drivers/atm/he.c
index a8da3a5..67cf6c2 100644
--- a/drivers/atm/he.c
+++ b/drivers/atm/he.c
@@ -1692,7 +1692,7 @@ he_service_rbrq(struct he_dev *he_dev, int group)
if (RBRQ_HBUF_ERR(he_dev->rbrq_head)) {
hprintk("HBUF_ERR! (cid 0x%x)\n", cid);
- atomic_inc(&vcc->stats->rx_drop);
+ atomic_inc_unchecked(&vcc->stats->rx_drop);
goto return_host_buffers;
}
@@ -1719,7 +1719,7 @@ he_service_rbrq(struct he_dev *he_dev, int group)
RBRQ_LEN_ERR(he_dev->rbrq_head)
? "LEN_ERR" : "",
vcc->vpi, vcc->vci);
- atomic_inc(&vcc->stats->rx_err);
+ atomic_inc_unchecked(&vcc->stats->rx_err);
goto return_host_buffers;
}
@@ -1771,7 +1771,7 @@ he_service_rbrq(struct he_dev *he_dev, int group)
vcc->push(vcc, skb);
spin_lock(&he_dev->global_lock);
- atomic_inc(&vcc->stats->rx);
+ atomic_inc_unchecked(&vcc->stats->rx);
return_host_buffers:
++pdus_assembled;
@@ -2097,7 +2097,7 @@ __enqueue_tpd(struct he_dev *he_dev, struct he_tpd *tpd, unsigned cid)
tpd->vcc->pop(tpd->vcc, tpd->skb);
else
dev_kfree_skb_any(tpd->skb);
- atomic_inc(&tpd->vcc->stats->tx_err);
+ atomic_inc_unchecked(&tpd->vcc->stats->tx_err);
}
dma_pool_free(he_dev->tpd_pool, tpd, TPD_ADDR(tpd->status));
return;
@@ -2509,7 +2509,7 @@ he_send(struct atm_vcc *vcc, struct sk_buff *skb)
vcc->pop(vcc, skb);
else
dev_kfree_skb_any(skb);
- atomic_inc(&vcc->stats->tx_err);
+ atomic_inc_unchecked(&vcc->stats->tx_err);
return -EINVAL;
}
@@ -2520,7 +2520,7 @@ he_send(struct atm_vcc *vcc, struct sk_buff *skb)
vcc->pop(vcc, skb);
else
dev_kfree_skb_any(skb);
- atomic_inc(&vcc->stats->tx_err);
+ atomic_inc_unchecked(&vcc->stats->tx_err);
return -EINVAL;
}
#endif
@@ -2532,7 +2532,7 @@ he_send(struct atm_vcc *vcc, struct sk_buff *skb)
vcc->pop(vcc, skb);
else
dev_kfree_skb_any(skb);
- atomic_inc(&vcc->stats->tx_err);
+ atomic_inc_unchecked(&vcc->stats->tx_err);
spin_unlock_irqrestore(&he_dev->global_lock, flags);
return -ENOMEM;
}
@@ -2574,7 +2574,7 @@ he_send(struct atm_vcc *vcc, struct sk_buff *skb)
vcc->pop(vcc, skb);
else
dev_kfree_skb_any(skb);
- atomic_inc(&vcc->stats->tx_err);
+ atomic_inc_unchecked(&vcc->stats->tx_err);
spin_unlock_irqrestore(&he_dev->global_lock, flags);
return -ENOMEM;
}
@@ -2605,7 +2605,7 @@ he_send(struct atm_vcc *vcc, struct sk_buff *skb)
__enqueue_tpd(he_dev, tpd, cid);
spin_unlock_irqrestore(&he_dev->global_lock, flags);
- atomic_inc(&vcc->stats->tx);
+ atomic_inc_unchecked(&vcc->stats->tx);
return 0;
}
diff --git a/drivers/atm/horizon.c b/drivers/atm/horizon.c
index 527bbd5..96570c8 100644
--- a/drivers/atm/horizon.c
+++ b/drivers/atm/horizon.c
@@ -1018,7 +1018,7 @@ static void rx_schedule (hrz_dev * dev, int irq) {
{
struct atm_vcc * vcc = ATM_SKB(skb)->vcc;
// VC layer stats
- atomic_inc(&vcc->stats->rx);
+ atomic_inc_unchecked(&vcc->stats->rx);
__net_timestamp(skb);
// end of our responsibility
vcc->push (vcc, skb);
@@ -1170,7 +1170,7 @@ static void tx_schedule (hrz_dev * const dev, int irq) {
dev->tx_iovec = NULL;
// VC layer stats
- atomic_inc(&ATM_SKB(skb)->vcc->stats->tx);
+ atomic_inc_unchecked(&ATM_SKB(skb)->vcc->stats->tx);
// free the skb
hrz_kfree_skb (skb);
diff --git a/drivers/atm/idt77252.c b/drivers/atm/idt77252.c
index 074616b..d6b3d5f 100644
--- a/drivers/atm/idt77252.c
+++ b/drivers/atm/idt77252.c
@@ -811,7 +811,7 @@ drain_scq(struct idt77252_dev *card, struct vc_map *vc)
else
dev_kfree_skb(skb);
- atomic_inc(&vcc->stats->tx);
+ atomic_inc_unchecked(&vcc->stats->tx);
}
atomic_dec(&scq->used);
@@ -1073,13 +1073,13 @@ dequeue_rx(struct idt77252_dev *card, struct rsq_entry *rsqe)
if ((sb = dev_alloc_skb(64)) == NULL) {
printk("%s: Can't allocate buffers for aal0.\n",
card->name);
- atomic_add(i, &vcc->stats->rx_drop);
+ atomic_add_unchecked(i, &vcc->stats->rx_drop);
break;
}
if (!atm_charge(vcc, sb->truesize)) {
RXPRINTK("%s: atm_charge() dropped aal0 packets.\n",
card->name);
- atomic_add(i - 1, &vcc->stats->rx_drop);
+ atomic_add_unchecked(i - 1, &vcc->stats->rx_drop);
dev_kfree_skb(sb);
break;
}
@@ -1096,7 +1096,7 @@ dequeue_rx(struct idt77252_dev *card, struct rsq_entry *rsqe)
ATM_SKB(sb)->vcc = vcc;
__net_timestamp(sb);
vcc->push(vcc, sb);
- atomic_inc(&vcc->stats->rx);
+ atomic_inc_unchecked(&vcc->stats->rx);
cell += ATM_CELL_PAYLOAD;
}
@@ -1133,13 +1133,13 @@ dequeue_rx(struct idt77252_dev *card, struct rsq_entry *rsqe)
"(CDC: %08x)\n",
card->name, len, rpp->len, readl(SAR_REG_CDC));
recycle_rx_pool_skb(card, rpp);
- atomic_inc(&vcc->stats->rx_err);
+ atomic_inc_unchecked(&vcc->stats->rx_err);
return;
}
if (stat & SAR_RSQE_CRC) {
RXPRINTK("%s: AAL5 CRC error.\n", card->name);
recycle_rx_pool_skb(card, rpp);
- atomic_inc(&vcc->stats->rx_err);
+ atomic_inc_unchecked(&vcc->stats->rx_err);
return;
}
if (skb_queue_len(&rpp->queue) > 1) {
@@ -1150,7 +1150,7 @@ dequeue_rx(struct idt77252_dev *card, struct rsq_entry *rsqe)
RXPRINTK("%s: Can't alloc RX skb.\n",
card->name);
recycle_rx_pool_skb(card, rpp);
- atomic_inc(&vcc->stats->rx_err);
+ atomic_inc_unchecked(&vcc->stats->rx_err);
return;
}
if (!atm_charge(vcc, skb->truesize)) {
@@ -1169,7 +1169,7 @@ dequeue_rx(struct idt77252_dev *card, struct rsq_entry *rsqe)
__net_timestamp(skb);
vcc->push(vcc, skb);
- atomic_inc(&vcc->stats->rx);
+ atomic_inc_unchecked(&vcc->stats->rx);
return;
}
@@ -1191,7 +1191,7 @@ dequeue_rx(struct idt77252_dev *card, struct rsq_entry *rsqe)
__net_timestamp(skb);
vcc->push(vcc, skb);
- atomic_inc(&vcc->stats->rx);
+ atomic_inc_unchecked(&vcc->stats->rx);
if (skb->truesize > SAR_FB_SIZE_3)
add_rx_skb(card, 3, SAR_FB_SIZE_3, 1);
@@ -1302,14 +1302,14 @@ idt77252_rx_raw(struct idt77252_dev *card)
if (vcc->qos.aal != ATM_AAL0) {
RPRINTK("%s: raw cell for non AAL0 vc %u.%u\n",
card->name, vpi, vci);
- atomic_inc(&vcc->stats->rx_drop);
+ atomic_inc_unchecked(&vcc->stats->rx_drop);
goto drop;
}
if ((sb = dev_alloc_skb(64)) == NULL) {
printk("%s: Can't allocate buffers for AAL0.\n",
card->name);
- atomic_inc(&vcc->stats->rx_err);
+ atomic_inc_unchecked(&vcc->stats->rx_err);
goto drop;
}
@@ -1328,7 +1328,7 @@ idt77252_rx_raw(struct idt77252_dev *card)
ATM_SKB(sb)->vcc = vcc;
__net_timestamp(sb);
vcc->push(vcc, sb);
- atomic_inc(&vcc->stats->rx);
+ atomic_inc_unchecked(&vcc->stats->rx);
drop:
skb_pull(queue, 64);
@@ -1953,13 +1953,13 @@ idt77252_send_skb(struct atm_vcc *vcc, struct sk_buff *skb, int oam)
if (vc == NULL) {
printk("%s: NULL connection in send().\n", card->name);
- atomic_inc(&vcc->stats->tx_err);
+ atomic_inc_unchecked(&vcc->stats->tx_err);
dev_kfree_skb(skb);
return -EINVAL;
}
if (!test_bit(VCF_TX, &vc->flags)) {
printk("%s: Trying to transmit on a non-tx VC.\n", card->name);
- atomic_inc(&vcc->stats->tx_err);
+ atomic_inc_unchecked(&vcc->stats->tx_err);
dev_kfree_skb(skb);
return -EINVAL;
}
@@ -1971,14 +1971,14 @@ idt77252_send_skb(struct atm_vcc *vcc, struct sk_buff *skb, int oam)
break;
default:
printk("%s: Unsupported AAL: %d\n", card->name, vcc->qos.aal);
- atomic_inc(&vcc->stats->tx_err);
+ atomic_inc_unchecked(&vcc->stats->tx_err);
dev_kfree_skb(skb);
return -EINVAL;
}
if (skb_shinfo(skb)->nr_frags != 0) {
printk("%s: No scatter-gather yet.\n", card->name);
- atomic_inc(&vcc->stats->tx_err);
+ atomic_inc_unchecked(&vcc->stats->tx_err);
dev_kfree_skb(skb);
return -EINVAL;
}
@@ -1986,7 +1986,7 @@ idt77252_send_skb(struct atm_vcc *vcc, struct sk_buff *skb, int oam)
err = queue_skb(card, vc, skb, oam);
if (err) {
- atomic_inc(&vcc->stats->tx_err);
+ atomic_inc_unchecked(&vcc->stats->tx_err);
dev_kfree_skb(skb);
return err;
}
@@ -2009,7 +2009,7 @@ idt77252_send_oam(struct atm_vcc *vcc, void *cell, int flags)
skb = dev_alloc_skb(64);
if (!skb) {
printk("%s: Out of memory in send_oam().\n", card->name);
- atomic_inc(&vcc->stats->tx_err);
+ atomic_inc_unchecked(&vcc->stats->tx_err);
return -ENOMEM;
}
atomic_add(skb->truesize, &sk_atm(vcc)->sk_wmem_alloc);
diff --git a/drivers/atm/iphase.c b/drivers/atm/iphase.c
index 65e6590..df77d04 100644
--- a/drivers/atm/iphase.c
+++ b/drivers/atm/iphase.c
@@ -1145,7 +1145,7 @@ static int rx_pkt(struct atm_dev *dev)
status = (u_short) (buf_desc_ptr->desc_mode);
if (status & (RX_CER | RX_PTE | RX_OFL))
{
- atomic_inc(&vcc->stats->rx_err);
+ atomic_inc_unchecked(&vcc->stats->rx_err);
IF_ERR(printk("IA: bad packet, dropping it");)
if (status & RX_CER) {
IF_ERR(printk(" cause: packet CRC error\n");)
@@ -1168,7 +1168,7 @@ static int rx_pkt(struct atm_dev *dev)
len = dma_addr - buf_addr;
if (len > iadev->rx_buf_sz) {
printk("Over %d bytes sdu received, dropped!!!\n", iadev->rx_buf_sz);
- atomic_inc(&vcc->stats->rx_err);
+ atomic_inc_unchecked(&vcc->stats->rx_err);
goto out_free_desc;
}
@@ -1318,7 +1318,7 @@ static void rx_dle_intr(struct atm_dev *dev)
ia_vcc = INPH_IA_VCC(vcc);
if (ia_vcc == NULL)
{
- atomic_inc(&vcc->stats->rx_err);
+ atomic_inc_unchecked(&vcc->stats->rx_err);
atm_return(vcc, skb->truesize);
dev_kfree_skb_any(skb);
goto INCR_DLE;
@@ -1330,7 +1330,7 @@ static void rx_dle_intr(struct atm_dev *dev)
if ((length > iadev->rx_buf_sz) || (length >
(skb->len - sizeof(struct cpcs_trailer))))
{
- atomic_inc(&vcc->stats->rx_err);
+ atomic_inc_unchecked(&vcc->stats->rx_err);
IF_ERR(printk("rx_dle_intr: Bad AAL5 trailer %d (skb len %d)",
length, skb->len);)
atm_return(vcc, skb->truesize);
@@ -1346,7 +1346,7 @@ static void rx_dle_intr(struct atm_dev *dev)
IF_RX(printk("rx_dle_intr: skb push");)
vcc->push(vcc,skb);
- atomic_inc(&vcc->stats->rx);
+ atomic_inc_unchecked(&vcc->stats->rx);
iadev->rx_pkt_cnt++;
}
INCR_DLE:
@@ -2828,15 +2828,15 @@ static int ia_ioctl(struct atm_dev *dev, unsigned int cmd, void __user *arg)
{
struct k_sonet_stats *stats;
stats = &PRIV(_ia_dev[board])->sonet_stats;
- printk("section_bip: %d\n", atomic_read(&stats->section_bip));
- printk("line_bip : %d\n", atomic_read(&stats->line_bip));
- printk("path_bip : %d\n", atomic_read(&stats->path_bip));
- printk("line_febe : %d\n", atomic_read(&stats->line_febe));
- printk("path_febe : %d\n", atomic_read(&stats->path_febe));
- printk("corr_hcs : %d\n", atomic_read(&stats->corr_hcs));
- printk("uncorr_hcs : %d\n", atomic_read(&stats->uncorr_hcs));
- printk("tx_cells : %d\n", atomic_read(&stats->tx_cells));
- printk("rx_cells : %d\n", atomic_read(&stats->rx_cells));
+ printk("section_bip: %d\n", atomic_read_unchecked(&stats->section_bip));
+ printk("line_bip : %d\n", atomic_read_unchecked(&stats->line_bip));
+ printk("path_bip : %d\n", atomic_read_unchecked(&stats->path_bip));
+ printk("line_febe : %d\n", atomic_read_unchecked(&stats->line_febe));
+ printk("path_febe : %d\n", atomic_read_unchecked(&stats->path_febe));
+ printk("corr_hcs : %d\n", atomic_read_unchecked(&stats->corr_hcs));
+ printk("uncorr_hcs : %d\n", atomic_read_unchecked(&stats->uncorr_hcs));
+ printk("tx_cells : %d\n", atomic_read_unchecked(&stats->tx_cells));
+ printk("rx_cells : %d\n", atomic_read_unchecked(&stats->rx_cells));
}
ia_cmds.status = 0;
break;
@@ -2941,7 +2941,7 @@ static int ia_pkt_tx (struct atm_vcc *vcc, struct sk_buff *skb) {
if ((desc == 0) || (desc > iadev->num_tx_desc))
{
IF_ERR(printk(DEV_LABEL "invalid desc for send: %d\n", desc);)
- atomic_inc(&vcc->stats->tx);
+ atomic_inc_unchecked(&vcc->stats->tx);
if (vcc->pop)
vcc->pop(vcc, skb);
else
@@ -3046,14 +3046,14 @@ static int ia_pkt_tx (struct atm_vcc *vcc, struct sk_buff *skb) {
ATM_DESC(skb) = vcc->vci;
skb_queue_tail(&iadev->tx_dma_q, skb);
- atomic_inc(&vcc->stats->tx);
+ atomic_inc_unchecked(&vcc->stats->tx);
iadev->tx_pkt_cnt++;
/* Increment transaction counter */
writel(2, iadev->dma+IPHASE5575_TX_COUNTER);
#if 0
/* add flow control logic */
- if (atomic_read(&vcc->stats->tx) % 20 == 0) {
+ if (atomic_read_unchecked(&vcc->stats->tx) % 20 == 0) {
if (iavcc->vc_desc_cnt > 10) {
vcc->tx_quota = vcc->tx_quota * 3 / 4;
printk("Tx1: vcc->tx_quota = %d \n", (u32)vcc->tx_quota );
diff --git a/drivers/atm/lanai.c b/drivers/atm/lanai.c
index ce43ae3..969de38 100644
--- a/drivers/atm/lanai.c
+++ b/drivers/atm/lanai.c
@@ -1295,7 +1295,7 @@ static void lanai_send_one_aal5(struct lanai_dev *lanai,
vcc_tx_add_aal5_trailer(lvcc, skb->len, 0, 0);
lanai_endtx(lanai, lvcc);
lanai_free_skb(lvcc->tx.atmvcc, skb);
- atomic_inc(&lvcc->tx.atmvcc->stats->tx);
+ atomic_inc_unchecked(&lvcc->tx.atmvcc->stats->tx);
}
/* Try to fill the buffer - don't call unless there is backlog */
@@ -1418,7 +1418,7 @@ static void vcc_rx_aal5(struct lanai_vcc *lvcc, int endptr)
ATM_SKB(skb)->vcc = lvcc->rx.atmvcc;
__net_timestamp(skb);
lvcc->rx.atmvcc->push(lvcc->rx.atmvcc, skb);
- atomic_inc(&lvcc->rx.atmvcc->stats->rx);
+ atomic_inc_unchecked(&lvcc->rx.atmvcc->stats->rx);
out:
lvcc->rx.buf.ptr = end;
cardvcc_write(lvcc, endptr, vcc_rxreadptr);
@@ -1659,7 +1659,7 @@ static int handle_service(struct lanai_dev *lanai, u32 s)
DPRINTK("(itf %d) got RX service entry 0x%X for non-AAL5 "
"vcc %d\n", lanai->number, (unsigned int) s, vci);
lanai->stats.service_rxnotaal5++;
- atomic_inc(&lvcc->rx.atmvcc->stats->rx_err);
+ atomic_inc_unchecked(&lvcc->rx.atmvcc->stats->rx_err);
return 0;
}
if (likely(!(s & (SERVICE_TRASH | SERVICE_STREAM | SERVICE_CRCERR)))) {
@@ -1671,7 +1671,7 @@ static int handle_service(struct lanai_dev *lanai, u32 s)
int bytes;
read_unlock(&vcc_sklist_lock);
DPRINTK("got trashed rx pdu on vci %d\n", vci);
- atomic_inc(&lvcc->rx.atmvcc->stats->rx_err);
+ atomic_inc_unchecked(&lvcc->rx.atmvcc->stats->rx_err);
lvcc->stats.x.aal5.service_trash++;
bytes = (SERVICE_GET_END(s) * 16) -
(((unsigned long) lvcc->rx.buf.ptr) -
@@ -1683,7 +1683,7 @@ static int handle_service(struct lanai_dev *lanai, u32 s)
}
if (s & SERVICE_STREAM) {
read_unlock(&vcc_sklist_lock);
- atomic_inc(&lvcc->rx.atmvcc->stats->rx_err);
+ atomic_inc_unchecked(&lvcc->rx.atmvcc->stats->rx_err);
lvcc->stats.x.aal5.service_stream++;
printk(KERN_ERR DEV_LABEL "(itf %d): Got AAL5 stream "
"PDU on VCI %d!\n", lanai->number, vci);
@@ -1691,7 +1691,7 @@ static int handle_service(struct lanai_dev *lanai, u32 s)
return 0;
}
DPRINTK("got rx crc error on vci %d\n", vci);
- atomic_inc(&lvcc->rx.atmvcc->stats->rx_err);
+ atomic_inc_unchecked(&lvcc->rx.atmvcc->stats->rx_err);
lvcc->stats.x.aal5.service_rxcrc++;
lvcc->rx.buf.ptr = &lvcc->rx.buf.start[SERVICE_GET_END(s) * 4];
cardvcc_write(lvcc, SERVICE_GET_END(s), vcc_rxreadptr);
diff --git a/drivers/atm/nicstar.c b/drivers/atm/nicstar.c
index ddc4ceb..36e29aa 100644
--- a/drivers/atm/nicstar.c
+++ b/drivers/atm/nicstar.c
@@ -1632,7 +1632,7 @@ static int ns_send(struct atm_vcc *vcc, struct sk_buff *skb)
if ((vc = (vc_map *) vcc->dev_data) == NULL) {
printk("nicstar%d: vcc->dev_data == NULL on ns_send().\n",
card->index);
- atomic_inc(&vcc->stats->tx_err);
+ atomic_inc_unchecked(&vcc->stats->tx_err);
dev_kfree_skb_any(skb);
return -EINVAL;
}
@@ -1640,7 +1640,7 @@ static int ns_send(struct atm_vcc *vcc, struct sk_buff *skb)
if (!vc->tx) {
printk("nicstar%d: Trying to transmit on a non-tx VC.\n",
card->index);
- atomic_inc(&vcc->stats->tx_err);
+ atomic_inc_unchecked(&vcc->stats->tx_err);
dev_kfree_skb_any(skb);
return -EINVAL;
}
@@ -1648,14 +1648,14 @@ static int ns_send(struct atm_vcc *vcc, struct sk_buff *skb)
if (vcc->qos.aal != ATM_AAL5 && vcc->qos.aal != ATM_AAL0) {
printk("nicstar%d: Only AAL0 and AAL5 are supported.\n",
card->index);
- atomic_inc(&vcc->stats->tx_err);
+ atomic_inc_unchecked(&vcc->stats->tx_err);
dev_kfree_skb_any(skb);
return -EINVAL;
}
if (skb_shinfo(skb)->nr_frags != 0) {
printk("nicstar%d: No scatter-gather yet.\n", card->index);
- atomic_inc(&vcc->stats->tx_err);
+ atomic_inc_unchecked(&vcc->stats->tx_err);
dev_kfree_skb_any(skb);
return -EINVAL;
}
@@ -1703,11 +1703,11 @@ static int ns_send(struct atm_vcc *vcc, struct sk_buff *skb)
}
if (push_scqe(card, vc, scq, &scqe, skb) != 0) {
- atomic_inc(&vcc->stats->tx_err);
+ atomic_inc_unchecked(&vcc->stats->tx_err);
dev_kfree_skb_any(skb);
return -EIO;
}
- atomic_inc(&vcc->stats->tx);
+ atomic_inc_unchecked(&vcc->stats->tx);
return 0;
}
@@ -2024,14 +2024,14 @@ static void dequeue_rx(ns_dev * card, ns_rsqe * rsqe)
printk
("nicstar%d: Can't allocate buffers for aal0.\n",
card->index);
- atomic_add(i, &vcc->stats->rx_drop);
+ atomic_add_unchecked(i, &vcc->stats->rx_drop);
break;
}
if (!atm_charge(vcc, sb->truesize)) {
RXPRINTK
("nicstar%d: atm_charge() dropped aal0 packets.\n",
card->index);
- atomic_add(i - 1, &vcc->stats->rx_drop); /* already increased by 1 */
+ atomic_add_unchecked(i - 1, &vcc->stats->rx_drop); /* already increased by 1 */
dev_kfree_skb_any(sb);
break;
}
@@ -2046,7 +2046,7 @@ static void dequeue_rx(ns_dev * card, ns_rsqe * rsqe)
ATM_SKB(sb)->vcc = vcc;
__net_timestamp(sb);
vcc->push(vcc, sb);
- atomic_inc(&vcc->stats->rx);
+ atomic_inc_unchecked(&vcc->stats->rx);
cell += ATM_CELL_PAYLOAD;
}
@@ -2063,7 +2063,7 @@ static void dequeue_rx(ns_dev * card, ns_rsqe * rsqe)
if (iovb == NULL) {
printk("nicstar%d: Out of iovec buffers.\n",
card->index);
- atomic_inc(&vcc->stats->rx_drop);
+ atomic_inc_unchecked(&vcc->stats->rx_drop);
recycle_rx_buf(card, skb);
return;
}
@@ -2087,7 +2087,7 @@ static void dequeue_rx(ns_dev * card, ns_rsqe * rsqe)
small or large buffer itself. */
} else if (NS_PRV_IOVCNT(iovb) >= NS_MAX_IOVECS) {
printk("nicstar%d: received too big AAL5 SDU.\n", card->index);
- atomic_inc(&vcc->stats->rx_err);
+ atomic_inc_unchecked(&vcc->stats->rx_err);
recycle_iovec_rx_bufs(card, (struct iovec *)iovb->data,
NS_MAX_IOVECS);
NS_PRV_IOVCNT(iovb) = 0;
@@ -2107,7 +2107,7 @@ static void dequeue_rx(ns_dev * card, ns_rsqe * rsqe)
("nicstar%d: Expected a small buffer, and this is not one.\n",
card->index);
which_list(card, skb);
- atomic_inc(&vcc->stats->rx_err);
+ atomic_inc_unchecked(&vcc->stats->rx_err);
recycle_rx_buf(card, skb);
vc->rx_iov = NULL;
recycle_iov_buf(card, iovb);
@@ -2120,7 +2120,7 @@ static void dequeue_rx(ns_dev * card, ns_rsqe * rsqe)
("nicstar%d: Expected a large buffer, and this is not one.\n",
card->index);
which_list(card, skb);
- atomic_inc(&vcc->stats->rx_err);
+ atomic_inc_unchecked(&vcc->stats->rx_err);
recycle_iovec_rx_bufs(card, (struct iovec *)iovb->data,
NS_PRV_IOVCNT(iovb));
vc->rx_iov = NULL;
@@ -2143,7 +2143,7 @@ static void dequeue_rx(ns_dev * card, ns_rsqe * rsqe)
printk(" - PDU size mismatch.\n");
else
printk(".\n");
- atomic_inc(&vcc->stats->rx_err);
+ atomic_inc_unchecked(&vcc->stats->rx_err);
recycle_iovec_rx_bufs(card, (struct iovec *)iovb->data,
NS_PRV_IOVCNT(iovb));
vc->rx_iov = NULL;
@@ -2157,14 +2157,14 @@ static void dequeue_rx(ns_dev * card, ns_rsqe * rsqe)
/* skb points to a small buffer */
if (!atm_charge(vcc, skb->truesize)) {
push_rxbufs(card, skb);
- atomic_inc(&vcc->stats->rx_drop);
+ atomic_inc_unchecked(&vcc->stats->rx_drop);
} else {
skb_put(skb, len);
dequeue_sm_buf(card, skb);
ATM_SKB(skb)->vcc = vcc;
__net_timestamp(skb);
vcc->push(vcc, skb);
- atomic_inc(&vcc->stats->rx);
+ atomic_inc_unchecked(&vcc->stats->rx);
}
} else if (NS_PRV_IOVCNT(iovb) == 2) { /* One small plus one large buffer */
struct sk_buff *sb;
@@ -2175,14 +2175,14 @@ static void dequeue_rx(ns_dev * card, ns_rsqe * rsqe)
if (len <= NS_SMBUFSIZE) {
if (!atm_charge(vcc, sb->truesize)) {
push_rxbufs(card, sb);
- atomic_inc(&vcc->stats->rx_drop);
+ atomic_inc_unchecked(&vcc->stats->rx_drop);
} else {
skb_put(sb, len);
dequeue_sm_buf(card, sb);
ATM_SKB(sb)->vcc = vcc;
__net_timestamp(sb);
vcc->push(vcc, sb);
- atomic_inc(&vcc->stats->rx);
+ atomic_inc_unchecked(&vcc->stats->rx);
}
push_rxbufs(card, skb);
@@ -2191,7 +2191,7 @@ static void dequeue_rx(ns_dev * card, ns_rsqe * rsqe)
if (!atm_charge(vcc, skb->truesize)) {
push_rxbufs(card, skb);
- atomic_inc(&vcc->stats->rx_drop);
+ atomic_inc_unchecked(&vcc->stats->rx_drop);
} else {
dequeue_lg_buf(card, skb);
skb_push(skb, NS_SMBUFSIZE);
@@ -2201,7 +2201,7 @@ static void dequeue_rx(ns_dev * card, ns_rsqe * rsqe)
ATM_SKB(skb)->vcc = vcc;
__net_timestamp(skb);
vcc->push(vcc, skb);
- atomic_inc(&vcc->stats->rx);
+ atomic_inc_unchecked(&vcc->stats->rx);
}
push_rxbufs(card, sb);
@@ -2222,7 +2222,7 @@ static void dequeue_rx(ns_dev * card, ns_rsqe * rsqe)
printk
("nicstar%d: Out of huge buffers.\n",
card->index);
- atomic_inc(&vcc->stats->rx_drop);
+ atomic_inc_unchecked(&vcc->stats->rx_drop);
recycle_iovec_rx_bufs(card,
(struct iovec *)
iovb->data,
@@ -2273,7 +2273,7 @@ static void dequeue_rx(ns_dev * card, ns_rsqe * rsqe)
card->hbpool.count++;
} else
dev_kfree_skb_any(hb);
- atomic_inc(&vcc->stats->rx_drop);
+ atomic_inc_unchecked(&vcc->stats->rx_drop);
} else {
/* Copy the small buffer to the huge buffer */
sb = (struct sk_buff *)iov->iov_base;
@@ -2307,7 +2307,7 @@ static void dequeue_rx(ns_dev * card, ns_rsqe * rsqe)
ATM_SKB(hb)->vcc = vcc;
__net_timestamp(hb);
vcc->push(vcc, hb);
- atomic_inc(&vcc->stats->rx);
+ atomic_inc_unchecked(&vcc->stats->rx);
}
}
diff --git a/drivers/atm/solos-pci.c b/drivers/atm/solos-pci.c
index 74e18b0..f16afa0 100644
--- a/drivers/atm/solos-pci.c
+++ b/drivers/atm/solos-pci.c
@@ -838,7 +838,7 @@ static void solos_bh(unsigned long card_arg)
}
atm_charge(vcc, skb->truesize);
vcc->push(vcc, skb);
- atomic_inc(&vcc->stats->rx);
+ atomic_inc_unchecked(&vcc->stats->rx);
break;
case PKT_STATUS:
@@ -1116,7 +1116,7 @@ static uint32_t fpga_tx(struct solos_card *card)
vcc = SKB_CB(oldskb)->vcc;
if (vcc) {
- atomic_inc(&vcc->stats->tx);
+ atomic_inc_unchecked(&vcc->stats->tx);
solos_pop(vcc, oldskb);
} else {
dev_kfree_skb_irq(oldskb);
diff --git a/drivers/atm/suni.c b/drivers/atm/suni.c
index 0215934..ce9f5b1 100644
--- a/drivers/atm/suni.c
+++ b/drivers/atm/suni.c
@@ -49,8 +49,8 @@ static DEFINE_SPINLOCK(sunis_lock);
#define ADD_LIMITED(s,v) \
- atomic_add((v),&stats->s); \
- if (atomic_read(&stats->s) < 0) atomic_set(&stats->s,INT_MAX);
+ atomic_add_unchecked((v),&stats->s); \
+ if (atomic_read_unchecked(&stats->s) < 0) atomic_set_unchecked(&stats->s,INT_MAX);
static void suni_hz(unsigned long from_timer)
diff --git a/drivers/atm/uPD98402.c b/drivers/atm/uPD98402.c
index 5120a96..e2572bd 100644
--- a/drivers/atm/uPD98402.c
+++ b/drivers/atm/uPD98402.c
@@ -42,7 +42,7 @@ static int fetch_stats(struct atm_dev *dev,struct sonet_stats __user *arg,int ze
struct sonet_stats tmp;
int error = 0;
- atomic_add(GET(HECCT),&PRIV(dev)->sonet_stats.uncorr_hcs);
+ atomic_add_unchecked(GET(HECCT),&PRIV(dev)->sonet_stats.uncorr_hcs);
sonet_copy_stats(&PRIV(dev)->sonet_stats,&tmp);
if (arg) error = copy_to_user(arg,&tmp,sizeof(tmp));
if (zero && !error) {
@@ -161,9 +161,9 @@ static int uPD98402_ioctl(struct atm_dev *dev,unsigned int cmd,void __user *arg)
#define ADD_LIMITED(s,v) \
- { atomic_add(GET(v),&PRIV(dev)->sonet_stats.s); \
- if (atomic_read(&PRIV(dev)->sonet_stats.s) < 0) \
- atomic_set(&PRIV(dev)->sonet_stats.s,INT_MAX); }
+ { atomic_add_unchecked(GET(v),&PRIV(dev)->sonet_stats.s); \
+ if (atomic_read_unchecked(&PRIV(dev)->sonet_stats.s) < 0) \
+ atomic_set_unchecked(&PRIV(dev)->sonet_stats.s,INT_MAX); }
static void stat_event(struct atm_dev *dev)
@@ -194,7 +194,7 @@ static void uPD98402_int(struct atm_dev *dev)
if (reason & uPD98402_INT_PFM) stat_event(dev);
if (reason & uPD98402_INT_PCO) {
(void) GET(PCOCR); /* clear interrupt cause */
- atomic_add(GET(HECCT),
+ atomic_add_unchecked(GET(HECCT),
&PRIV(dev)->sonet_stats.uncorr_hcs);
}
if ((reason & uPD98402_INT_RFO) &&
@@ -222,9 +222,9 @@ static int uPD98402_start(struct atm_dev *dev)
PUT(~(uPD98402_INT_PFM | uPD98402_INT_ALM | uPD98402_INT_RFO |
uPD98402_INT_LOS),PIMR); /* enable them */
(void) fetch_stats(dev,NULL,1); /* clear kernel counters */
- atomic_set(&PRIV(dev)->sonet_stats.corr_hcs,-1);
- atomic_set(&PRIV(dev)->sonet_stats.tx_cells,-1);
- atomic_set(&PRIV(dev)->sonet_stats.rx_cells,-1);
+ atomic_set_unchecked(&PRIV(dev)->sonet_stats.corr_hcs,-1);
+ atomic_set_unchecked(&PRIV(dev)->sonet_stats.tx_cells,-1);
+ atomic_set_unchecked(&PRIV(dev)->sonet_stats.rx_cells,-1);
return 0;
}
diff --git a/drivers/atm/zatm.c b/drivers/atm/zatm.c
index cecfb94..87009ec 100644
--- a/drivers/atm/zatm.c
+++ b/drivers/atm/zatm.c
@@ -459,7 +459,7 @@ printk("dummy: 0x%08lx, 0x%08lx\n",dummy[0],dummy[1]);
}
if (!size) {
dev_kfree_skb_irq(skb);
- if (vcc) atomic_inc(&vcc->stats->rx_err);
+ if (vcc) atomic_inc_unchecked(&vcc->stats->rx_err);
continue;
}
if (!atm_charge(vcc,skb->truesize)) {
@@ -469,7 +469,7 @@ printk("dummy: 0x%08lx, 0x%08lx\n",dummy[0],dummy[1]);
skb->len = size;
ATM_SKB(skb)->vcc = vcc;
vcc->push(vcc,skb);
- atomic_inc(&vcc->stats->rx);
+ atomic_inc_unchecked(&vcc->stats->rx);
}
zout(pos & 0xffff,MTA(mbx));
#if 0 /* probably a stupid idea */
@@ -733,7 +733,7 @@ if (*ZATM_PRV_DSC(skb) != (uPD98401_TXPD_V | uPD98401_TXPD_DP |
skb_queue_head(&zatm_vcc->backlog,skb);
break;
}
- atomic_inc(&vcc->stats->tx);
+ atomic_inc_unchecked(&vcc->stats->tx);
wake_up(&zatm_vcc->tx_wait);
}
diff --git a/drivers/base/power/wakeup.c b/drivers/base/power/wakeup.c
index 51f15bc..892a668 100644
--- a/drivers/base/power/wakeup.c
+++ b/drivers/base/power/wakeup.c
@@ -33,14 +33,14 @@ static bool pm_abort_suspend __read_mostly;
* They need to be modified together atomically, so it's better to use one
* atomic variable to hold them both.
*/
-static atomic_t combined_event_count = ATOMIC_INIT(0);
+static atomic_unchecked_t combined_event_count = ATOMIC_INIT(0);
#define IN_PROGRESS_BITS (sizeof(int) * 4)
#define MAX_IN_PROGRESS ((1 << IN_PROGRESS_BITS) - 1)
static void split_counters(unsigned int *cnt, unsigned int *inpr)
{
- unsigned int comb = atomic_read(&combined_event_count);
+ unsigned int comb = atomic_read_unchecked(&combined_event_count);
*cnt = (comb >> IN_PROGRESS_BITS);
*inpr = comb & MAX_IN_PROGRESS;
@@ -537,7 +537,7 @@ static void wakeup_source_activate(struct wakeup_source *ws)
ws->start_prevent_time = ws->last_time;
/* Increment the counter of events in progress. */
- cec = atomic_inc_return(&combined_event_count);
+ cec = atomic_inc_return_unchecked(&combined_event_count);
trace_wakeup_source_activate(ws->name, cec);
}
@@ -663,7 +663,7 @@ static void wakeup_source_deactivate(struct wakeup_source *ws)
* Increment the counter of registered wakeup events and decrement the
* couter of wakeup events in progress simultaneously.
*/
- cec = atomic_add_return(MAX_IN_PROGRESS, &combined_event_count);
+ cec = atomic_add_return_unchecked(MAX_IN_PROGRESS, &combined_event_count);
trace_wakeup_source_deactivate(ws->name, cec);
split_counters(&cnt, &inpr);
diff --git a/drivers/block/drbd/drbd_bitmap.c b/drivers/block/drbd/drbd_bitmap.c
index 434c77d..6d3219a 100644
--- a/drivers/block/drbd/drbd_bitmap.c
+++ b/drivers/block/drbd/drbd_bitmap.c
@@ -1036,7 +1036,7 @@ static void bm_page_io_async(struct drbd_bm_aio_ctx *ctx, int page_nr) __must_ho
submit_bio(rw, bio);
/* this should not count as user activity and cause the
* resync to throttle -- see drbd_rs_should_slow_down(). */
- atomic_add(len >> 9, &device->rs_sect_ev);
+ atomic_add_unchecked(len >> 9, &device->rs_sect_ev);
}
}
diff --git a/drivers/block/drbd/drbd_int.h b/drivers/block/drbd/drbd_int.h
index efd19c2..6ccfa94 100644
--- a/drivers/block/drbd/drbd_int.h
+++ b/drivers/block/drbd/drbd_int.h
@@ -386,7 +386,7 @@ struct drbd_epoch {
struct drbd_connection *connection;
struct list_head list;
unsigned int barrier_nr;
- atomic_t epoch_size; /* increased on every request added. */
+ atomic_unchecked_t epoch_size; /* increased on every request added. */
atomic_t active; /* increased on every req. added, and dec on every finished. */
unsigned long flags;
};
@@ -947,7 +947,7 @@ struct drbd_device {
unsigned int al_tr_number;
int al_tr_cycle;
wait_queue_head_t seq_wait;
- atomic_t packet_seq;
+ atomic_unchecked_t packet_seq;
unsigned int peer_seq;
spinlock_t peer_seq_lock;
unsigned long comm_bm_set; /* communicated number of set bits. */
@@ -956,8 +956,8 @@ struct drbd_device {
struct mutex own_state_mutex;
struct mutex *state_mutex; /* either own_state_mutex or first_peer_device(device)->connection->cstate_mutex */
char congestion_reason; /* Why we where congested... */
- atomic_t rs_sect_in; /* for incoming resync data rate, SyncTarget */
- atomic_t rs_sect_ev; /* for submitted resync data rate, both */
+ atomic_unchecked_t rs_sect_in; /* for incoming resync data rate, SyncTarget */
+ atomic_unchecked_t rs_sect_ev; /* for submitted resync data rate, both */
int rs_last_sect_ev; /* counter to compare with */
int rs_last_events; /* counter of read or write "events" (unit sectors)
* on the lower level device when we last looked. */
diff --git a/drivers/block/drbd/drbd_main.c b/drivers/block/drbd/drbd_main.c
index a151853..b9b5baa 100644
--- a/drivers/block/drbd/drbd_main.c
+++ b/drivers/block/drbd/drbd_main.c
@@ -1328,7 +1328,7 @@ static int _drbd_send_ack(struct drbd_peer_device *peer_device, enum drbd_packet
p->sector = sector;
p->block_id = block_id;
p->blksize = blksize;
- p->seq_num = cpu_to_be32(atomic_inc_return(&peer_device->device->packet_seq));
+ p->seq_num = cpu_to_be32(atomic_inc_return_unchecked(&peer_device->device->packet_seq));
return drbd_send_command(peer_device, sock, cmd, sizeof(*p), NULL, 0);
}
@@ -1634,7 +1634,7 @@ int drbd_send_dblock(struct drbd_peer_device *peer_device, struct drbd_request *
return -EIO;
p->sector = cpu_to_be64(req->i.sector);
p->block_id = (unsigned long)req;
- p->seq_num = cpu_to_be32(atomic_inc_return(&device->packet_seq));
+ p->seq_num = cpu_to_be32(atomic_inc_return_unchecked(&device->packet_seq));
dp_flags = bio_flags_to_wire(peer_device->connection, req->master_bio->bi_rw);
if (device->state.conn >= C_SYNC_SOURCE &&
device->state.conn <= C_PAUSED_SYNC_T)
@@ -1915,8 +1915,8 @@ void drbd_init_set_defaults(struct drbd_device *device)
atomic_set(&device->unacked_cnt, 0);
atomic_set(&device->local_cnt, 0);
atomic_set(&device->pp_in_use_by_net, 0);
- atomic_set(&device->rs_sect_in, 0);
- atomic_set(&device->rs_sect_ev, 0);
+ atomic_set_unchecked(&device->rs_sect_in, 0);
+ atomic_set_unchecked(&device->rs_sect_ev, 0);
atomic_set(&device->ap_in_flight, 0);
atomic_set(&device->md_io.in_use, 0);
@@ -2683,8 +2683,8 @@ void drbd_destroy_connection(struct kref *kref)
struct drbd_connection *connection = container_of(kref, struct drbd_connection, kref);
struct drbd_resource *resource = connection->resource;
- if (atomic_read(&connection->current_epoch->epoch_size) != 0)
- drbd_err(connection, "epoch_size:%d\n", atomic_read(&connection->current_epoch->epoch_size));
+ if (atomic_read_unchecked(&connection->current_epoch->epoch_size) != 0)
+ drbd_err(connection, "epoch_size:%d\n", atomic_read_unchecked(&connection->current_epoch->epoch_size));
kfree(connection->current_epoch);
idr_destroy(&connection->peer_devices);
diff --git a/drivers/block/drbd/drbd_nl.c b/drivers/block/drbd/drbd_nl.c
index 74df8cf..e41fc24 100644
--- a/drivers/block/drbd/drbd_nl.c
+++ b/drivers/block/drbd/drbd_nl.c
@@ -3637,13 +3637,13 @@ finish:
void drbd_bcast_event(struct drbd_device *device, const struct sib_info *sib)
{
- static atomic_t drbd_genl_seq = ATOMIC_INIT(2); /* two. */
+ static atomic_unchecked_t drbd_genl_seq = ATOMIC_INIT(2); /* two. */
struct sk_buff *msg;
struct drbd_genlmsghdr *d_out;
unsigned seq;
int err = -ENOMEM;
- seq = atomic_inc_return(&drbd_genl_seq);
+ seq = atomic_inc_return_unchecked(&drbd_genl_seq);
msg = genlmsg_new(NLMSG_GOODSIZE, GFP_NOIO);
if (!msg)
goto failed;
diff --git a/drivers/block/drbd/drbd_receiver.c b/drivers/block/drbd/drbd_receiver.c
index c097909..11820ed 100644
--- a/drivers/block/drbd/drbd_receiver.c
+++ b/drivers/block/drbd/drbd_receiver.c
@@ -870,7 +870,7 @@ int drbd_connected(struct drbd_peer_device *peer_device)
struct drbd_device *device = peer_device->device;
int err;
- atomic_set(&device->packet_seq, 0);
+ atomic_set_unchecked(&device->packet_seq, 0);
device->peer_seq = 0;
device->state_mutex = peer_device->connection->agreed_pro_version < 100 ?
@@ -1233,7 +1233,7 @@ static enum finish_epoch drbd_may_finish_epoch(struct drbd_connection *connectio
do {
next_epoch = NULL;
- epoch_size = atomic_read(&epoch->epoch_size);
+ epoch_size = atomic_read_unchecked(&epoch->epoch_size);
switch (ev & ~EV_CLEANUP) {
case EV_PUT:
@@ -1273,7 +1273,7 @@ static enum finish_epoch drbd_may_finish_epoch(struct drbd_connection *connectio
rv = FE_DESTROYED;
} else {
epoch->flags = 0;
- atomic_set(&epoch->epoch_size, 0);
+ atomic_set_unchecked(&epoch->epoch_size, 0);
/* atomic_set(&epoch->active, 0); is already zero */
if (rv == FE_STILL_LIVE)
rv = FE_RECYCLED;
@@ -1550,7 +1550,7 @@ static int receive_Barrier(struct drbd_connection *connection, struct packet_inf
conn_wait_active_ee_empty(connection);
drbd_flush(connection);
- if (atomic_read(&connection->current_epoch->epoch_size)) {
+ if (atomic_read_unchecked(&connection->current_epoch->epoch_size)) {
epoch = kmalloc(sizeof(struct drbd_epoch), GFP_NOIO);
if (epoch)
break;
@@ -1564,11 +1564,11 @@ static int receive_Barrier(struct drbd_connection *connection, struct packet_inf
}
epoch->flags = 0;
- atomic_set(&epoch->epoch_size, 0);
+ atomic_set_unchecked(&epoch->epoch_size, 0);
atomic_set(&epoch->active, 0);
spin_lock(&connection->epoch_lock);
- if (atomic_read(&connection->current_epoch->epoch_size)) {
+ if (atomic_read_unchecked(&connection->current_epoch->epoch_size)) {
list_add(&epoch->list, &connection->current_epoch->list);
connection->current_epoch = epoch;
connection->epochs++;
@@ -1802,7 +1802,7 @@ static int recv_resync_read(struct drbd_peer_device *peer_device, sector_t secto
list_add_tail(&peer_req->w.list, &device->sync_ee);
spin_unlock_irq(&device->resource->req_lock);
- atomic_add(pi->size >> 9, &device->rs_sect_ev);
+ atomic_add_unchecked(pi->size >> 9, &device->rs_sect_ev);
if (drbd_submit_peer_request(device, peer_req, WRITE, DRBD_FAULT_RS_WR) == 0)
return 0;
@@ -1900,7 +1900,7 @@ static int receive_RSDataReply(struct drbd_connection *connection, struct packet
drbd_send_ack_dp(peer_device, P_NEG_ACK, p, pi->size);
}
- atomic_add(pi->size >> 9, &device->rs_sect_in);
+ atomic_add_unchecked(pi->size >> 9, &device->rs_sect_in);
return err;
}
@@ -2290,7 +2290,7 @@ static int receive_Data(struct drbd_connection *connection, struct packet_info *
err = wait_for_and_update_peer_seq(peer_device, peer_seq);
drbd_send_ack_dp(peer_device, P_NEG_ACK, p, pi->size);
- atomic_inc(&connection->current_epoch->epoch_size);
+ atomic_inc_unchecked(&connection->current_epoch->epoch_size);
err2 = drbd_drain_block(peer_device, pi->size);
if (!err)
err = err2;
@@ -2334,7 +2334,7 @@ static int receive_Data(struct drbd_connection *connection, struct packet_info *
spin_lock(&connection->epoch_lock);
peer_req->epoch = connection->current_epoch;
- atomic_inc(&peer_req->epoch->epoch_size);
+ atomic_inc_unchecked(&peer_req->epoch->epoch_size);
atomic_inc(&peer_req->epoch->active);
spin_unlock(&connection->epoch_lock);
@@ -2479,7 +2479,7 @@ bool drbd_rs_c_min_rate_throttle(struct drbd_device *device)
curr_events = (int)part_stat_read(&disk->part0, sectors[0]) +
(int)part_stat_read(&disk->part0, sectors[1]) -
- atomic_read(&device->rs_sect_ev);
+ atomic_read_unchecked(&device->rs_sect_ev);
if (atomic_read(&device->ap_actlog_cnt)
|| curr_events - device->rs_last_events > 64) {
@@ -2618,7 +2618,7 @@ static int receive_DataRequest(struct drbd_connection *connection, struct packet
device->use_csums = true;
} else if (pi->cmd == P_OV_REPLY) {
/* track progress, we may need to throttle */
- atomic_add(size >> 9, &device->rs_sect_in);
+ atomic_add_unchecked(size >> 9, &device->rs_sect_in);
peer_req->w.cb = w_e_end_ov_reply;
dec_rs_pending(device);
/* drbd_rs_begin_io done when we sent this request,
@@ -2691,7 +2691,7 @@ static int receive_DataRequest(struct drbd_connection *connection, struct packet
goto out_free_e;
submit_for_resync:
- atomic_add(size >> 9, &device->rs_sect_ev);
+ atomic_add_unchecked(size >> 9, &device->rs_sect_ev);
submit:
update_receiver_timing_details(connection, drbd_submit_peer_request);
@@ -4678,7 +4678,7 @@ static void conn_disconnect(struct drbd_connection *connection)
if (!list_empty(&connection->current_epoch->list))
drbd_err(connection, "ASSERTION FAILED: connection->current_epoch->list not empty\n");
/* ok, no more ee's on the fly, it is safe to reset the epoch_size */
- atomic_set(&connection->current_epoch->epoch_size, 0);
+ atomic_set_unchecked(&connection->current_epoch->epoch_size, 0);
connection->send.seen_any_write_yet = false;
drbd_info(connection, "Connection closed\n");
@@ -5182,7 +5182,7 @@ static int got_IsInSync(struct drbd_connection *connection, struct packet_info *
put_ldev(device);
}
dec_rs_pending(device);
- atomic_add(blksize >> 9, &device->rs_sect_in);
+ atomic_add_unchecked(blksize >> 9, &device->rs_sect_in);
return 0;
}
diff --git a/drivers/block/drbd/drbd_worker.c b/drivers/block/drbd/drbd_worker.c
index d0fae55..4469096 100644
--- a/drivers/block/drbd/drbd_worker.c
+++ b/drivers/block/drbd/drbd_worker.c
@@ -408,7 +408,7 @@ static int read_for_csum(struct drbd_peer_device *peer_device, sector_t sector,
list_add_tail(&peer_req->w.list, &device->read_ee);
spin_unlock_irq(&device->resource->req_lock);
- atomic_add(size >> 9, &device->rs_sect_ev);
+ atomic_add_unchecked(size >> 9, &device->rs_sect_ev);
if (drbd_submit_peer_request(device, peer_req, READ, DRBD_FAULT_RS_RD) == 0)
return 0;
@@ -553,7 +553,7 @@ static int drbd_rs_number_requests(struct drbd_device *device)
unsigned int sect_in; /* Number of sectors that came in since the last turn */
int number, mxb;
- sect_in = atomic_xchg(&device->rs_sect_in, 0);
+ sect_in = atomic_xchg_unchecked(&device->rs_sect_in, 0);
device->rs_in_flight -= sect_in;
rcu_read_lock();
@@ -1595,8 +1595,8 @@ void drbd_rs_controller_reset(struct drbd_device *device)
struct gendisk *disk = device->ldev->backing_bdev->bd_contains->bd_disk;
struct fifo_buffer *plan;
- atomic_set(&device->rs_sect_in, 0);
- atomic_set(&device->rs_sect_ev, 0);
+ atomic_set_unchecked(&device->rs_sect_in, 0);
+ atomic_set_unchecked(&device->rs_sect_ev, 0);
device->rs_in_flight = 0;
device->rs_last_events =
(int)part_stat_read(&disk->part0, sectors[0]) +
diff --git a/drivers/char/ipmi/ipmi_msghandler.c b/drivers/char/ipmi/ipmi_msghandler.c
index bf75f63..359fa10 100644
--- a/drivers/char/ipmi/ipmi_msghandler.c
+++ b/drivers/char/ipmi/ipmi_msghandler.c
@@ -436,7 +436,7 @@ struct ipmi_smi {
struct proc_dir_entry *proc_dir;
char proc_dir_name[10];
- atomic_t stats[IPMI_NUM_STATS];
+ atomic_unchecked_t stats[IPMI_NUM_STATS];
/*
* run_to_completion duplicate of smb_info, smi_info
@@ -468,9 +468,9 @@ static LIST_HEAD(smi_watchers);
static DEFINE_MUTEX(smi_watchers_mutex);
#define ipmi_inc_stat(intf, stat) \
- atomic_inc(&(intf)->stats[IPMI_STAT_ ## stat])
+ atomic_inc_unchecked(&(intf)->stats[IPMI_STAT_ ## stat])
#define ipmi_get_stat(intf, stat) \
- ((unsigned int) atomic_read(&(intf)->stats[IPMI_STAT_ ## stat]))
+ ((unsigned int) atomic_read_unchecked(&(intf)->stats[IPMI_STAT_ ## stat]))
static char *addr_src_to_str[] = { "invalid", "hotmod", "hardcoded", "SPMI",
"ACPI", "SMBIOS", "PCI",
@@ -2828,7 +2828,7 @@ int ipmi_register_smi(struct ipmi_smi_handlers *handlers,
INIT_LIST_HEAD(&intf->cmd_rcvrs);
init_waitqueue_head(&intf->waitq);
for (i = 0; i < IPMI_NUM_STATS; i++)
- atomic_set(&intf->stats[i], 0);
+ atomic_set_unchecked(&intf->stats[i], 0);
intf->proc_dir = NULL;
diff --git a/drivers/char/ipmi/ipmi_si_intf.c b/drivers/char/ipmi/ipmi_si_intf.c
index 8a45e92..e41b1c7 100644
--- a/drivers/char/ipmi/ipmi_si_intf.c
+++ b/drivers/char/ipmi/ipmi_si_intf.c
@@ -289,7 +289,7 @@ struct smi_info {
unsigned char slave_addr;
/* Counters and things for the proc filesystem. */
- atomic_t stats[SI_NUM_STATS];
+ atomic_unchecked_t stats[SI_NUM_STATS];
struct task_struct *thread;
@@ -298,9 +298,9 @@ struct smi_info {
};
#define smi_inc_stat(smi, stat) \
- atomic_inc(&(smi)->stats[SI_STAT_ ## stat])
+ atomic_inc_unchecked(&(smi)->stats[SI_STAT_ ## stat])
#define smi_get_stat(smi, stat) \
- ((unsigned int) atomic_read(&(smi)->stats[SI_STAT_ ## stat]))
+ ((unsigned int) atomic_read_unchecked(&(smi)->stats[SI_STAT_ ## stat]))
#define SI_MAX_PARMS 4
@@ -3500,7 +3500,7 @@ static int try_smi_init(struct smi_info *new_smi)
atomic_set(&new_smi->req_events, 0);
new_smi->run_to_completion = false;
for (i = 0; i < SI_NUM_STATS; i++)
- atomic_set(&new_smi->stats[i], 0);
+ atomic_set_unchecked(&new_smi->stats[i], 0);
new_smi->interrupt_disabled = true;
atomic_set(&new_smi->need_watch, 0);
diff --git a/drivers/crypto/hifn_795x.c b/drivers/crypto/hifn_795x.c
index 8d2a772..33826c9 100644
--- a/drivers/crypto/hifn_795x.c
+++ b/drivers/crypto/hifn_795x.c
@@ -51,7 +51,7 @@ module_param_string(hifn_pll_ref, hifn_pll_ref, sizeof(hifn_pll_ref), 0444);
MODULE_PARM_DESC(hifn_pll_ref,
"PLL reference clock (pci[freq] or ext[freq], default ext)");
-static atomic_t hifn_dev_number;
+static atomic_unchecked_t hifn_dev_number;
#define ACRYPTO_OP_DECRYPT 0
#define ACRYPTO_OP_ENCRYPT 1
@@ -2577,7 +2577,7 @@ static int hifn_probe(struct pci_dev *pdev, const struct pci_device_id *id)
goto err_out_disable_pci_device;
snprintf(name, sizeof(name), "hifn%d",
- atomic_inc_return(&hifn_dev_number)-1);
+ atomic_inc_return_unchecked(&hifn_dev_number)-1);
err = pci_request_regions(pdev, name);
if (err)
diff --git a/drivers/edac/edac_device.c b/drivers/edac/edac_device.c
index 592af5f..bb1d583 100644
--- a/drivers/edac/edac_device.c
+++ b/drivers/edac/edac_device.c
@@ -477,9 +477,9 @@ void edac_device_reset_delay_period(struct edac_device_ctl_info *edac_dev,
*/
int edac_device_alloc_index(void)
{
- static atomic_t device_indexes = ATOMIC_INIT(0);
+ static atomic_unchecked_t device_indexes = ATOMIC_INIT(0);
- return atomic_inc_return(&device_indexes) - 1;
+ return atomic_inc_return_unchecked(&device_indexes) - 1;
}
EXPORT_SYMBOL_GPL(edac_device_alloc_index);
diff --git a/drivers/edac/edac_pci.c b/drivers/edac/edac_pci.c
index 2cf44b4d..6dd2dc7 100644
--- a/drivers/edac/edac_pci.c
+++ b/drivers/edac/edac_pci.c
@@ -29,7 +29,7 @@
static DEFINE_MUTEX(edac_pci_ctls_mutex);
static LIST_HEAD(edac_pci_list);
-static atomic_t pci_indexes = ATOMIC_INIT(0);
+static atomic_unchecked_t pci_indexes = ATOMIC_INIT(0);
/*
* edac_pci_alloc_ctl_info
@@ -315,7 +315,7 @@ EXPORT_SYMBOL_GPL(edac_pci_reset_delay_period);
*/
int edac_pci_alloc_index(void)
{
- return atomic_inc_return(&pci_indexes) - 1;
+ return atomic_inc_return_unchecked(&pci_indexes) - 1;
}
EXPORT_SYMBOL_GPL(edac_pci_alloc_index);
diff --git a/drivers/edac/edac_pci_sysfs.c b/drivers/edac/edac_pci_sysfs.c
index 24d877f..40f4ecb 100644
--- a/drivers/edac/edac_pci_sysfs.c
+++ b/drivers/edac/edac_pci_sysfs.c
@@ -23,8 +23,8 @@ static int edac_pci_log_pe = 1; /* log PCI parity errors */
static int edac_pci_log_npe = 1; /* log PCI non-parity error errors */
static int edac_pci_poll_msec = 1000; /* one second workq period */
-static atomic_t pci_parity_count = ATOMIC_INIT(0);
-static atomic_t pci_nonparity_count = ATOMIC_INIT(0);
+static atomic_unchecked_t pci_parity_count = ATOMIC_INIT(0);
+static atomic_unchecked_t pci_nonparity_count = ATOMIC_INIT(0);
static struct kobject *edac_pci_top_main_kobj;
static atomic_t edac_pci_sysfs_refcount = ATOMIC_INIT(0);
@@ -576,7 +576,7 @@ static void edac_pci_dev_parity_test(struct pci_dev *dev)
edac_printk(KERN_CRIT, EDAC_PCI,
"Signaled System Error on %s\n",
pci_name(dev));
- atomic_inc(&pci_nonparity_count);
+ atomic_inc_unchecked(&pci_nonparity_count);
}
if (status & (PCI_STATUS_PARITY)) {
@@ -584,7 +584,7 @@ static void edac_pci_dev_parity_test(struct pci_dev *dev)
"Master Data Parity Error on %s\n",
pci_name(dev));
- atomic_inc(&pci_parity_count);
+ atomic_inc_unchecked(&pci_parity_count);
}
if (status & (PCI_STATUS_DETECTED_PARITY)) {
@@ -592,7 +592,7 @@ static void edac_pci_dev_parity_test(struct pci_dev *dev)
"Detected Parity Error on %s\n",
pci_name(dev));
- atomic_inc(&pci_parity_count);
+ atomic_inc_unchecked(&pci_parity_count);
}
}
@@ -615,7 +615,7 @@ static void edac_pci_dev_parity_test(struct pci_dev *dev)
edac_printk(KERN_CRIT, EDAC_PCI, "Bridge "
"Signaled System Error on %s\n",
pci_name(dev));
- atomic_inc(&pci_nonparity_count);
+ atomic_inc_unchecked(&pci_nonparity_count);
}
if (status & (PCI_STATUS_PARITY)) {
@@ -623,7 +623,7 @@ static void edac_pci_dev_parity_test(struct pci_dev *dev)
"Master Data Parity Error on "
"%s\n", pci_name(dev));
- atomic_inc(&pci_parity_count);
+ atomic_inc_unchecked(&pci_parity_count);
}
if (status & (PCI_STATUS_DETECTED_PARITY)) {
@@ -631,7 +631,7 @@ static void edac_pci_dev_parity_test(struct pci_dev *dev)
"Detected Parity Error on %s\n",
pci_name(dev));
- atomic_inc(&pci_parity_count);
+ atomic_inc_unchecked(&pci_parity_count);
}
}
}
@@ -669,7 +669,7 @@ void edac_pci_do_parity_check(void)
if (!check_pci_errors)
return;
- before_count = atomic_read(&pci_parity_count);
+ before_count = atomic_read_unchecked(&pci_parity_count);
/* scan all PCI devices looking for a Parity Error on devices and
* bridges.
@@ -681,7 +681,7 @@ void edac_pci_do_parity_check(void)
/* Only if operator has selected panic on PCI Error */
if (edac_pci_get_panic_on_pe()) {
/* If the count is different 'after' from 'before' */
- if (before_count != atomic_read(&pci_parity_count))
+ if (before_count != atomic_read_unchecked(&pci_parity_count))
panic("EDAC: PCI Parity Error");
}
}
diff --git a/drivers/firewire/core-card.c b/drivers/firewire/core-card.c
index 57ea7f4..9c461ac 100644
--- a/drivers/firewire/core-card.c
+++ b/drivers/firewire/core-card.c
@@ -528,9 +528,9 @@ void fw_card_initialize(struct fw_card *card,
const struct fw_card_driver *driver,
struct device *device)
{
- static atomic_t index = ATOMIC_INIT(-1);
+ static atomic_unchecked_t index = ATOMIC_INIT(-1);
- card->index = atomic_inc_return(&index);
+ card->index = atomic_inc_return_unchecked(&index);
card->driver = driver;
card->device = device;
card->current_tlabel = 0;
diff --git a/drivers/firmware/efi/cper.c b/drivers/firmware/efi/cper.c
index d425374..1da1716 100644
--- a/drivers/firmware/efi/cper.c
+++ b/drivers/firmware/efi/cper.c
@@ -44,12 +44,12 @@ static char rcd_decode_str[CPER_REC_LEN];
*/
u64 cper_next_record_id(void)
{
- static atomic64_t seq;
+ static atomic64_unchecked_t seq;
- if (!atomic64_read(&seq))
- atomic64_set(&seq, ((u64)get_seconds()) << 32);
+ if (!atomic64_read_unchecked(&seq))
+ atomic64_set_unchecked(&seq, ((u64)get_seconds()) << 32);
- return atomic64_inc_return(&seq);
+ return atomic64_inc_return_unchecked(&seq);
}
EXPORT_SYMBOL_GPL(cper_next_record_id);
diff --git a/drivers/gpio/gpio-vr41xx.c b/drivers/gpio/gpio-vr41xx.c
index c1caa45..f0f97d2 100644
--- a/drivers/gpio/gpio-vr41xx.c
+++ b/drivers/gpio/gpio-vr41xx.c
@@ -224,7 +224,7 @@ static int giu_get_irq(unsigned int irq)
printk(KERN_ERR "spurious GIU interrupt: %04x(%04x),%04x(%04x)\n",
maskl, pendl, maskh, pendh);
- atomic_inc(&irq_err_count);
+ atomic_inc_unchecked(&irq_err_count);
return -EINVAL;
}
diff --git a/drivers/gpu/drm/i810/i810_drv.h b/drivers/gpu/drm/i810/i810_drv.h
index 93ec5dc..82acbaf 100644
--- a/drivers/gpu/drm/i810/i810_drv.h
+++ b/drivers/gpu/drm/i810/i810_drv.h
@@ -110,8 +110,8 @@ typedef struct drm_i810_private {
int page_flipping;
wait_queue_head_t irq_queue;
- atomic_t irq_received;
- atomic_t irq_emitted;
+ atomic_unchecked_t irq_received;
+ atomic_unchecked_t irq_emitted;
int front_offset;
} drm_i810_private_t;
diff --git a/drivers/gpu/drm/mga/mga_drv.h b/drivers/gpu/drm/mga/mga_drv.h
index b4a20149..219ab78 100644
--- a/drivers/gpu/drm/mga/mga_drv.h
+++ b/drivers/gpu/drm/mga/mga_drv.h
@@ -122,9 +122,9 @@ typedef struct drm_mga_private {
u32 clear_cmd;
u32 maccess;
- atomic_t vbl_received; /**< Number of vblanks received. */
+ atomic_unchecked_t vbl_received; /**< Number of vblanks received. */
wait_queue_head_t fence_queue;
- atomic_t last_fence_retired;
+ atomic_unchecked_t last_fence_retired;
u32 next_fence_to_post;
unsigned int fb_cpp;
diff --git a/drivers/gpu/drm/mga/mga_irq.c b/drivers/gpu/drm/mga/mga_irq.c
index 1b071b8..de8601a 100644
--- a/drivers/gpu/drm/mga/mga_irq.c
+++ b/drivers/gpu/drm/mga/mga_irq.c
@@ -43,7 +43,7 @@ u32 mga_get_vblank_counter(struct drm_device *dev, int crtc)
if (crtc != 0)
return 0;
- return atomic_read(&dev_priv->vbl_received);
+ return atomic_read_unchecked(&dev_priv->vbl_received);
}
@@ -59,7 +59,7 @@ irqreturn_t mga_driver_irq_handler(int irq, void *arg)
/* VBLANK interrupt */
if (status & MGA_VLINEPEN) {
MGA_WRITE(MGA_ICLEAR, MGA_VLINEICLR);
- atomic_inc(&dev_priv->vbl_received);
+ atomic_inc_unchecked(&dev_priv->vbl_received);
drm_handle_vblank(dev, 0);
handled = 1;
}
@@ -78,7 +78,7 @@ irqreturn_t mga_driver_irq_handler(int irq, void *arg)
if ((prim_start & ~0x03) != (prim_end & ~0x03))
MGA_WRITE(MGA_PRIMEND, prim_end);
- atomic_inc(&dev_priv->last_fence_retired);
+ atomic_inc_unchecked(&dev_priv->last_fence_retired);
wake_up(&dev_priv->fence_queue);
handled = 1;
}
@@ -129,7 +129,7 @@ int mga_driver_fence_wait(struct drm_device *dev, unsigned int *sequence)
* using fences.
*/
DRM_WAIT_ON(ret, dev_priv->fence_queue, 3 * HZ,
- (((cur_fence = atomic_read(&dev_priv->last_fence_retired))
+ (((cur_fence = atomic_read_unchecked(&dev_priv->last_fence_retired))
- *sequence) <= (1 << 23)));
*sequence = cur_fence;
diff --git a/drivers/gpu/drm/qxl/qxl_cmd.c b/drivers/gpu/drm/qxl/qxl_cmd.c
index fdc1833..f307630 100644
--- a/drivers/gpu/drm/qxl/qxl_cmd.c
+++ b/drivers/gpu/drm/qxl/qxl_cmd.c
@@ -285,27 +285,27 @@ static int wait_for_io_cmd_user(struct qxl_device *qdev, uint8_t val, long port,
int ret;
mutex_lock(&qdev->async_io_mutex);
- irq_num = atomic_read(&qdev->irq_received_io_cmd);
+ irq_num = atomic_read_unchecked(&qdev->irq_received_io_cmd);
if (qdev->last_sent_io_cmd > irq_num) {
if (intr)
ret = wait_event_interruptible_timeout(qdev->io_cmd_event,
- atomic_read(&qdev->irq_received_io_cmd) > irq_num, 5*HZ);
+ atomic_read_unchecked(&qdev->irq_received_io_cmd) > irq_num, 5*HZ);
else
ret = wait_event_timeout(qdev->io_cmd_event,
- atomic_read(&qdev->irq_received_io_cmd) > irq_num, 5*HZ);
+ atomic_read_unchecked(&qdev->irq_received_io_cmd) > irq_num, 5*HZ);
/* 0 is timeout, just bail the "hw" has gone away */
if (ret <= 0)
goto out;
- irq_num = atomic_read(&qdev->irq_received_io_cmd);
+ irq_num = atomic_read_unchecked(&qdev->irq_received_io_cmd);
}
outb(val, addr);
qdev->last_sent_io_cmd = irq_num + 1;
if (intr)
ret = wait_event_interruptible_timeout(qdev->io_cmd_event,
- atomic_read(&qdev->irq_received_io_cmd) > irq_num, 5*HZ);
+ atomic_read_unchecked(&qdev->irq_received_io_cmd) > irq_num, 5*HZ);
else
ret = wait_event_timeout(qdev->io_cmd_event,
- atomic_read(&qdev->irq_received_io_cmd) > irq_num, 5*HZ);
+ atomic_read_unchecked(&qdev->irq_received_io_cmd) > irq_num, 5*HZ);
out:
if (ret > 0)
ret = 0;
diff --git a/drivers/gpu/drm/qxl/qxl_debugfs.c b/drivers/gpu/drm/qxl/qxl_debugfs.c
index 6911b8c..89d6867 100644
--- a/drivers/gpu/drm/qxl/qxl_debugfs.c
+++ b/drivers/gpu/drm/qxl/qxl_debugfs.c
@@ -42,10 +42,10 @@ qxl_debugfs_irq_received(struct seq_file *m, void *data)
struct drm_info_node *node = (struct drm_info_node *) m->private;
struct qxl_device *qdev = node->minor->dev->dev_private;
- seq_printf(m, "%d\n", atomic_read(&qdev->irq_received));
- seq_printf(m, "%d\n", atomic_read(&qdev->irq_received_display));
- seq_printf(m, "%d\n", atomic_read(&qdev->irq_received_cursor));
- seq_printf(m, "%d\n", atomic_read(&qdev->irq_received_io_cmd));
+ seq_printf(m, "%d\n", atomic_read_unchecked(&qdev->irq_received));
+ seq_printf(m, "%d\n", atomic_read_unchecked(&qdev->irq_received_display));
+ seq_printf(m, "%d\n", atomic_read_unchecked(&qdev->irq_received_cursor));
+ seq_printf(m, "%d\n", atomic_read_unchecked(&qdev->irq_received_io_cmd));
seq_printf(m, "%d\n", qdev->irq_received_error);
return 0;
}
diff --git a/drivers/gpu/drm/qxl/qxl_drv.h b/drivers/gpu/drm/qxl/qxl_drv.h
index 01a8694..584fb48 100644
--- a/drivers/gpu/drm/qxl/qxl_drv.h
+++ b/drivers/gpu/drm/qxl/qxl_drv.h
@@ -290,10 +290,10 @@ struct qxl_device {
unsigned int last_sent_io_cmd;
/* interrupt handling */
- atomic_t irq_received;
- atomic_t irq_received_display;
- atomic_t irq_received_cursor;
- atomic_t irq_received_io_cmd;
+ atomic_unchecked_t irq_received;
+ atomic_unchecked_t irq_received_display;
+ atomic_unchecked_t irq_received_cursor;
+ atomic_unchecked_t irq_received_io_cmd;
unsigned irq_received_error;
wait_queue_head_t display_event;
wait_queue_head_t cursor_event;
diff --git a/drivers/gpu/drm/qxl/qxl_irq.c b/drivers/gpu/drm/qxl/qxl_irq.c
index 0bf1e20..42a7310 100644
--- a/drivers/gpu/drm/qxl/qxl_irq.c
+++ b/drivers/gpu/drm/qxl/qxl_irq.c
@@ -36,19 +36,19 @@ irqreturn_t qxl_irq_handler(int irq, void *arg)
if (!pending)
return IRQ_NONE;
- atomic_inc(&qdev->irq_received);
+ atomic_inc_unchecked(&qdev->irq_received);
if (pending & QXL_INTERRUPT_DISPLAY) {
- atomic_inc(&qdev->irq_received_display);
+ atomic_inc_unchecked(&qdev->irq_received_display);
wake_up_all(&qdev->display_event);
qxl_queue_garbage_collect(qdev, false);
}
if (pending & QXL_INTERRUPT_CURSOR) {
- atomic_inc(&qdev->irq_received_cursor);
+ atomic_inc_unchecked(&qdev->irq_received_cursor);
wake_up_all(&qdev->cursor_event);
}
if (pending & QXL_INTERRUPT_IO_CMD) {
- atomic_inc(&qdev->irq_received_io_cmd);
+ atomic_inc_unchecked(&qdev->irq_received_io_cmd);
wake_up_all(&qdev->io_cmd_event);
}
if (pending & QXL_INTERRUPT_ERROR) {
@@ -85,10 +85,10 @@ int qxl_irq_init(struct qxl_device *qdev)
init_waitqueue_head(&qdev->io_cmd_event);
INIT_WORK(&qdev->client_monitors_config_work,
qxl_client_monitors_config_work_func);
- atomic_set(&qdev->irq_received, 0);
- atomic_set(&qdev->irq_received_display, 0);
- atomic_set(&qdev->irq_received_cursor, 0);
- atomic_set(&qdev->irq_received_io_cmd, 0);
+ atomic_set_unchecked(&qdev->irq_received, 0);
+ atomic_set_unchecked(&qdev->irq_received_display, 0);
+ atomic_set_unchecked(&qdev->irq_received_cursor, 0);
+ atomic_set_unchecked(&qdev->irq_received_io_cmd, 0);
qdev->irq_received_error = 0;
ret = drm_irq_install(qdev->ddev, qdev->ddev->pdev->irq);
qdev->ram_header->int_mask = QXL_INTERRUPT_MASK;
diff --git a/drivers/gpu/drm/r128/r128_cce.c b/drivers/gpu/drm/r128/r128_cce.c
index 2c45ac9..5d740f8 100644
--- a/drivers/gpu/drm/r128/r128_cce.c
+++ b/drivers/gpu/drm/r128/r128_cce.c
@@ -377,7 +377,7 @@ static int r128_do_init_cce(struct drm_device *dev, drm_r128_init_t *init)
/* GH: Simple idle check.
*/
- atomic_set(&dev_priv->idle_count, 0);
+ atomic_set_unchecked(&dev_priv->idle_count, 0);
/* We don't support anything other than bus-mastering ring mode,
* but the ring can be in either AGP or PCI space for the ring
diff --git a/drivers/gpu/drm/r128/r128_drv.h b/drivers/gpu/drm/r128/r128_drv.h
index 723e5d6..102dbaf 100644
--- a/drivers/gpu/drm/r128/r128_drv.h
+++ b/drivers/gpu/drm/r128/r128_drv.h
@@ -93,14 +93,14 @@ typedef struct drm_r128_private {
int is_pci;
unsigned long cce_buffers_offset;
- atomic_t idle_count;
+ atomic_unchecked_t idle_count;
int page_flipping;
int current_page;
u32 crtc_offset;
u32 crtc_offset_cntl;
- atomic_t vbl_received;
+ atomic_unchecked_t vbl_received;
u32 color_fmt;
unsigned int front_offset;
diff --git a/drivers/gpu/drm/r128/r128_irq.c b/drivers/gpu/drm/r128/r128_irq.c
index c2ae496..30b5993 100644
--- a/drivers/gpu/drm/r128/r128_irq.c
+++ b/drivers/gpu/drm/r128/r128_irq.c
@@ -41,7 +41,7 @@ u32 r128_get_vblank_counter(struct drm_device *dev, int crtc)
if (crtc != 0)
return 0;
- return atomic_read(&dev_priv->vbl_received);
+ return atomic_read_unchecked(&dev_priv->vbl_received);
}
irqreturn_t r128_driver_irq_handler(int irq, void *arg)
@@ -55,7 +55,7 @@ irqreturn_t r128_driver_irq_handler(int irq, void *arg)
/* VBLANK interrupt */
if (status & R128_CRTC_VBLANK_INT) {
R128_WRITE(R128_GEN_INT_STATUS, R128_CRTC_VBLANK_INT_AK);
- atomic_inc(&dev_priv->vbl_received);
+ atomic_inc_unchecked(&dev_priv->vbl_received);
drm_handle_vblank(dev, 0);
return IRQ_HANDLED;
}
diff --git a/drivers/gpu/drm/r128/r128_state.c b/drivers/gpu/drm/r128/r128_state.c
index 8fd2d9f..18c9660 100644
--- a/drivers/gpu/drm/r128/r128_state.c
+++ b/drivers/gpu/drm/r128/r128_state.c
@@ -320,10 +320,10 @@ static void r128_clear_box(drm_r128_private_t *dev_priv,
static void r128_cce_performance_boxes(drm_r128_private_t *dev_priv)
{
- if (atomic_read(&dev_priv->idle_count) == 0)
+ if (atomic_read_unchecked(&dev_priv->idle_count) == 0)
r128_clear_box(dev_priv, 64, 4, 8, 8, 0, 255, 0);
else
- atomic_set(&dev_priv->idle_count, 0);
+ atomic_set_unchecked(&dev_priv->idle_count, 0);
}
#endif
diff --git a/drivers/gpu/drm/radeon/radeon_drv.h b/drivers/gpu/drm/radeon/radeon_drv.h
index 46bd393..6ae4719 100644
--- a/drivers/gpu/drm/radeon/radeon_drv.h
+++ b/drivers/gpu/drm/radeon/radeon_drv.h
@@ -264,7 +264,7 @@ typedef struct drm_radeon_private {
/* SW interrupt */
wait_queue_head_t swi_queue;
- atomic_t swi_emitted;
+ atomic_unchecked_t swi_emitted;
int vblank_crtc;
uint32_t irq_enable_reg;
uint32_t r500_disp_irq_reg;
diff --git a/drivers/gpu/drm/radeon/radeon_irq.c b/drivers/gpu/drm/radeon/radeon_irq.c
index 244b19b..c19226d 100644
--- a/drivers/gpu/drm/radeon/radeon_irq.c
+++ b/drivers/gpu/drm/radeon/radeon_irq.c
@@ -226,8 +226,8 @@ static int radeon_emit_irq(struct drm_device * dev)
unsigned int ret;
RING_LOCALS;
- atomic_inc(&dev_priv->swi_emitted);
- ret = atomic_read(&dev_priv->swi_emitted);
+ atomic_inc_unchecked(&dev_priv->swi_emitted);
+ ret = atomic_read_unchecked(&dev_priv->swi_emitted);
BEGIN_RING(4);
OUT_RING_REG(RADEON_LAST_SWI_REG, ret);
@@ -353,7 +353,7 @@ int radeon_driver_irq_postinstall(struct drm_device *dev)
drm_radeon_private_t *dev_priv =
(drm_radeon_private_t *) dev->dev_private;
- atomic_set(&dev_priv->swi_emitted, 0);
+ atomic_set_unchecked(&dev_priv->swi_emitted, 0);
init_waitqueue_head(&dev_priv->swi_queue);
dev->max_vblank_count = 0x001fffff;
diff --git a/drivers/gpu/drm/via/via_drv.h b/drivers/gpu/drm/via/via_drv.h
index ef8c500..01030c8 100644
--- a/drivers/gpu/drm/via/via_drv.h
+++ b/drivers/gpu/drm/via/via_drv.h
@@ -53,7 +53,7 @@ typedef struct drm_via_ring_buffer {
typedef uint32_t maskarray_t[5];
typedef struct drm_via_irq {
- atomic_t irq_received;
+ atomic_unchecked_t irq_received;
uint32_t pending_mask;
uint32_t enable_mask;
wait_queue_head_t irq_queue;
@@ -77,7 +77,7 @@ typedef struct drm_via_private {
struct timeval last_vblank;
int last_vblank_valid;
unsigned usec_per_vblank;
- atomic_t vbl_received;
+ atomic_unchecked_t vbl_received;
drm_via_state_t hc_state;
char pci_buf[VIA_PCI_BUF_SIZE];
const uint32_t *fire_offsets[VIA_FIRE_BUF_SIZE];
diff --git a/drivers/gpu/drm/via/via_irq.c b/drivers/gpu/drm/via/via_irq.c
index 1319433..a993b0c 100644
--- a/drivers/gpu/drm/via/via_irq.c
+++ b/drivers/gpu/drm/via/via_irq.c
@@ -101,7 +101,7 @@ u32 via_get_vblank_counter(struct drm_device *dev, int crtc)
if (crtc != 0)
return 0;
- return atomic_read(&dev_priv->vbl_received);
+ return atomic_read_unchecked(&dev_priv->vbl_received);
}
irqreturn_t via_driver_irq_handler(int irq, void *arg)
@@ -116,8 +116,8 @@ irqreturn_t via_driver_irq_handler(int irq, void *arg)
status = VIA_READ(VIA_REG_INTERRUPT);
if (status & VIA_IRQ_VBLANK_PENDING) {
- atomic_inc(&dev_priv->vbl_received);
- if (!(atomic_read(&dev_priv->vbl_received) & 0x0F)) {
+ atomic_inc_unchecked(&dev_priv->vbl_received);
+ if (!(atomic_read_unchecked(&dev_priv->vbl_received) & 0x0F)) {
do_gettimeofday(&cur_vblank);
if (dev_priv->last_vblank_valid) {
dev_priv->usec_per_vblank =
@@ -127,7 +127,7 @@ irqreturn_t via_driver_irq_handler(int irq, void *arg)
dev_priv->last_vblank = cur_vblank;
dev_priv->last_vblank_valid = 1;
}
- if (!(atomic_read(&dev_priv->vbl_received) & 0xFF)) {
+ if (!(atomic_read_unchecked(&dev_priv->vbl_received) & 0xFF)) {
DRM_DEBUG("US per vblank is: %u\n",
dev_priv->usec_per_vblank);
}
@@ -137,7 +137,7 @@ irqreturn_t via_driver_irq_handler(int irq, void *arg)
for (i = 0; i < dev_priv->num_irqs; ++i) {
if (status & cur_irq->pending_mask) {
- atomic_inc(&cur_irq->irq_received);
+ atomic_inc_unchecked(&cur_irq->irq_received);
wake_up(&cur_irq->irq_queue);
handled = 1;
if (dev_priv->irq_map[drm_via_irq_dma0_td] == i)
@@ -242,11 +242,11 @@ via_driver_irq_wait(struct drm_device *dev, unsigned int irq, int force_sequence
DRM_WAIT_ON(ret, cur_irq->irq_queue, 3 * HZ,
((VIA_READ(masks[irq][2]) & masks[irq][3]) ==
masks[irq][4]));
- cur_irq_sequence = atomic_read(&cur_irq->irq_received);
+ cur_irq_sequence = atomic_read_unchecked(&cur_irq->irq_received);
} else {
DRM_WAIT_ON(ret, cur_irq->irq_queue, 3 * HZ,
(((cur_irq_sequence =
- atomic_read(&cur_irq->irq_received)) -
+ atomic_read_unchecked(&cur_irq->irq_received)) -
*sequence) <= (1 << 23)));
}
*sequence = cur_irq_sequence;
@@ -284,7 +284,7 @@ void via_driver_irq_preinstall(struct drm_device *dev)
}
for (i = 0; i < dev_priv->num_irqs; ++i) {
- atomic_set(&cur_irq->irq_received, 0);
+ atomic_set_unchecked(&cur_irq->irq_received, 0);
cur_irq->enable_mask = dev_priv->irq_masks[i][0];
cur_irq->pending_mask = dev_priv->irq_masks[i][1];
init_waitqueue_head(&cur_irq->irq_queue);
@@ -366,7 +366,7 @@ int via_wait_irq(struct drm_device *dev, void *data, struct drm_file *file_priv)
switch (irqwait->request.type & ~VIA_IRQ_FLAGS_MASK) {
case VIA_IRQ_RELATIVE:
irqwait->request.sequence +=
- atomic_read(&cur_irq->irq_received);
+ atomic_read_unchecked(&cur_irq->irq_received);
irqwait->request.type &= ~_DRM_VBLANK_RELATIVE;
case VIA_IRQ_ABSOLUTE:
break;
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h
index d8896ed..f5ca485 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h
@@ -447,7 +447,7 @@ struct vmw_private {
* Fencing and IRQs.
*/
- atomic_t marker_seq;
+ atomic_unchecked_t marker_seq;
wait_queue_head_t fence_queue;
wait_queue_head_t fifo_queue;
spinlock_t waiter_lock;
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_fifo.c b/drivers/gpu/drm/vmwgfx/vmwgfx_fifo.c
index 39f2b03..004a773 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_fifo.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_fifo.c
@@ -152,7 +152,7 @@ int vmw_fifo_init(struct vmw_private *dev_priv, struct vmw_fifo_state *fifo)
(unsigned int) min,
(unsigned int) fifo->capabilities);
- atomic_set(&dev_priv->marker_seq, dev_priv->last_read_seqno);
+ atomic_set_unchecked(&dev_priv->marker_seq, dev_priv->last_read_seqno);
iowrite32(dev_priv->last_read_seqno, fifo_mem + SVGA_FIFO_FENCE);
vmw_marker_queue_init(&fifo->marker_queue);
return vmw_fifo_send_fence(dev_priv, &dummy);
@@ -492,7 +492,7 @@ int vmw_fifo_send_fence(struct vmw_private *dev_priv, uint32_t *seqno)
fm = vmw_fifo_reserve(dev_priv, bytes);
if (unlikely(fm == NULL)) {
- *seqno = atomic_read(&dev_priv->marker_seq);
+ *seqno = atomic_read_unchecked(&dev_priv->marker_seq);
ret = -ENOMEM;
(void)vmw_fallback_wait(dev_priv, false, true, *seqno,
false, 3*HZ);
@@ -500,7 +500,7 @@ int vmw_fifo_send_fence(struct vmw_private *dev_priv, uint32_t *seqno)
}
do {
- *seqno = atomic_add_return(1, &dev_priv->marker_seq);
+ *seqno = atomic_add_return_unchecked(1, &dev_priv->marker_seq);
} while (*seqno == 0);
if (!(fifo_state->capabilities & SVGA_FIFO_CAP_FENCE)) {
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_irq.c b/drivers/gpu/drm/vmwgfx/vmwgfx_irq.c
index 9fe9827..0aa2fc0 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_irq.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_irq.c
@@ -102,7 +102,7 @@ bool vmw_seqno_passed(struct vmw_private *dev_priv,
* emitted. Then the fence is stale and signaled.
*/
- ret = ((atomic_read(&dev_priv->marker_seq) - seqno)
+ ret = ((atomic_read_unchecked(&dev_priv->marker_seq) - seqno)
> VMW_FENCE_WRAP);
return ret;
@@ -133,7 +133,7 @@ int vmw_fallback_wait(struct vmw_private *dev_priv,
if (fifo_idle)
down_read(&fifo_state->rwsem);
- signal_seq = atomic_read(&dev_priv->marker_seq);
+ signal_seq = atomic_read_unchecked(&dev_priv->marker_seq);
ret = 0;
for (;;) {
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_marker.c b/drivers/gpu/drm/vmwgfx/vmwgfx_marker.c
index efd1ffd..0ae13ca 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_marker.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_marker.c
@@ -135,7 +135,7 @@ int vmw_wait_lag(struct vmw_private *dev_priv,
while (!vmw_lag_lt(queue, us)) {
spin_lock(&queue->lock);
if (list_empty(&queue->head))
- seqno = atomic_read(&dev_priv->marker_seq);
+ seqno = atomic_read_unchecked(&dev_priv->marker_seq);
else {
marker = list_first_entry(&queue->head,
struct vmw_marker, head);
diff --git a/drivers/hid/hid-core.c b/drivers/hid/hid-core.c
index e6fce23..85949a0 100644
--- a/drivers/hid/hid-core.c
+++ b/drivers/hid/hid-core.c
@@ -2550,7 +2550,7 @@ EXPORT_SYMBOL_GPL(hid_ignore);
int hid_add_device(struct hid_device *hdev)
{
- static atomic_t id = ATOMIC_INIT(0);
+ static atomic_unchecked_t id = ATOMIC_INIT(0);
int ret;
if (WARN_ON(hdev->status & HID_STAT_ADDED))
@@ -2593,7 +2593,7 @@ int hid_add_device(struct hid_device *hdev)
/* XXX hack, any other cleaner solution after the driver core
* is converted to allow more than 20 bytes as the device name? */
dev_set_name(&hdev->dev, "%04X:%04X:%04X.%04X", hdev->bus,
- hdev->vendor, hdev->product, atomic_inc_return(&id));
+ hdev->vendor, hdev->product, atomic_inc_return_unchecked(&id));
hid_debug_register(hdev, dev_name(&hdev->dev));
ret = device_add(&hdev->dev);
diff --git a/drivers/hv/channel.c b/drivers/hv/channel.c
index 603ce97..7f27468 100644
--- a/drivers/hv/channel.c
+++ b/drivers/hv/channel.c
@@ -382,7 +382,7 @@ int vmbus_establish_gpadl(struct vmbus_channel *channel, void *kbuffer,
int ret = 0;
next_gpadl_handle =
- (atomic_inc_return(&vmbus_connection.next_gpadl_handle) - 1);
+ (atomic_inc_return_unchecked(&vmbus_connection.next_gpadl_handle) - 1);
ret = create_gpadl_header(kbuffer, size, &msginfo, &msgcount);
if (ret)
diff --git a/drivers/hv/hv_balloon.c b/drivers/hv/hv_balloon.c
index 8a725cd..91abaf0 100644
--- a/drivers/hv/hv_balloon.c
+++ b/drivers/hv/hv_balloon.c
@@ -469,7 +469,7 @@ MODULE_PARM_DESC(hot_add, "If set attempt memory hot_add");
module_param(pressure_report_delay, uint, (S_IRUGO | S_IWUSR));
MODULE_PARM_DESC(pressure_report_delay, "Delay in secs in reporting pressure");
-static atomic_t trans_id = ATOMIC_INIT(0);
+static atomic_unchecked_t trans_id = ATOMIC_INIT(0);
static int dm_ring_size = (5 * PAGE_SIZE);
@@ -943,7 +943,7 @@ static void hot_add_req(struct work_struct *dummy)
pr_info("Memory hot add failed\n");
dm->state = DM_INITIALIZED;
- resp.hdr.trans_id = atomic_inc_return(&trans_id);
+ resp.hdr.trans_id = atomic_inc_return_unchecked(&trans_id);
vmbus_sendpacket(dm->dev->channel, &resp,
sizeof(struct dm_hot_add_response),
(unsigned long)NULL,
@@ -1024,7 +1024,7 @@ static void post_status(struct hv_dynmem_device *dm)
memset(&status, 0, sizeof(struct dm_status));
status.hdr.type = DM_STATUS_REPORT;
status.hdr.size = sizeof(struct dm_status);
- status.hdr.trans_id = atomic_inc_return(&trans_id);
+ status.hdr.trans_id = atomic_inc_return_unchecked(&trans_id);
/*
* The host expects the guest to report free and committed memory.
@@ -1048,7 +1048,7 @@ static void post_status(struct hv_dynmem_device *dm)
* send the status. This can happen if we were interrupted
* after we picked our transaction ID.
*/
- if (status.hdr.trans_id != atomic_read(&trans_id))
+ if (status.hdr.trans_id != atomic_read_unchecked(&trans_id))
return;
/*
@@ -1193,7 +1193,7 @@ static void balloon_up(struct work_struct *dummy)
*/
do {
- bl_resp->hdr.trans_id = atomic_inc_return(&trans_id);
+ bl_resp->hdr.trans_id = atomic_inc_return_unchecked(&trans_id);
ret = vmbus_sendpacket(dm_device.dev->channel,
bl_resp,
bl_resp->hdr.size,
@@ -1239,7 +1239,7 @@ static void balloon_down(struct hv_dynmem_device *dm,
memset(&resp, 0, sizeof(struct dm_unballoon_response));
resp.hdr.type = DM_UNBALLOON_RESPONSE;
- resp.hdr.trans_id = atomic_inc_return(&trans_id);
+ resp.hdr.trans_id = atomic_inc_return_unchecked(&trans_id);
resp.hdr.size = sizeof(struct dm_unballoon_response);
vmbus_sendpacket(dm_device.dev->channel, &resp,
@@ -1300,7 +1300,7 @@ static void version_resp(struct hv_dynmem_device *dm,
memset(&version_req, 0, sizeof(struct dm_version_request));
version_req.hdr.type = DM_VERSION_REQUEST;
version_req.hdr.size = sizeof(struct dm_version_request);
- version_req.hdr.trans_id = atomic_inc_return(&trans_id);
+ version_req.hdr.trans_id = atomic_inc_return_unchecked(&trans_id);
version_req.version.version = DYNMEM_PROTOCOL_VERSION_WIN7;
version_req.is_last_attempt = 1;
@@ -1473,7 +1473,7 @@ static int balloon_probe(struct hv_device *dev,
memset(&version_req, 0, sizeof(struct dm_version_request));
version_req.hdr.type = DM_VERSION_REQUEST;
version_req.hdr.size = sizeof(struct dm_version_request);
- version_req.hdr.trans_id = atomic_inc_return(&trans_id);
+ version_req.hdr.trans_id = atomic_inc_return_unchecked(&trans_id);
version_req.version.version = DYNMEM_PROTOCOL_VERSION_WIN8;
version_req.is_last_attempt = 0;
@@ -1504,7 +1504,7 @@ static int balloon_probe(struct hv_device *dev,
memset(&cap_msg, 0, sizeof(struct dm_capabilities));
cap_msg.hdr.type = DM_CAPABILITIES_REPORT;
cap_msg.hdr.size = sizeof(struct dm_capabilities);
- cap_msg.hdr.trans_id = atomic_inc_return(&trans_id);
+ cap_msg.hdr.trans_id = atomic_inc_return_unchecked(&trans_id);
cap_msg.caps.cap_bits.balloon = 1;
cap_msg.caps.cap_bits.hot_add = 1;
diff --git a/drivers/hv/hyperv_vmbus.h b/drivers/hv/hyperv_vmbus.h
index cddc0c9..2eb587d 100644
--- a/drivers/hv/hyperv_vmbus.h
+++ b/drivers/hv/hyperv_vmbus.h
@@ -645,7 +645,7 @@ enum vmbus_connect_state {
struct vmbus_connection {
enum vmbus_connect_state conn_state;
- atomic_t next_gpadl_handle;
+ atomic_unchecked_t next_gpadl_handle;
struct completion unload_event;
/*
diff --git a/drivers/hwmon/sht15.c b/drivers/hwmon/sht15.c
index 497a7f8..3fffedf 100644
--- a/drivers/hwmon/sht15.c
+++ b/drivers/hwmon/sht15.c
@@ -169,7 +169,7 @@ struct sht15_data {
int supply_uv;
bool supply_uv_valid;
struct work_struct update_supply_work;
- atomic_t interrupt_handled;
+ atomic_unchecked_t interrupt_handled;
};
/**
@@ -542,13 +542,13 @@ static int sht15_measurement(struct sht15_data *data,
ret = gpio_direction_input(data->pdata->gpio_data);
if (ret)
return ret;
- atomic_set(&data->interrupt_handled, 0);
+ atomic_set_unchecked(&data->interrupt_handled, 0);
enable_irq(gpio_to_irq(data->pdata->gpio_data));
if (gpio_get_value(data->pdata->gpio_data) == 0) {
disable_irq_nosync(gpio_to_irq(data->pdata->gpio_data));
/* Only relevant if the interrupt hasn't occurred. */
- if (!atomic_read(&data->interrupt_handled))
+ if (!atomic_read_unchecked(&data->interrupt_handled))
schedule_work(&data->read_work);
}
ret = wait_event_timeout(data->wait_queue,
@@ -820,7 +820,7 @@ static irqreturn_t sht15_interrupt_fired(int irq, void *d)
/* First disable the interrupt */
disable_irq_nosync(irq);
- atomic_inc(&data->interrupt_handled);
+ atomic_inc_unchecked(&data->interrupt_handled);
/* Then schedule a reading work struct */
if (data->state != SHT15_READING_NOTHING)
schedule_work(&data->read_work);
@@ -842,11 +842,11 @@ static void sht15_bh_read_data(struct work_struct *work_s)
* If not, then start the interrupt again - care here as could
* have gone low in meantime so verify it hasn't!
*/
- atomic_set(&data->interrupt_handled, 0);
+ atomic_set_unchecked(&data->interrupt_handled, 0);
enable_irq(gpio_to_irq(data->pdata->gpio_data));
/* If still not occurred or another handler was scheduled */
if (gpio_get_value(data->pdata->gpio_data)
- || atomic_read(&data->interrupt_handled))
+ || atomic_read_unchecked(&data->interrupt_handled))
return;
}
diff --git a/drivers/infiniband/core/cm.c b/drivers/infiniband/core/cm.c
index 8be7352..02f9653 100644
--- a/drivers/infiniband/core/cm.c
+++ b/drivers/infiniband/core/cm.c
@@ -115,7 +115,7 @@ static char const counter_group_names[CM_COUNTER_GROUPS]
struct cm_counter_group {
struct kobject obj;
- atomic_long_t counter[CM_ATTR_COUNT];
+ atomic_long_unchecked_t counter[CM_ATTR_COUNT];
};
struct cm_counter_attribute {
@@ -1416,7 +1416,7 @@ static void cm_dup_req_handler(struct cm_work *work,
struct ib_mad_send_buf *msg = NULL;
int ret;
- atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
+ atomic_long_inc_unchecked(&work->port->counter_group[CM_RECV_DUPLICATES].
counter[CM_REQ_COUNTER]);
/* Quick state check to discard duplicate REQs. */
@@ -1803,7 +1803,7 @@ static void cm_dup_rep_handler(struct cm_work *work)
if (!cm_id_priv)
return;
- atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
+ atomic_long_inc_unchecked(&work->port->counter_group[CM_RECV_DUPLICATES].
counter[CM_REP_COUNTER]);
ret = cm_alloc_response_msg(work->port, work->mad_recv_wc, &msg);
if (ret)
@@ -1970,7 +1970,7 @@ static int cm_rtu_handler(struct cm_work *work)
if (cm_id_priv->id.state != IB_CM_REP_SENT &&
cm_id_priv->id.state != IB_CM_MRA_REP_RCVD) {
spin_unlock_irq(&cm_id_priv->lock);
- atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
+ atomic_long_inc_unchecked(&work->port->counter_group[CM_RECV_DUPLICATES].
counter[CM_RTU_COUNTER]);
goto out;
}
@@ -2153,7 +2153,7 @@ static int cm_dreq_handler(struct cm_work *work)
cm_id_priv = cm_acquire_id(dreq_msg->remote_comm_id,
dreq_msg->local_comm_id);
if (!cm_id_priv) {
- atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
+ atomic_long_inc_unchecked(&work->port->counter_group[CM_RECV_DUPLICATES].
counter[CM_DREQ_COUNTER]);
cm_issue_drep(work->port, work->mad_recv_wc);
return -EINVAL;
@@ -2178,7 +2178,7 @@ static int cm_dreq_handler(struct cm_work *work)
case IB_CM_MRA_REP_RCVD:
break;
case IB_CM_TIMEWAIT:
- atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
+ atomic_long_inc_unchecked(&work->port->counter_group[CM_RECV_DUPLICATES].
counter[CM_DREQ_COUNTER]);
if (cm_alloc_response_msg(work->port, work->mad_recv_wc, &msg))
goto unlock;
@@ -2192,7 +2192,7 @@ static int cm_dreq_handler(struct cm_work *work)
cm_free_msg(msg);
goto deref;
case IB_CM_DREQ_RCVD:
- atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
+ atomic_long_inc_unchecked(&work->port->counter_group[CM_RECV_DUPLICATES].
counter[CM_DREQ_COUNTER]);
goto unlock;
default:
@@ -2559,7 +2559,7 @@ static int cm_mra_handler(struct cm_work *work)
ib_modify_mad(cm_id_priv->av.port->mad_agent,
cm_id_priv->msg, timeout)) {
if (cm_id_priv->id.lap_state == IB_CM_MRA_LAP_RCVD)
- atomic_long_inc(&work->port->
+ atomic_long_inc_unchecked(&work->port->
counter_group[CM_RECV_DUPLICATES].
counter[CM_MRA_COUNTER]);
goto out;
@@ -2568,7 +2568,7 @@ static int cm_mra_handler(struct cm_work *work)
break;
case IB_CM_MRA_REQ_RCVD:
case IB_CM_MRA_REP_RCVD:
- atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
+ atomic_long_inc_unchecked(&work->port->counter_group[CM_RECV_DUPLICATES].
counter[CM_MRA_COUNTER]);
/* fall through */
default:
@@ -2730,7 +2730,7 @@ static int cm_lap_handler(struct cm_work *work)
case IB_CM_LAP_IDLE:
break;
case IB_CM_MRA_LAP_SENT:
- atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
+ atomic_long_inc_unchecked(&work->port->counter_group[CM_RECV_DUPLICATES].
counter[CM_LAP_COUNTER]);
if (cm_alloc_response_msg(work->port, work->mad_recv_wc, &msg))
goto unlock;
@@ -2746,7 +2746,7 @@ static int cm_lap_handler(struct cm_work *work)
cm_free_msg(msg);
goto deref;
case IB_CM_LAP_RCVD:
- atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
+ atomic_long_inc_unchecked(&work->port->counter_group[CM_RECV_DUPLICATES].
counter[CM_LAP_COUNTER]);
goto unlock;
default:
@@ -3030,7 +3030,7 @@ static int cm_sidr_req_handler(struct cm_work *work)
cur_cm_id_priv = cm_insert_remote_sidr(cm_id_priv);
if (cur_cm_id_priv) {
spin_unlock_irq(&cm.lock);
- atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
+ atomic_long_inc_unchecked(&work->port->counter_group[CM_RECV_DUPLICATES].
counter[CM_SIDR_REQ_COUNTER]);
goto out; /* Duplicate message. */
}
@@ -3245,10 +3245,10 @@ static void cm_send_handler(struct ib_mad_agent *mad_agent,
if (!msg->context[0] && (attr_index != CM_REJ_COUNTER))
msg->retries = 1;
- atomic_long_add(1 + msg->retries,
+ atomic_long_add_unchecked(1 + msg->retries,
&port->counter_group[CM_XMIT].counter[attr_index]);
if (msg->retries)
- atomic_long_add(msg->retries,
+ atomic_long_add_unchecked(msg->retries,
&port->counter_group[CM_XMIT_RETRIES].
counter[attr_index]);
@@ -3474,7 +3474,7 @@ static void cm_recv_handler(struct ib_mad_agent *mad_agent,
}
attr_id = be16_to_cpu(mad_recv_wc->recv_buf.mad->mad_hdr.attr_id);
- atomic_long_inc(&port->counter_group[CM_RECV].
+ atomic_long_inc_unchecked(&port->counter_group[CM_RECV].
counter[attr_id - CM_ATTR_ID_OFFSET]);
work = kmalloc(sizeof *work + sizeof(struct ib_sa_path_rec) * paths,
@@ -3717,7 +3717,7 @@ static ssize_t cm_show_counter(struct kobject *obj, struct attribute *attr,
cm_attr = container_of(attr, struct cm_counter_attribute, attr);
return sprintf(buf, "%ld\n",
- atomic_long_read(&group->counter[cm_attr->index]));
+ atomic_long_read_unchecked(&group->counter[cm_attr->index]));
}
static const struct sysfs_ops cm_counter_ops = {
diff --git a/drivers/infiniband/core/fmr_pool.c b/drivers/infiniband/core/fmr_pool.c
index 9f5ad7c..588cd84 100644
--- a/drivers/infiniband/core/fmr_pool.c
+++ b/drivers/infiniband/core/fmr_pool.c
@@ -98,8 +98,8 @@ struct ib_fmr_pool {
struct task_struct *thread;
- atomic_t req_ser;
- atomic_t flush_ser;
+ atomic_unchecked_t req_ser;
+ atomic_unchecked_t flush_ser;
wait_queue_head_t force_wait;
};
@@ -179,10 +179,10 @@ static int ib_fmr_cleanup_thread(void *pool_ptr)
struct ib_fmr_pool *pool = pool_ptr;
do {
- if (atomic_read(&pool->flush_ser) - atomic_read(&pool->req_ser) < 0) {
+ if (atomic_read_unchecked(&pool->flush_ser) - atomic_read_unchecked(&pool->req_ser) < 0) {
ib_fmr_batch_release(pool);
- atomic_inc(&pool->flush_ser);
+ atomic_inc_unchecked(&pool->flush_ser);
wake_up_interruptible(&pool->force_wait);
if (pool->flush_function)
@@ -190,7 +190,7 @@ static int ib_fmr_cleanup_thread(void *pool_ptr)
}
set_current_state(TASK_INTERRUPTIBLE);
- if (atomic_read(&pool->flush_ser) - atomic_read(&pool->req_ser) >= 0 &&
+ if (atomic_read_unchecked(&pool->flush_ser) - atomic_read_unchecked(&pool->req_ser) >= 0 &&
!kthread_should_stop())
schedule();
__set_current_state(TASK_RUNNING);
@@ -282,8 +282,8 @@ struct ib_fmr_pool *ib_create_fmr_pool(struct ib_pd *pd,
pool->dirty_watermark = params->dirty_watermark;
pool->dirty_len = 0;
spin_lock_init(&pool->pool_lock);
- atomic_set(&pool->req_ser, 0);
- atomic_set(&pool->flush_ser, 0);
+ atomic_set_unchecked(&pool->req_ser, 0);
+ atomic_set_unchecked(&pool->flush_ser, 0);
init_waitqueue_head(&pool->force_wait);
pool->thread = kthread_run(ib_fmr_cleanup_thread,
@@ -411,11 +411,11 @@ int ib_flush_fmr_pool(struct ib_fmr_pool *pool)
}
spin_unlock_irq(&pool->pool_lock);
- serial = atomic_inc_return(&pool->req_ser);
+ serial = atomic_inc_return_unchecked(&pool->req_ser);
wake_up_process(pool->thread);
if (wait_event_interruptible(pool->force_wait,
- atomic_read(&pool->flush_ser) - serial >= 0))
+ atomic_read_unchecked(&pool->flush_ser) - serial >= 0))
return -EINTR;
return 0;
@@ -525,7 +525,7 @@ int ib_fmr_pool_unmap(struct ib_pool_fmr *fmr)
} else {
list_add_tail(&fmr->list, &pool->dirty_list);
if (++pool->dirty_len >= pool->dirty_watermark) {
- atomic_inc(&pool->req_ser);
+ atomic_inc_unchecked(&pool->req_ser);
wake_up_process(pool->thread);
}
}
diff --git a/drivers/infiniband/hw/cxgb4/mem.c b/drivers/infiniband/hw/cxgb4/mem.c
index cff815b..75576dd 100644
--- a/drivers/infiniband/hw/cxgb4/mem.c
+++ b/drivers/infiniband/hw/cxgb4/mem.c
@@ -256,7 +256,7 @@ static int write_tpt_entry(struct c4iw_rdev *rdev, u32 reset_tpt_entry,
int err;
struct fw_ri_tpte tpt;
u32 stag_idx;
- static atomic_t key;
+ static atomic_unchecked_t key;
if (c4iw_fatal_error(rdev))
return -EIO;
@@ -277,7 +277,7 @@ static int write_tpt_entry(struct c4iw_rdev *rdev, u32 reset_tpt_entry,
if (rdev->stats.stag.cur > rdev->stats.stag.max)
rdev->stats.stag.max = rdev->stats.stag.cur;
mutex_unlock(&rdev->stats.lock);
- *stag = (stag_idx << 8) | (atomic_inc_return(&key) & 0xff);
+ *stag = (stag_idx << 8) | (atomic_inc_return_unchecked(&key) & 0xff);
}
PDBG("%s stag_state 0x%0x type 0x%0x pdid 0x%0x, stag_idx 0x%x\n",
__func__, stag_state, type, pdid, stag_idx);
diff --git a/drivers/infiniband/hw/ipath/ipath_rc.c b/drivers/infiniband/hw/ipath/ipath_rc.c
index 79b3dbc..96e5fcc 100644
--- a/drivers/infiniband/hw/ipath/ipath_rc.c
+++ b/drivers/infiniband/hw/ipath/ipath_rc.c
@@ -1868,7 +1868,7 @@ void ipath_rc_rcv(struct ipath_ibdev *dev, struct ipath_ib_header *hdr,
struct ib_atomic_eth *ateth;
struct ipath_ack_entry *e;
u64 vaddr;
- atomic64_t *maddr;
+ atomic64_unchecked_t *maddr;
u64 sdata;
u32 rkey;
u8 next;
@@ -1903,11 +1903,11 @@ void ipath_rc_rcv(struct ipath_ibdev *dev, struct ipath_ib_header *hdr,
IB_ACCESS_REMOTE_ATOMIC)))
goto nack_acc_unlck;
/* Perform atomic OP and save result. */
- maddr = (atomic64_t *) qp->r_sge.sge.vaddr;
+ maddr = (atomic64_unchecked_t *) qp->r_sge.sge.vaddr;
sdata = be64_to_cpu(ateth->swap_data);
e = &qp->s_ack_queue[qp->r_head_ack_queue];
e->atomic_data = (opcode == OP(FETCH_ADD)) ?
- (u64) atomic64_add_return(sdata, maddr) - sdata :
+ (u64) atomic64_add_return_unchecked(sdata, maddr) - sdata :
(u64) cmpxchg((u64 *) qp->r_sge.sge.vaddr,
be64_to_cpu(ateth->compare_data),
sdata);
diff --git a/drivers/infiniband/hw/ipath/ipath_ruc.c b/drivers/infiniband/hw/ipath/ipath_ruc.c
index 1f95bba..9530f87 100644
--- a/drivers/infiniband/hw/ipath/ipath_ruc.c
+++ b/drivers/infiniband/hw/ipath/ipath_ruc.c
@@ -266,7 +266,7 @@ static void ipath_ruc_loopback(struct ipath_qp *sqp)
unsigned long flags;
struct ib_wc wc;
u64 sdata;
- atomic64_t *maddr;
+ atomic64_unchecked_t *maddr;
enum ib_wc_status send_status;
/*
@@ -382,11 +382,11 @@ again:
IB_ACCESS_REMOTE_ATOMIC)))
goto acc_err;
/* Perform atomic OP and save result. */
- maddr = (atomic64_t *) qp->r_sge.sge.vaddr;
+ maddr = (atomic64_unchecked_t *) qp->r_sge.sge.vaddr;
sdata = wqe->wr.wr.atomic.compare_add;
*(u64 *) sqp->s_sge.sge.vaddr =
(wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) ?
- (u64) atomic64_add_return(sdata, maddr) - sdata :
+ (u64) atomic64_add_return_unchecked(sdata, maddr) - sdata :
(u64) cmpxchg((u64 *) qp->r_sge.sge.vaddr,
sdata, wqe->wr.wr.atomic.swap);
goto send_comp;
diff --git a/drivers/infiniband/hw/mlx4/mad.c b/drivers/infiniband/hw/mlx4/mad.c
index 68b3dfa..3e0c511 100644
--- a/drivers/infiniband/hw/mlx4/mad.c
+++ b/drivers/infiniband/hw/mlx4/mad.c
@@ -98,7 +98,7 @@ __be64 mlx4_ib_gen_node_guid(void)
__be64 mlx4_ib_get_new_demux_tid(struct mlx4_ib_demux_ctx *ctx)
{
- return cpu_to_be64(atomic_inc_return(&ctx->tid)) |
+ return cpu_to_be64(atomic_inc_return_unchecked(&ctx->tid)) |
cpu_to_be64(0xff00000000000000LL);
}
diff --git a/drivers/infiniband/hw/mlx4/mcg.c b/drivers/infiniband/hw/mlx4/mcg.c
index a0559a8..86a2320 100644
--- a/drivers/infiniband/hw/mlx4/mcg.c
+++ b/drivers/infiniband/hw/mlx4/mcg.c
@@ -1042,7 +1042,7 @@ int mlx4_ib_mcg_port_init(struct mlx4_ib_demux_ctx *ctx)
{
char name[20];
- atomic_set(&ctx->tid, 0);
+ atomic_set_unchecked(&ctx->tid, 0);
sprintf(name, "mlx4_ib_mcg%d", ctx->port);
ctx->mcg_wq = create_singlethread_workqueue(name);
if (!ctx->mcg_wq)
diff --git a/drivers/infiniband/hw/mlx4/mlx4_ib.h b/drivers/infiniband/hw/mlx4/mlx4_ib.h
index 334387f..e640d74 100644
--- a/drivers/infiniband/hw/mlx4/mlx4_ib.h
+++ b/drivers/infiniband/hw/mlx4/mlx4_ib.h
@@ -436,7 +436,7 @@ struct mlx4_ib_demux_ctx {
struct list_head mcg_mgid0_list;
struct workqueue_struct *mcg_wq;
struct mlx4_ib_demux_pv_ctx **tun;
- atomic_t tid;
+ atomic_unchecked_t tid;
int flushing; /* flushing the work queue */
};
diff --git a/drivers/infiniband/hw/nes/nes.c b/drivers/infiniband/hw/nes/nes.c
index 9f9d5c5..3c19aac 100644
--- a/drivers/infiniband/hw/nes/nes.c
+++ b/drivers/infiniband/hw/nes/nes.c
@@ -97,7 +97,7 @@ MODULE_PARM_DESC(limit_maxrdreqsz, "Limit max read request size to 256 Bytes");
LIST_HEAD(nes_adapter_list);
static LIST_HEAD(nes_dev_list);
-atomic_t qps_destroyed;
+atomic_unchecked_t qps_destroyed;
static unsigned int ee_flsh_adapter;
static unsigned int sysfs_nonidx_addr;
@@ -279,7 +279,7 @@ static void nes_cqp_rem_ref_callback(struct nes_device *nesdev, struct nes_cqp_r
struct nes_qp *nesqp = cqp_request->cqp_callback_pointer;
struct nes_adapter *nesadapter = nesdev->nesadapter;
- atomic_inc(&qps_destroyed);
+ atomic_inc_unchecked(&qps_destroyed);
/* Free the control structures */
diff --git a/drivers/infiniband/hw/nes/nes.h b/drivers/infiniband/hw/nes/nes.h
index bd9d132..70d84f4 100644
--- a/drivers/infiniband/hw/nes/nes.h
+++ b/drivers/infiniband/hw/nes/nes.h
@@ -180,17 +180,17 @@ extern unsigned int nes_debug_level;
extern unsigned int wqm_quanta;
extern struct list_head nes_adapter_list;
-extern atomic_t cm_connects;
-extern atomic_t cm_accepts;
-extern atomic_t cm_disconnects;
-extern atomic_t cm_closes;
-extern atomic_t cm_connecteds;
-extern atomic_t cm_connect_reqs;
-extern atomic_t cm_rejects;
-extern atomic_t mod_qp_timouts;
-extern atomic_t qps_created;
-extern atomic_t qps_destroyed;
-extern atomic_t sw_qps_destroyed;
+extern atomic_unchecked_t cm_connects;
+extern atomic_unchecked_t cm_accepts;
+extern atomic_unchecked_t cm_disconnects;
+extern atomic_unchecked_t cm_closes;
+extern atomic_unchecked_t cm_connecteds;
+extern atomic_unchecked_t cm_connect_reqs;
+extern atomic_unchecked_t cm_rejects;
+extern atomic_unchecked_t mod_qp_timouts;
+extern atomic_unchecked_t qps_created;
+extern atomic_unchecked_t qps_destroyed;
+extern atomic_unchecked_t sw_qps_destroyed;
extern u32 mh_detected;
extern u32 mh_pauses_sent;
extern u32 cm_packets_sent;
@@ -199,16 +199,16 @@ extern u32 cm_packets_created;
extern u32 cm_packets_received;
extern u32 cm_packets_dropped;
extern u32 cm_packets_retrans;
-extern atomic_t cm_listens_created;
-extern atomic_t cm_listens_destroyed;
+extern atomic_unchecked_t cm_listens_created;
+extern atomic_unchecked_t cm_listens_destroyed;
extern u32 cm_backlog_drops;
-extern atomic_t cm_loopbacks;
-extern atomic_t cm_nodes_created;
-extern atomic_t cm_nodes_destroyed;
-extern atomic_t cm_accel_dropped_pkts;
-extern atomic_t cm_resets_recvd;
-extern atomic_t pau_qps_created;
-extern atomic_t pau_qps_destroyed;
+extern atomic_unchecked_t cm_loopbacks;
+extern atomic_unchecked_t cm_nodes_created;
+extern atomic_unchecked_t cm_nodes_destroyed;
+extern atomic_unchecked_t cm_accel_dropped_pkts;
+extern atomic_unchecked_t cm_resets_recvd;
+extern atomic_unchecked_t pau_qps_created;
+extern atomic_unchecked_t pau_qps_destroyed;
extern u32 int_mod_timer_init;
extern u32 int_mod_cq_depth_256;
diff --git a/drivers/infiniband/hw/nes/nes_cm.c b/drivers/infiniband/hw/nes/nes_cm.c
index 8a3ad17..e805492 100644
--- a/drivers/infiniband/hw/nes/nes_cm.c
+++ b/drivers/infiniband/hw/nes/nes_cm.c
@@ -69,14 +69,14 @@ u32 cm_packets_dropped;
u32 cm_packets_retrans;
u32 cm_packets_created;
u32 cm_packets_received;
-atomic_t cm_listens_created;
-atomic_t cm_listens_destroyed;
+atomic_unchecked_t cm_listens_created;
+atomic_unchecked_t cm_listens_destroyed;
u32 cm_backlog_drops;
-atomic_t cm_loopbacks;
-atomic_t cm_nodes_created;
-atomic_t cm_nodes_destroyed;
-atomic_t cm_accel_dropped_pkts;
-atomic_t cm_resets_recvd;
+atomic_unchecked_t cm_loopbacks;
+atomic_unchecked_t cm_nodes_created;
+atomic_unchecked_t cm_nodes_destroyed;
+atomic_unchecked_t cm_accel_dropped_pkts;
+atomic_unchecked_t cm_resets_recvd;
static inline int mini_cm_accelerated(struct nes_cm_core *, struct nes_cm_node *);
static struct nes_cm_listener *mini_cm_listen(struct nes_cm_core *, struct nes_vnic *, struct nes_cm_info *);
@@ -150,13 +150,13 @@ static struct nes_cm_ops nes_cm_api = {
static struct nes_cm_core *g_cm_core;
-atomic_t cm_connects;
-atomic_t cm_accepts;
-atomic_t cm_disconnects;
-atomic_t cm_closes;
-atomic_t cm_connecteds;
-atomic_t cm_connect_reqs;
-atomic_t cm_rejects;
+atomic_unchecked_t cm_connects;
+atomic_unchecked_t cm_accepts;
+atomic_unchecked_t cm_disconnects;
+atomic_unchecked_t cm_closes;
+atomic_unchecked_t cm_connecteds;
+atomic_unchecked_t cm_connect_reqs;
+atomic_unchecked_t cm_rejects;
int nes_add_ref_cm_node(struct nes_cm_node *cm_node)
{
@@ -1461,7 +1461,7 @@ static int mini_cm_dec_refcnt_listen(struct nes_cm_core *cm_core,
kfree(listener);
listener = NULL;
ret = 0;
- atomic_inc(&cm_listens_destroyed);
+ atomic_inc_unchecked(&cm_listens_destroyed);
} else {
spin_unlock_irqrestore(&cm_core->listen_list_lock, flags);
}
@@ -1670,7 +1670,7 @@ static struct nes_cm_node *make_cm_node(struct nes_cm_core *cm_core,
cm_node->rem_mac);
add_hte_node(cm_core, cm_node);
- atomic_inc(&cm_nodes_created);
+ atomic_inc_unchecked(&cm_nodes_created);
return cm_node;
}
@@ -1731,7 +1731,7 @@ static int rem_ref_cm_node(struct nes_cm_core *cm_core,
}
atomic_dec(&cm_core->node_cnt);
- atomic_inc(&cm_nodes_destroyed);
+ atomic_inc_unchecked(&cm_nodes_destroyed);
nesqp = cm_node->nesqp;
if (nesqp) {
nesqp->cm_node = NULL;
@@ -1795,7 +1795,7 @@ static int process_options(struct nes_cm_node *cm_node, u8 *optionsloc,
static void drop_packet(struct sk_buff *skb)
{
- atomic_inc(&cm_accel_dropped_pkts);
+ atomic_inc_unchecked(&cm_accel_dropped_pkts);
dev_kfree_skb_any(skb);
}
@@ -1858,7 +1858,7 @@ static void handle_rst_pkt(struct nes_cm_node *cm_node, struct sk_buff *skb,
{
int reset = 0; /* whether to send reset in case of err.. */
- atomic_inc(&cm_resets_recvd);
+ atomic_inc_unchecked(&cm_resets_recvd);
nes_debug(NES_DBG_CM, "Received Reset, cm_node = %p, state = %u."
" refcnt=%d\n", cm_node, cm_node->state,
atomic_read(&cm_node->ref_count));
@@ -2526,7 +2526,7 @@ static struct nes_cm_node *mini_cm_connect(struct nes_cm_core *cm_core,
rem_ref_cm_node(cm_node->cm_core, cm_node);
return NULL;
}
- atomic_inc(&cm_loopbacks);
+ atomic_inc_unchecked(&cm_loopbacks);
loopbackremotenode->loopbackpartner = cm_node;
loopbackremotenode->tcp_cntxt.rcv_wscale =
NES_CM_DEFAULT_RCV_WND_SCALE;
@@ -2807,7 +2807,7 @@ static int mini_cm_recv_pkt(struct nes_cm_core *cm_core,
nes_queue_mgt_skbs(skb, nesvnic, cm_node->nesqp);
else {
rem_ref_cm_node(cm_core, cm_node);
- atomic_inc(&cm_accel_dropped_pkts);
+ atomic_inc_unchecked(&cm_accel_dropped_pkts);
dev_kfree_skb_any(skb);
}
break;
@@ -3118,7 +3118,7 @@ static int nes_cm_disconn_true(struct nes_qp *nesqp)
if ((cm_id) && (cm_id->event_handler)) {
if (issue_disconn) {
- atomic_inc(&cm_disconnects);
+ atomic_inc_unchecked(&cm_disconnects);
cm_event.event = IW_CM_EVENT_DISCONNECT;
cm_event.status = disconn_status;
cm_event.local_addr = cm_id->local_addr;
@@ -3140,7 +3140,7 @@ static int nes_cm_disconn_true(struct nes_qp *nesqp)
}
if (issue_close) {
- atomic_inc(&cm_closes);
+ atomic_inc_unchecked(&cm_closes);
nes_disconnect(nesqp, 1);
cm_id->provider_data = nesqp;
@@ -3278,7 +3278,7 @@ int nes_accept(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
nes_debug(NES_DBG_CM, "QP%u, cm_node=%p, jiffies = %lu listener = %p\n",
nesqp->hwqp.qp_id, cm_node, jiffies, cm_node->listener);
- atomic_inc(&cm_accepts);
+ atomic_inc_unchecked(&cm_accepts);
nes_debug(NES_DBG_CM, "netdev refcnt = %u.\n",
netdev_refcnt_read(nesvnic->netdev));
@@ -3476,7 +3476,7 @@ int nes_reject(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len)
struct nes_cm_core *cm_core;
u8 *start_buff;
- atomic_inc(&cm_rejects);
+ atomic_inc_unchecked(&cm_rejects);
cm_node = (struct nes_cm_node *)cm_id->provider_data;
loopback = cm_node->loopbackpartner;
cm_core = cm_node->cm_core;
@@ -3541,7 +3541,7 @@ int nes_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
ntohs(raddr->sin_port), ntohl(laddr->sin_addr.s_addr),
ntohs(laddr->sin_port));
- atomic_inc(&cm_connects);
+ atomic_inc_unchecked(&cm_connects);
nesqp->active_conn = 1;
/* cache the cm_id in the qp */
@@ -3688,7 +3688,7 @@ int nes_create_listen(struct iw_cm_id *cm_id, int backlog)
g_cm_core->api->stop_listener(g_cm_core, (void *)cm_node);
return err;
}
- atomic_inc(&cm_listens_created);
+ atomic_inc_unchecked(&cm_listens_created);
}
cm_id->add_ref(cm_id);
@@ -3795,7 +3795,7 @@ static void cm_event_connected(struct nes_cm_event *event)
if (nesqp->destroyed)
return;
- atomic_inc(&cm_connecteds);
+ atomic_inc_unchecked(&cm_connecteds);
nes_debug(NES_DBG_CM, "QP%u attempting to connect to 0x%08X:0x%04X on"
" local port 0x%04X. jiffies = %lu.\n",
nesqp->hwqp.qp_id, ntohl(raddr->sin_addr.s_addr),
@@ -3980,7 +3980,7 @@ static void cm_event_reset(struct nes_cm_event *event)
cm_id->add_ref(cm_id);
ret = cm_id->event_handler(cm_id, &cm_event);
- atomic_inc(&cm_closes);
+ atomic_inc_unchecked(&cm_closes);
cm_event.event = IW_CM_EVENT_CLOSE;
cm_event.status = 0;
cm_event.provider_data = cm_id->provider_data;
@@ -4020,7 +4020,7 @@ static void cm_event_mpa_req(struct nes_cm_event *event)
return;
cm_id = cm_node->cm_id;
- atomic_inc(&cm_connect_reqs);
+ atomic_inc_unchecked(&cm_connect_reqs);
nes_debug(NES_DBG_CM, "cm_node = %p - cm_id = %p, jiffies = %lu\n",
cm_node, cm_id, jiffies);
@@ -4069,7 +4069,7 @@ static void cm_event_mpa_reject(struct nes_cm_event *event)
return;
cm_id = cm_node->cm_id;
- atomic_inc(&cm_connect_reqs);
+ atomic_inc_unchecked(&cm_connect_reqs);
nes_debug(NES_DBG_CM, "cm_node = %p - cm_id = %p, jiffies = %lu\n",
cm_node, cm_id, jiffies);
diff --git a/drivers/infiniband/hw/nes/nes_mgt.c b/drivers/infiniband/hw/nes/nes_mgt.c
index 4166452..fc952c3 100644
--- a/drivers/infiniband/hw/nes/nes_mgt.c
+++ b/drivers/infiniband/hw/nes/nes_mgt.c
@@ -40,8 +40,8 @@
#include "nes.h"
#include "nes_mgt.h"
-atomic_t pau_qps_created;
-atomic_t pau_qps_destroyed;
+atomic_unchecked_t pau_qps_created;
+atomic_unchecked_t pau_qps_destroyed;
static void nes_replenish_mgt_rq(struct nes_vnic_mgt *mgtvnic)
{
@@ -621,7 +621,7 @@ void nes_destroy_pau_qp(struct nes_device *nesdev, struct nes_qp *nesqp)
{
struct sk_buff *skb;
unsigned long flags;
- atomic_inc(&pau_qps_destroyed);
+ atomic_inc_unchecked(&pau_qps_destroyed);
/* Free packets that have not yet been forwarded */
/* Lock is acquired by skb_dequeue when removing the skb */
@@ -810,7 +810,7 @@ static void nes_mgt_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *
cq->cq_vbase[head].cqe_words[NES_NIC_CQE_HASH_RCVNXT]);
skb_queue_head_init(&nesqp->pau_list);
spin_lock_init(&nesqp->pau_lock);
- atomic_inc(&pau_qps_created);
+ atomic_inc_unchecked(&pau_qps_created);
nes_change_quad_hash(nesdev, mgtvnic->nesvnic, nesqp);
}
diff --git a/drivers/infiniband/hw/nes/nes_nic.c b/drivers/infiniband/hw/nes/nes_nic.c
index 70acda9..a96de9d 100644
--- a/drivers/infiniband/hw/nes/nes_nic.c
+++ b/drivers/infiniband/hw/nes/nes_nic.c
@@ -1274,39 +1274,39 @@ static void nes_netdev_get_ethtool_stats(struct net_device *netdev,
target_stat_values[++index] = mh_detected;
target_stat_values[++index] = mh_pauses_sent;
target_stat_values[++index] = nesvnic->endnode_ipv4_tcp_retransmits;
- target_stat_values[++index] = atomic_read(&cm_connects);
- target_stat_values[++index] = atomic_read(&cm_accepts);
- target_stat_values[++index] = atomic_read(&cm_disconnects);
- target_stat_values[++index] = atomic_read(&cm_connecteds);
- target_stat_values[++index] = atomic_read(&cm_connect_reqs);
- target_stat_values[++index] = atomic_read(&cm_rejects);
- target_stat_values[++index] = atomic_read(&mod_qp_timouts);
- target_stat_values[++index] = atomic_read(&qps_created);
- target_stat_values[++index] = atomic_read(&sw_qps_destroyed);
- target_stat_values[++index] = atomic_read(&qps_destroyed);
- target_stat_values[++index] = atomic_read(&cm_closes);
+ target_stat_values[++index] = atomic_read_unchecked(&cm_connects);
+ target_stat_values[++index] = atomic_read_unchecked(&cm_accepts);
+ target_stat_values[++index] = atomic_read_unchecked(&cm_disconnects);
+ target_stat_values[++index] = atomic_read_unchecked(&cm_connecteds);
+ target_stat_values[++index] = atomic_read_unchecked(&cm_connect_reqs);
+ target_stat_values[++index] = atomic_read_unchecked(&cm_rejects);
+ target_stat_values[++index] = atomic_read_unchecked(&mod_qp_timouts);
+ target_stat_values[++index] = atomic_read_unchecked(&qps_created);
+ target_stat_values[++index] = atomic_read_unchecked(&sw_qps_destroyed);
+ target_stat_values[++index] = atomic_read_unchecked(&qps_destroyed);
+ target_stat_values[++index] = atomic_read_unchecked(&cm_closes);
target_stat_values[++index] = cm_packets_sent;
target_stat_values[++index] = cm_packets_bounced;
target_stat_values[++index] = cm_packets_created;
target_stat_values[++index] = cm_packets_received;
target_stat_values[++index] = cm_packets_dropped;
target_stat_values[++index] = cm_packets_retrans;
- target_stat_values[++index] = atomic_read(&cm_listens_created);
- target_stat_values[++index] = atomic_read(&cm_listens_destroyed);
+ target_stat_values[++index] = atomic_read_unchecked(&cm_listens_created);
+ target_stat_values[++index] = atomic_read_unchecked(&cm_listens_destroyed);
target_stat_values[++index] = cm_backlog_drops;
- target_stat_values[++index] = atomic_read(&cm_loopbacks);
- target_stat_values[++index] = atomic_read(&cm_nodes_created);
- target_stat_values[++index] = atomic_read(&cm_nodes_destroyed);
- target_stat_values[++index] = atomic_read(&cm_accel_dropped_pkts);
- target_stat_values[++index] = atomic_read(&cm_resets_recvd);
+ target_stat_values[++index] = atomic_read_unchecked(&cm_loopbacks);
+ target_stat_values[++index] = atomic_read_unchecked(&cm_nodes_created);
+ target_stat_values[++index] = atomic_read_unchecked(&cm_nodes_destroyed);
+ target_stat_values[++index] = atomic_read_unchecked(&cm_accel_dropped_pkts);
+ target_stat_values[++index] = atomic_read_unchecked(&cm_resets_recvd);
target_stat_values[++index] = nesadapter->free_4kpbl;
target_stat_values[++index] = nesadapter->free_256pbl;
target_stat_values[++index] = int_mod_timer_init;
target_stat_values[++index] = nesvnic->lro_mgr.stats.aggregated;
target_stat_values[++index] = nesvnic->lro_mgr.stats.flushed;
target_stat_values[++index] = nesvnic->lro_mgr.stats.no_desc;
- target_stat_values[++index] = atomic_read(&pau_qps_created);
- target_stat_values[++index] = atomic_read(&pau_qps_destroyed);
+ target_stat_values[++index] = atomic_read_unchecked(&pau_qps_created);
+ target_stat_values[++index] = atomic_read_unchecked(&pau_qps_destroyed);
}
/**
diff --git a/drivers/infiniband/hw/nes/nes_verbs.c b/drivers/infiniband/hw/nes/nes_verbs.c
index fbc43e5..3672792 100644
--- a/drivers/infiniband/hw/nes/nes_verbs.c
+++ b/drivers/infiniband/hw/nes/nes_verbs.c
@@ -46,9 +46,9 @@
#include <rdma/ib_umem.h>
-atomic_t mod_qp_timouts;
-atomic_t qps_created;
-atomic_t sw_qps_destroyed;
+atomic_unchecked_t mod_qp_timouts;
+atomic_unchecked_t qps_created;
+atomic_unchecked_t sw_qps_destroyed;
static void nes_unregister_ofa_device(struct nes_ib_device *nesibdev);
@@ -1137,7 +1137,7 @@ static struct ib_qp *nes_create_qp(struct ib_pd *ibpd,
if (init_attr->create_flags)
return ERR_PTR(-EINVAL);
- atomic_inc(&qps_created);
+ atomic_inc_unchecked(&qps_created);
switch (init_attr->qp_type) {
case IB_QPT_RC:
if (nes_drv_opt & NES_DRV_OPT_NO_INLINE_DATA) {
@@ -1471,7 +1471,7 @@ static int nes_destroy_qp(struct ib_qp *ibqp)
struct iw_cm_event cm_event;
int ret = 0;
- atomic_inc(&sw_qps_destroyed);
+ atomic_inc_unchecked(&sw_qps_destroyed);
nesqp->destroyed = 1;
/* Blow away the connection if it exists. */
diff --git a/drivers/input/gameport/gameport.c b/drivers/input/gameport/gameport.c
index e853a21..56fc5a8 100644
--- a/drivers/input/gameport/gameport.c
+++ b/drivers/input/gameport/gameport.c
@@ -527,14 +527,14 @@ EXPORT_SYMBOL(gameport_set_phys);
*/
static void gameport_init_port(struct gameport *gameport)
{
- static atomic_t gameport_no = ATOMIC_INIT(-1);
+ static atomic_unchecked_t gameport_no = ATOMIC_INIT(-1);
__module_get(THIS_MODULE);
mutex_init(&gameport->drv_mutex);
device_initialize(&gameport->dev);
dev_set_name(&gameport->dev, "gameport%lu",
- (unsigned long)atomic_inc_return(&gameport_no));
+ (unsigned long)atomic_inc_return_unchecked(&gameport_no));
gameport->dev.bus = &gameport_bus;
gameport->dev.release = gameport_release_port;
if (gameport->parent)
diff --git a/drivers/input/input.c b/drivers/input/input.c
index 78d2499..1f0318e 100644
--- a/drivers/input/input.c
+++ b/drivers/input/input.c
@@ -1775,7 +1775,7 @@ EXPORT_SYMBOL_GPL(input_class);
*/
struct input_dev *input_allocate_device(void)
{
- static atomic_t input_no = ATOMIC_INIT(-1);
+ static atomic_unchecked_t input_no = ATOMIC_INIT(-1);
struct input_dev *dev;
dev = kzalloc(sizeof(struct input_dev), GFP_KERNEL);
@@ -1790,7 +1790,7 @@ struct input_dev *input_allocate_device(void)
INIT_LIST_HEAD(&dev->node);
dev_set_name(&dev->dev, "input%lu",
- (unsigned long)atomic_inc_return(&input_no));
+ (unsigned long)atomic_inc_return_unchecked(&input_no));
__module_get(THIS_MODULE);
}
diff --git a/drivers/input/joystick/xpad.c b/drivers/input/joystick/xpad.c
index f8850f9..9708a2d 100644
--- a/drivers/input/joystick/xpad.c
+++ b/drivers/input/joystick/xpad.c
@@ -959,7 +959,7 @@ static void xpad_led_set(struct led_classdev *led_cdev,
static int xpad_led_probe(struct usb_xpad *xpad)
{
- static atomic_t led_seq = ATOMIC_INIT(-1);
+ static atomic_unchecked_t led_seq = ATOMIC_INIT(-1);
struct xpad_led *led;
struct led_classdev *led_cdev;
int error;
@@ -971,7 +971,7 @@ static int xpad_led_probe(struct usb_xpad *xpad)
if (!led)
return -ENOMEM;
- xpad->led_no = atomic_inc_return(&led_seq);
+ xpad->led_no = atomic_inc_return_unchecked(&led_seq);
snprintf(led->name, sizeof(led->name), "xpad%lu", xpad->led_no);
led->xpad = xpad;
diff --git a/drivers/input/misc/ims-pcu.c b/drivers/input/misc/ims-pcu.c
index ac1fa5f..5f7502c 100644
--- a/drivers/input/misc/ims-pcu.c
+++ b/drivers/input/misc/ims-pcu.c
@@ -1851,7 +1851,7 @@ static int ims_pcu_identify_type(struct ims_pcu *pcu, u8 *device_id)
static int ims_pcu_init_application_mode(struct ims_pcu *pcu)
{
- static atomic_t device_no = ATOMIC_INIT(-1);
+ static atomic_unchecked_t device_no = ATOMIC_INIT(-1);
const struct ims_pcu_device_info *info;
int error;
@@ -1882,7 +1882,7 @@ static int ims_pcu_init_application_mode(struct ims_pcu *pcu)
}
/* Device appears to be operable, complete initialization */
- pcu->device_no = atomic_inc_return(&device_no);
+ pcu->device_no = atomic_inc_return_unchecked(&device_no);
/*
* PCU-B devices, both GEN_1 and GEN_2 do not have OFN sensor
diff --git a/drivers/input/serio/serio.c b/drivers/input/serio/serio.c
index a05a517..323a2fd 100644
--- a/drivers/input/serio/serio.c
+++ b/drivers/input/serio/serio.c
@@ -514,7 +514,7 @@ static void serio_release_port(struct device *dev)
*/
static void serio_init_port(struct serio *serio)
{
- static atomic_t serio_no = ATOMIC_INIT(-1);
+ static atomic_unchecked_t serio_no = ATOMIC_INIT(-1);
__module_get(THIS_MODULE);
@@ -525,7 +525,7 @@ static void serio_init_port(struct serio *serio)
mutex_init(&serio->drv_mutex);
device_initialize(&serio->dev);
dev_set_name(&serio->dev, "serio%lu",
- (unsigned long)atomic_inc_return(&serio_no));
+ (unsigned long)atomic_inc_return_unchecked(&serio_no));
serio->dev.bus = &serio_bus;
serio->dev.release = serio_release_port;
serio->dev.groups = serio_device_attr_groups;
diff --git a/drivers/input/serio/serio_raw.c b/drivers/input/serio/serio_raw.c
index 71ef5d6..93380a9 100644
--- a/drivers/input/serio/serio_raw.c
+++ b/drivers/input/serio/serio_raw.c
@@ -292,7 +292,7 @@ static irqreturn_t serio_raw_interrupt(struct serio *serio, unsigned char data,
static int serio_raw_connect(struct serio *serio, struct serio_driver *drv)
{
- static atomic_t serio_raw_no = ATOMIC_INIT(-1);
+ static atomic_unchecked_t serio_raw_no = ATOMIC_INIT(-1);
struct serio_raw *serio_raw;
int err;
@@ -303,7 +303,7 @@ static int serio_raw_connect(struct serio *serio, struct serio_driver *drv)
}
snprintf(serio_raw->name, sizeof(serio_raw->name),
- "serio_raw%ld", (long)atomic_inc_return(&serio_raw_no));
+ "serio_raw%ld", (long)atomic_inc_return_unchecked(&serio_raw_no));
kref_init(&serio_raw->kref);
INIT_LIST_HEAD(&serio_raw->client_list);
init_waitqueue_head(&serio_raw->wait);
diff --git a/drivers/isdn/capi/capi.c b/drivers/isdn/capi/capi.c
index 6a2df32..dc962f1 100644
--- a/drivers/isdn/capi/capi.c
+++ b/drivers/isdn/capi/capi.c
@@ -81,8 +81,8 @@ struct capiminor {
struct capi20_appl *ap;
u32 ncci;
- atomic_t datahandle;
- atomic_t msgid;
+ atomic_unchecked_t datahandle;
+ atomic_unchecked_t msgid;
struct tty_port port;
int ttyinstop;
@@ -391,7 +391,7 @@ gen_data_b3_resp_for(struct capiminor *mp, struct sk_buff *skb)
capimsg_setu16(s, 2, mp->ap->applid);
capimsg_setu8 (s, 4, CAPI_DATA_B3);
capimsg_setu8 (s, 5, CAPI_RESP);
- capimsg_setu16(s, 6, atomic_inc_return(&mp->msgid));
+ capimsg_setu16(s, 6, atomic_inc_return_unchecked(&mp->msgid));
capimsg_setu32(s, 8, mp->ncci);
capimsg_setu16(s, 12, datahandle);
}
@@ -512,14 +512,14 @@ static void handle_minor_send(struct capiminor *mp)
mp->outbytes -= len;
spin_unlock_bh(&mp->outlock);
- datahandle = atomic_inc_return(&mp->datahandle);
+ datahandle = atomic_inc_return_unchecked(&mp->datahandle);
skb_push(skb, CAPI_DATA_B3_REQ_LEN);
memset(skb->data, 0, CAPI_DATA_B3_REQ_LEN);
capimsg_setu16(skb->data, 0, CAPI_DATA_B3_REQ_LEN);
capimsg_setu16(skb->data, 2, mp->ap->applid);
capimsg_setu8 (skb->data, 4, CAPI_DATA_B3);
capimsg_setu8 (skb->data, 5, CAPI_REQ);
- capimsg_setu16(skb->data, 6, atomic_inc_return(&mp->msgid));
+ capimsg_setu16(skb->data, 6, atomic_inc_return_unchecked(&mp->msgid));
capimsg_setu32(skb->data, 8, mp->ncci); /* NCCI */
capimsg_setu32(skb->data, 12, (u32)(long)skb->data);/* Data32 */
capimsg_setu16(skb->data, 16, len); /* Data length */
diff --git a/drivers/md/dm-raid1.c b/drivers/md/dm-raid1.c
index d83696b..44f22f7 100644
--- a/drivers/md/dm-raid1.c
+++ b/drivers/md/dm-raid1.c
@@ -42,7 +42,7 @@ enum dm_raid1_error {
struct mirror {
struct mirror_set *ms;
- atomic_t error_count;
+ atomic_unchecked_t error_count;
unsigned long error_type;
struct dm_dev *dev;
sector_t offset;
@@ -188,7 +188,7 @@ static struct mirror *get_valid_mirror(struct mirror_set *ms)
struct mirror *m;
for (m = ms->mirror; m < ms->mirror + ms->nr_mirrors; m++)
- if (!atomic_read(&m->error_count))
+ if (!atomic_read_unchecked(&m->error_count))
return m;
return NULL;
@@ -220,7 +220,7 @@ static void fail_mirror(struct mirror *m, enum dm_raid1_error error_type)
* simple way to tell if a device has encountered
* errors.
*/
- atomic_inc(&m->error_count);
+ atomic_inc_unchecked(&m->error_count);
if (test_and_set_bit(error_type, &m->error_type))
return;
@@ -378,7 +378,7 @@ static void reset_ms_flags(struct mirror_set *ms)
ms->leg_failure = 0;
for (m = 0; m < ms->nr_mirrors; m++) {
- atomic_set(&(ms->mirror[m].error_count), 0);
+ atomic_set_unchecked(&(ms->mirror[m].error_count), 0);
ms->mirror[m].error_type = 0;
}
}
@@ -423,7 +423,7 @@ static struct mirror *choose_mirror(struct mirror_set *ms, sector_t sector)
struct mirror *m = get_default_mirror(ms);
do {
- if (likely(!atomic_read(&m->error_count)))
+ if (likely(!atomic_read_unchecked(&m->error_count)))
return m;
if (m-- == ms->mirror)
@@ -437,7 +437,7 @@ static int default_ok(struct mirror *m)
{
struct mirror *default_mirror = get_default_mirror(m->ms);
- return !atomic_read(&default_mirror->error_count);
+ return !atomic_read_unchecked(&default_mirror->error_count);
}
static int mirror_available(struct mirror_set *ms, struct bio *bio)
@@ -574,7 +574,7 @@ static void do_reads(struct mirror_set *ms, struct bio_list *reads)
*/
if (likely(region_in_sync(ms, region, 1)))
m = choose_mirror(ms, bio->bi_iter.bi_sector);
- else if (m && atomic_read(&m->error_count))
+ else if (m && atomic_read_unchecked(&m->error_count))
m = NULL;
if (likely(m))
@@ -956,7 +956,7 @@ static int get_mirror(struct mirror_set *ms, struct dm_target *ti,
}
ms->mirror[mirror].ms = ms;
- atomic_set(&(ms->mirror[mirror].error_count), 0);
+ atomic_set_unchecked(&(ms->mirror[mirror].error_count), 0);
ms->mirror[mirror].error_type = 0;
ms->mirror[mirror].offset = offset;
@@ -1380,7 +1380,7 @@ static void mirror_resume(struct dm_target *ti)
*/
static char device_status_char(struct mirror *m)
{
- if (!atomic_read(&(m->error_count)))
+ if (!atomic_read_unchecked(&(m->error_count)))
return 'A';
return (test_bit(DM_RAID1_FLUSH_ERROR, &(m->error_type))) ? 'F' :
diff --git a/drivers/md/dm-stripe.c b/drivers/md/dm-stripe.c
index a672a15..dc85e99 100644
--- a/drivers/md/dm-stripe.c
+++ b/drivers/md/dm-stripe.c
@@ -21,7 +21,7 @@ struct stripe {
struct dm_dev *dev;
sector_t physical_start;
- atomic_t error_count;
+ atomic_unchecked_t error_count;
};
struct stripe_c {
@@ -188,7 +188,7 @@ static int stripe_ctr(struct dm_target *ti, unsigned int argc, char **argv)
kfree(sc);
return r;
}
- atomic_set(&(sc->stripe[i].error_count), 0);
+ atomic_set_unchecked(&(sc->stripe[i].error_count), 0);
}
ti->private = sc;
@@ -332,7 +332,7 @@ static void stripe_status(struct dm_target *ti, status_type_t type,
DMEMIT("%d ", sc->stripes);
for (i = 0; i < sc->stripes; i++) {
DMEMIT("%s ", sc->stripe[i].dev->name);
- buffer[i] = atomic_read(&(sc->stripe[i].error_count)) ?
+ buffer[i] = atomic_read_unchecked(&(sc->stripe[i].error_count)) ?
'D' : 'A';
}
buffer[i] = '\0';
@@ -377,8 +377,8 @@ static int stripe_end_io(struct dm_target *ti, struct bio *bio, int error)
*/
for (i = 0; i < sc->stripes; i++)
if (!strcmp(sc->stripe[i].dev->name, major_minor)) {
- atomic_inc(&(sc->stripe[i].error_count));
- if (atomic_read(&(sc->stripe[i].error_count)) <
+ atomic_inc_unchecked(&(sc->stripe[i].error_count));
+ if (atomic_read_unchecked(&(sc->stripe[i].error_count)) <
DM_IO_ERROR_THRESHOLD)
schedule_work(&sc->trigger_event);
}
diff --git a/drivers/md/dm.c b/drivers/md/dm.c
index 3e32f4e..01e0a7f 100644
--- a/drivers/md/dm.c
+++ b/drivers/md/dm.c
@@ -194,9 +194,9 @@ struct mapped_device {
/*
* Event handling.
*/
- atomic_t event_nr;
+ atomic_unchecked_t event_nr;
wait_queue_head_t eventq;
- atomic_t uevent_seq;
+ atomic_unchecked_t uevent_seq;
struct list_head uevent_list;
spinlock_t uevent_lock; /* Protect access to uevent_list */
@@ -2339,8 +2339,8 @@ static struct mapped_device *alloc_dev(int minor)
spin_lock_init(&md->deferred_lock);
atomic_set(&md->holders, 1);
atomic_set(&md->open_count, 0);
- atomic_set(&md->event_nr, 0);
- atomic_set(&md->uevent_seq, 0);
+ atomic_set_unchecked(&md->event_nr, 0);
+ atomic_set_unchecked(&md->uevent_seq, 0);
INIT_LIST_HEAD(&md->uevent_list);
INIT_LIST_HEAD(&md->table_devices);
spin_lock_init(&md->uevent_lock);
@@ -2481,7 +2481,7 @@ static void event_callback(void *context)
dm_send_uevents(&uevents, &disk_to_dev(md->disk)->kobj);
- atomic_inc(&md->event_nr);
+ atomic_inc_unchecked(&md->event_nr);
wake_up(&md->eventq);
}
@@ -3479,18 +3479,18 @@ int dm_kobject_uevent(struct mapped_device *md, enum kobject_action action,
uint32_t dm_next_uevent_seq(struct mapped_device *md)
{
- return atomic_add_return(1, &md->uevent_seq);
+ return atomic_add_return_unchecked(1, &md->uevent_seq);
}
uint32_t dm_get_event_nr(struct mapped_device *md)
{
- return atomic_read(&md->event_nr);
+ return atomic_read_unchecked(&md->event_nr);
}
int dm_wait_event(struct mapped_device *md, int event_nr)
{
return wait_event_interruptible(md->eventq,
- (event_nr != atomic_read(&md->event_nr)));
+ (event_nr != atomic_read_unchecked(&md->event_nr)));
}
void dm_uevent_add(struct mapped_device *md, struct list_head *elist)
diff --git a/drivers/md/md.c b/drivers/md/md.c
index 95e7b72..11002b9 100644
--- a/drivers/md/md.c
+++ b/drivers/md/md.c
@@ -197,10 +197,10 @@ EXPORT_SYMBOL_GPL(bio_clone_mddev);
* start build, activate spare
*/
static DECLARE_WAIT_QUEUE_HEAD(md_event_waiters);
-static atomic_t md_event_count;
+static atomic_unchecked_t md_event_count;
void md_new_event(struct mddev *mddev)
{
- atomic_inc(&md_event_count);
+ atomic_inc_unchecked(&md_event_count);
wake_up(&md_event_waiters);
}
EXPORT_SYMBOL_GPL(md_new_event);
@@ -210,7 +210,7 @@ EXPORT_SYMBOL_GPL(md_new_event);
*/
static void md_new_event_inintr(struct mddev *mddev)
{
- atomic_inc(&md_event_count);
+ atomic_inc_unchecked(&md_event_count);
wake_up(&md_event_waiters);
}
@@ -1449,7 +1449,7 @@ static int super_1_load(struct md_rdev *rdev, struct md_rdev *refdev, int minor_
if ((le32_to_cpu(sb->feature_map) & MD_FEATURE_RESHAPE_ACTIVE) &&
(le32_to_cpu(sb->feature_map) & MD_FEATURE_NEW_OFFSET))
rdev->new_data_offset += (s32)le32_to_cpu(sb->new_offset);
- atomic_set(&rdev->corrected_errors, le32_to_cpu(sb->cnt_corrected_read));
+ atomic_set_unchecked(&rdev->corrected_errors, le32_to_cpu(sb->cnt_corrected_read));
rdev->sb_size = le32_to_cpu(sb->max_dev) * 2 + 256;
bmask = queue_logical_block_size(rdev->bdev->bd_disk->queue)-1;
@@ -1700,7 +1700,7 @@ static void super_1_sync(struct mddev *mddev, struct md_rdev *rdev)
else
sb->resync_offset = cpu_to_le64(0);
- sb->cnt_corrected_read = cpu_to_le32(atomic_read(&rdev->corrected_errors));
+ sb->cnt_corrected_read = cpu_to_le32(atomic_read_unchecked(&rdev->corrected_errors));
sb->raid_disks = cpu_to_le32(mddev->raid_disks);
sb->size = cpu_to_le64(mddev->dev_sectors);
@@ -2622,7 +2622,7 @@ __ATTR_PREALLOC(state, S_IRUGO|S_IWUSR, state_show, state_store);
static ssize_t
errors_show(struct md_rdev *rdev, char *page)
{
- return sprintf(page, "%d\n", atomic_read(&rdev->corrected_errors));
+ return sprintf(page, "%d\n", atomic_read_unchecked(&rdev->corrected_errors));
}
static ssize_t
@@ -2634,7 +2634,7 @@ errors_store(struct md_rdev *rdev, const char *buf, size_t len)
rv = kstrtouint(buf, 10, &n);
if (rv < 0)
return rv;
- atomic_set(&rdev->corrected_errors, n);
+ atomic_set_unchecked(&rdev->corrected_errors, n);
return len;
}
static struct rdev_sysfs_entry rdev_errors =
@@ -3071,8 +3071,8 @@ int md_rdev_init(struct md_rdev *rdev)
rdev->sb_loaded = 0;
rdev->bb_page = NULL;
atomic_set(&rdev->nr_pending, 0);
- atomic_set(&rdev->read_errors, 0);
- atomic_set(&rdev->corrected_errors, 0);
+ atomic_set_unchecked(&rdev->read_errors, 0);
+ atomic_set_unchecked(&rdev->corrected_errors, 0);
INIT_LIST_HEAD(&rdev->same_set);
init_waitqueue_head(&rdev->blocked_wait);
@@ -7256,7 +7256,7 @@ static int md_seq_show(struct seq_file *seq, void *v)
spin_unlock(&pers_lock);
seq_printf(seq, "\n");
- seq->poll_event = atomic_read(&md_event_count);
+ seq->poll_event = atomic_read_unchecked(&md_event_count);
return 0;
}
if (v == (void*)2) {
@@ -7359,7 +7359,7 @@ static int md_seq_open(struct inode *inode, struct file *file)
return error;
seq = file->private_data;
- seq->poll_event = atomic_read(&md_event_count);
+ seq->poll_event = atomic_read_unchecked(&md_event_count);
return error;
}
@@ -7376,7 +7376,7 @@ static unsigned int mdstat_poll(struct file *filp, poll_table *wait)
/* always allow read */
mask = POLLIN | POLLRDNORM;
- if (seq->poll_event != atomic_read(&md_event_count))
+ if (seq->poll_event != atomic_read_unchecked(&md_event_count))
mask |= POLLERR | POLLPRI;
return mask;
}
@@ -7472,7 +7472,7 @@ static int is_mddev_idle(struct mddev *mddev, int init)
struct gendisk *disk = rdev->bdev->bd_contains->bd_disk;
curr_events = (int)part_stat_read(&disk->part0, sectors[0]) +
(int)part_stat_read(&disk->part0, sectors[1]) -
- atomic_read(&disk->sync_io);
+ atomic_read_unchecked(&disk->sync_io);
/* sync IO will cause sync_io to increase before the disk_stats
* as sync_io is counted when a request starts, and
* disk_stats is counted when it completes.
diff --git a/drivers/md/md.h b/drivers/md/md.h
index 7da6e9c..f0c1f10 100644
--- a/drivers/md/md.h
+++ b/drivers/md/md.h
@@ -96,13 +96,13 @@ struct md_rdev {
* only maintained for arrays that
* support hot removal
*/
- atomic_t read_errors; /* number of consecutive read errors that
+ atomic_unchecked_t read_errors; /* number of consecutive read errors that
* we have tried to ignore.
*/
struct timespec last_read_error; /* monotonic time since our
* last read error
*/
- atomic_t corrected_errors; /* number of corrected read errors,
+ atomic_unchecked_t corrected_errors; /* number of corrected read errors,
* for reporting to userspace and storing
* in superblock.
*/
@@ -487,7 +487,7 @@ extern void mddev_unlock(struct mddev *mddev);
static inline void md_sync_acct(struct block_device *bdev, unsigned long nr_sectors)
{
- atomic_add(nr_sectors, &bdev->bd_contains->bd_disk->sync_io);
+ atomic_add_unchecked(nr_sectors, &bdev->bd_contains->bd_disk->sync_io);
}
struct md_personality
diff --git a/drivers/md/raid1.c b/drivers/md/raid1.c
index d10d300..6169233 100644
--- a/drivers/md/raid1.c
+++ b/drivers/md/raid1.c
@@ -1937,7 +1937,7 @@ static int fix_sync_read_error(struct r1bio *r1_bio)
if (r1_sync_page_io(rdev, sect, s,
bio->bi_io_vec[idx].bv_page,
READ) != 0)
- atomic_add(s, &rdev->corrected_errors);
+ atomic_add_unchecked(s, &rdev->corrected_errors);
}
sectors -= s;
sect += s;
@@ -2170,7 +2170,7 @@ static void fix_read_error(struct r1conf *conf, int read_disk,
!test_bit(Faulty, &rdev->flags)) {
if (r1_sync_page_io(rdev, sect, s,
conf->tmppage, READ)) {
- atomic_add(s, &rdev->corrected_errors);
+ atomic_add_unchecked(s, &rdev->corrected_errors);
printk(KERN_INFO
"md/raid1:%s: read error corrected "
"(%d sectors at %llu on %s)\n",
diff --git a/drivers/md/raid10.c b/drivers/md/raid10.c
index d4b70d9..3e3bbb8 100644
--- a/drivers/md/raid10.c
+++ b/drivers/md/raid10.c
@@ -1934,7 +1934,7 @@ static void end_sync_read(struct bio *bio, int error)
/* The write handler will notice the lack of
* R10BIO_Uptodate and record any errors etc
*/
- atomic_add(r10_bio->sectors,
+ atomic_add_unchecked(r10_bio->sectors,
&conf->mirrors[d].rdev->corrected_errors);
/* for reconstruct, we always reschedule after a read.
@@ -2281,7 +2281,7 @@ static void check_decay_read_errors(struct mddev *mddev, struct md_rdev *rdev)
{
struct timespec cur_time_mon;
unsigned long hours_since_last;
- unsigned int read_errors = atomic_read(&rdev->read_errors);
+ unsigned int read_errors = atomic_read_unchecked(&rdev->read_errors);
ktime_get_ts(&cur_time_mon);
@@ -2303,9 +2303,9 @@ static void check_decay_read_errors(struct mddev *mddev, struct md_rdev *rdev)
* overflowing the shift of read_errors by hours_since_last.
*/
if (hours_since_last >= 8 * sizeof(read_errors))
- atomic_set(&rdev->read_errors, 0);
+ atomic_set_unchecked(&rdev->read_errors, 0);
else
- atomic_set(&rdev->read_errors, read_errors >> hours_since_last);
+ atomic_set_unchecked(&rdev->read_errors, read_errors >> hours_since_last);
}
static int r10_sync_page_io(struct md_rdev *rdev, sector_t sector,
@@ -2359,8 +2359,8 @@ static void fix_read_error(struct r10conf *conf, struct mddev *mddev, struct r10
return;
check_decay_read_errors(mddev, rdev);
- atomic_inc(&rdev->read_errors);
- if (atomic_read(&rdev->read_errors) > max_read_errors) {
+ atomic_inc_unchecked(&rdev->read_errors);
+ if (atomic_read_unchecked(&rdev->read_errors) > max_read_errors) {
char b[BDEVNAME_SIZE];
bdevname(rdev->bdev, b);
@@ -2368,7 +2368,7 @@ static void fix_read_error(struct r10conf *conf, struct mddev *mddev, struct r10
"md/raid10:%s: %s: Raid device exceeded "
"read_error threshold [cur %d:max %d]\n",
mdname(mddev), b,
- atomic_read(&rdev->read_errors), max_read_errors);
+ atomic_read_unchecked(&rdev->read_errors), max_read_errors);
printk(KERN_NOTICE
"md/raid10:%s: %s: Failing raid device\n",
mdname(mddev), b);
@@ -2523,7 +2523,7 @@ static void fix_read_error(struct r10conf *conf, struct mddev *mddev, struct r10
sect +
choose_data_offset(r10_bio, rdev)),
bdevname(rdev->bdev, b));
- atomic_add(s, &rdev->corrected_errors);
+ atomic_add_unchecked(s, &rdev->corrected_errors);
}
rdev_dec_pending(rdev, mddev);
diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c
index 0d4f7b1..0902720 100644
--- a/drivers/md/raid5.c
+++ b/drivers/md/raid5.c
@@ -2331,21 +2331,21 @@ static void raid5_end_read_request(struct bio * bi, int error)
mdname(conf->mddev), STRIPE_SECTORS,
(unsigned long long)s,
bdevname(rdev->bdev, b));
- atomic_add(STRIPE_SECTORS, &rdev->corrected_errors);
+ atomic_add_unchecked(STRIPE_SECTORS, &rdev->corrected_errors);
clear_bit(R5_ReadError, &sh->dev[i].flags);
clear_bit(R5_ReWrite, &sh->dev[i].flags);
} else if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags))
clear_bit(R5_ReadNoMerge, &sh->dev[i].flags);
- if (atomic_read(&rdev->read_errors))
- atomic_set(&rdev->read_errors, 0);
+ if (atomic_read_unchecked(&rdev->read_errors))
+ atomic_set_unchecked(&rdev->read_errors, 0);
} else {
const char *bdn = bdevname(rdev->bdev, b);
int retry = 0;
int set_bad = 0;
clear_bit(R5_UPTODATE, &sh->dev[i].flags);
- atomic_inc(&rdev->read_errors);
+ atomic_inc_unchecked(&rdev->read_errors);
if (test_bit(R5_ReadRepl, &sh->dev[i].flags))
printk_ratelimited(
KERN_WARNING
@@ -2373,7 +2373,7 @@ static void raid5_end_read_request(struct bio * bi, int error)
mdname(conf->mddev),
(unsigned long long)s,
bdn);
- } else if (atomic_read(&rdev->read_errors)
+ } else if (atomic_read_unchecked(&rdev->read_errors)
> conf->max_nr_stripes)
printk(KERN_WARNING
"md/raid:%s: Too many read errors, failing device %s.\n",
diff --git a/drivers/media/pci/ivtv/ivtv-driver.c b/drivers/media/pci/ivtv/ivtv-driver.c
index 8616fa8..e16eeaf 100644
--- a/drivers/media/pci/ivtv/ivtv-driver.c
+++ b/drivers/media/pci/ivtv/ivtv-driver.c
@@ -83,7 +83,7 @@ static struct pci_device_id ivtv_pci_tbl[] = {
MODULE_DEVICE_TABLE(pci,ivtv_pci_tbl);
/* ivtv instance counter */
-static atomic_t ivtv_instance = ATOMIC_INIT(0);
+static atomic_unchecked_t ivtv_instance = ATOMIC_INIT(0);
/* Parameter declarations */
static int cardtype[IVTV_MAX_CARDS];
diff --git a/drivers/media/pci/solo6x10/solo6x10-p2m.c b/drivers/media/pci/solo6x10/solo6x10-p2m.c
index 8c84846..27b4f83 100644
--- a/drivers/media/pci/solo6x10/solo6x10-p2m.c
+++ b/drivers/media/pci/solo6x10/solo6x10-p2m.c
@@ -73,7 +73,7 @@ int solo_p2m_dma_desc(struct solo_dev *solo_dev,
/* Get next ID. According to Softlogic, 6110 has problems on !=0 P2M */
if (solo_dev->type != SOLO_DEV_6110 && multi_p2m) {
- p2m_id = atomic_inc_return(&solo_dev->p2m_count) % SOLO_NR_P2M;
+ p2m_id = atomic_inc_return_unchecked(&solo_dev->p2m_count) % SOLO_NR_P2M;
if (p2m_id < 0)
p2m_id = -p2m_id;
}
diff --git a/drivers/media/pci/solo6x10/solo6x10.h b/drivers/media/pci/solo6x10/solo6x10.h
index 1ca54b0..7d7cb9a 100644
--- a/drivers/media/pci/solo6x10/solo6x10.h
+++ b/drivers/media/pci/solo6x10/solo6x10.h
@@ -218,7 +218,7 @@ struct solo_dev {
/* P2M DMA Engine */
struct solo_p2m_dev p2m_dev[SOLO_NR_P2M];
- atomic_t p2m_count;
+ atomic_unchecked_t p2m_count;
int p2m_jiffies;
unsigned int p2m_timeouts;
diff --git a/drivers/media/pci/tw68/tw68-core.c b/drivers/media/pci/tw68/tw68-core.c
index c135165..dc69499 100644
--- a/drivers/media/pci/tw68/tw68-core.c
+++ b/drivers/media/pci/tw68/tw68-core.c
@@ -60,7 +60,7 @@ static unsigned int card[] = {[0 ... (TW68_MAXBOARDS - 1)] = UNSET };
module_param_array(card, int, NULL, 0444);
MODULE_PARM_DESC(card, "card type");
-static atomic_t tw68_instance = ATOMIC_INIT(0);
+static atomic_unchecked_t tw68_instance = ATOMIC_INIT(0);
/* ------------------------------------------------------------------ */
diff --git a/drivers/media/radio/radio-maxiradio.c b/drivers/media/radio/radio-maxiradio.c
index 5236035..c622c74 100644
--- a/drivers/media/radio/radio-maxiradio.c
+++ b/drivers/media/radio/radio-maxiradio.c
@@ -61,7 +61,7 @@ MODULE_PARM_DESC(radio_nr, "Radio device number");
/* TEA5757 pin mappings */
static const int clk = 1, data = 2, wren = 4, mo_st = 8, power = 16;
-static atomic_t maxiradio_instance = ATOMIC_INIT(0);
+static atomic_unchecked_t maxiradio_instance = ATOMIC_INIT(0);
#define PCI_VENDOR_ID_GUILLEMOT 0x5046
#define PCI_DEVICE_ID_GUILLEMOT_MAXIRADIO 0x1001
diff --git a/drivers/media/radio/radio-shark.c b/drivers/media/radio/radio-shark.c
index 050b3bb..79f62b9 100644
--- a/drivers/media/radio/radio-shark.c
+++ b/drivers/media/radio/radio-shark.c
@@ -79,7 +79,7 @@ struct shark_device {
u32 last_val;
};
-static atomic_t shark_instance = ATOMIC_INIT(0);
+static atomic_unchecked_t shark_instance = ATOMIC_INIT(0);
static void shark_write_val(struct snd_tea575x *tea, u32 val)
{
diff --git a/drivers/media/radio/radio-shark2.c b/drivers/media/radio/radio-shark2.c
index 8654e0d..0608a64 100644
--- a/drivers/media/radio/radio-shark2.c
+++ b/drivers/media/radio/radio-shark2.c
@@ -74,7 +74,7 @@ struct shark_device {
u8 *transfer_buffer;
};
-static atomic_t shark_instance = ATOMIC_INIT(0);
+static atomic_unchecked_t shark_instance = ATOMIC_INIT(0);
static int shark_write_reg(struct radio_tea5777 *tea, u64 reg)
{
diff --git a/drivers/media/radio/radio-si476x.c b/drivers/media/radio/radio-si476x.c
index 9cbb8cd..2bf2ff3 100644
--- a/drivers/media/radio/radio-si476x.c
+++ b/drivers/media/radio/radio-si476x.c
@@ -1445,7 +1445,7 @@ static int si476x_radio_probe(struct platform_device *pdev)
struct si476x_radio *radio;
struct v4l2_ctrl *ctrl;
- static atomic_t instance = ATOMIC_INIT(0);
+ static atomic_unchecked_t instance = ATOMIC_INIT(0);
radio = devm_kzalloc(&pdev->dev, sizeof(*radio), GFP_KERNEL);
if (!radio)
diff --git a/drivers/media/v4l2-core/v4l2-device.c b/drivers/media/v4l2-core/v4l2-device.c
index 5b0a30b..1974b38 100644
--- a/drivers/media/v4l2-core/v4l2-device.c
+++ b/drivers/media/v4l2-core/v4l2-device.c
@@ -74,9 +74,9 @@ int v4l2_device_put(struct v4l2_device *v4l2_dev)
EXPORT_SYMBOL_GPL(v4l2_device_put);
int v4l2_device_set_name(struct v4l2_device *v4l2_dev, const char *basename,
- atomic_t *instance)
+ atomic_unchecked_t *instance)
{
- int num = atomic_inc_return(instance) - 1;
+ int num = atomic_inc_return_unchecked(instance) - 1;
int len = strlen(basename);
if (basename[len - 1] >= '0' && basename[len - 1] <= '9')
diff --git a/drivers/misc/lis3lv02d/lis3lv02d.c b/drivers/misc/lis3lv02d/lis3lv02d.c
index fb8705f..dc2f679 100644
--- a/drivers/misc/lis3lv02d/lis3lv02d.c
+++ b/drivers/misc/lis3lv02d/lis3lv02d.c
@@ -497,7 +497,7 @@ static irqreturn_t lis302dl_interrupt(int irq, void *data)
* the lid is closed. This leads to interrupts as soon as a little move
* is done.
*/
- atomic_inc(&lis3->count);
+ atomic_inc_unchecked(&lis3->count);
wake_up_interruptible(&lis3->misc_wait);
kill_fasync(&lis3->async_queue, SIGIO, POLL_IN);
@@ -583,7 +583,7 @@ static int lis3lv02d_misc_open(struct inode *inode, struct file *file)
if (lis3->pm_dev)
pm_runtime_get_sync(lis3->pm_dev);
- atomic_set(&lis3->count, 0);
+ atomic_set_unchecked(&lis3->count, 0);
return 0;
}
@@ -615,7 +615,7 @@ static ssize_t lis3lv02d_misc_read(struct file *file, char __user *buf,
add_wait_queue(&lis3->misc_wait, &wait);
while (true) {
set_current_state(TASK_INTERRUPTIBLE);
- data = atomic_xchg(&lis3->count, 0);
+ data = atomic_xchg_unchecked(&lis3->count, 0);
if (data)
break;
@@ -656,7 +656,7 @@ static unsigned int lis3lv02d_misc_poll(struct file *file, poll_table *wait)
struct lis3lv02d, miscdev);
poll_wait(file, &lis3->misc_wait, wait);
- if (atomic_read(&lis3->count))
+ if (atomic_read_unchecked(&lis3->count))
return POLLIN | POLLRDNORM;
return 0;
}
diff --git a/drivers/misc/lis3lv02d/lis3lv02d.h b/drivers/misc/lis3lv02d/lis3lv02d.h
index c439c82..1f20f57 100644
--- a/drivers/misc/lis3lv02d/lis3lv02d.h
+++ b/drivers/misc/lis3lv02d/lis3lv02d.h
@@ -297,7 +297,7 @@ struct lis3lv02d {
struct input_polled_dev *idev; /* input device */
struct platform_device *pdev; /* platform device */
struct regulator_bulk_data regulators[2];
- atomic_t count; /* interrupt count after last read */
+ atomic_unchecked_t count; /* interrupt count after last read */
union axis_conversion ac; /* hw -> logical axis */
int mapped_btns[3];
diff --git a/drivers/misc/sgi-gru/gruhandles.c b/drivers/misc/sgi-gru/gruhandles.c
index 2f30bad..c4c13d0 100644
--- a/drivers/misc/sgi-gru/gruhandles.c
+++ b/drivers/misc/sgi-gru/gruhandles.c
@@ -44,8 +44,8 @@ static void update_mcs_stats(enum mcs_op op, unsigned long clks)
unsigned long nsec;
nsec = CLKS2NSEC(clks);
- atomic_long_inc(&mcs_op_statistics[op].count);
- atomic_long_add(nsec, &mcs_op_statistics[op].total);
+ atomic_long_inc_unchecked(&mcs_op_statistics[op].count);
+ atomic_long_add_unchecked(nsec, &mcs_op_statistics[op].total);
if (mcs_op_statistics[op].max < nsec)
mcs_op_statistics[op].max = nsec;
}
diff --git a/drivers/misc/sgi-gru/gruprocfs.c b/drivers/misc/sgi-gru/gruprocfs.c
index 4f76359..cdfcb2e 100644
--- a/drivers/misc/sgi-gru/gruprocfs.c
+++ b/drivers/misc/sgi-gru/gruprocfs.c
@@ -32,9 +32,9 @@
#define printstat(s, f) printstat_val(s, &gru_stats.f, #f)
-static void printstat_val(struct seq_file *s, atomic_long_t *v, char *id)
+static void printstat_val(struct seq_file *s, atomic_long_unchecked_t *v, char *id)
{
- unsigned long val = atomic_long_read(v);
+ unsigned long val = atomic_long_read_unchecked(v);
seq_printf(s, "%16lu %s\n", val, id);
}
@@ -134,8 +134,8 @@ static int mcs_statistics_show(struct seq_file *s, void *p)
seq_printf(s, "%-20s%12s%12s%12s\n", "#id", "count", "aver-clks", "max-clks");
for (op = 0; op < mcsop_last; op++) {
- count = atomic_long_read(&mcs_op_statistics[op].count);
- total = atomic_long_read(&mcs_op_statistics[op].total);
+ count = atomic_long_read_unchecked(&mcs_op_statistics[op].count);
+ total = atomic_long_read_unchecked(&mcs_op_statistics[op].total);
max = mcs_op_statistics[op].max;
seq_printf(s, "%-20s%12ld%12ld%12ld\n", id[op], count,
count ? total / count : 0, max);
diff --git a/drivers/misc/sgi-gru/grutables.h b/drivers/misc/sgi-gru/grutables.h
index 5c3ce24..4915ccb 100644
--- a/drivers/misc/sgi-gru/grutables.h
+++ b/drivers/misc/sgi-gru/grutables.h
@@ -167,82 +167,82 @@ extern unsigned int gru_max_gids;
* GRU statistics.
*/
struct gru_stats_s {
- atomic_long_t vdata_alloc;
- atomic_long_t vdata_free;
- atomic_long_t gts_alloc;
- atomic_long_t gts_free;
- atomic_long_t gms_alloc;
- atomic_long_t gms_free;
- atomic_long_t gts_double_allocate;
- atomic_long_t assign_context;
- atomic_long_t assign_context_failed;
- atomic_long_t free_context;
- atomic_long_t load_user_context;
- atomic_long_t load_kernel_context;
- atomic_long_t lock_kernel_context;
- atomic_long_t unlock_kernel_context;
- atomic_long_t steal_user_context;
- atomic_long_t steal_kernel_context;
- atomic_long_t steal_context_failed;
- atomic_long_t nopfn;
- atomic_long_t asid_new;
- atomic_long_t asid_next;
- atomic_long_t asid_wrap;
- atomic_long_t asid_reuse;
- atomic_long_t intr;
- atomic_long_t intr_cbr;
- atomic_long_t intr_tfh;
- atomic_long_t intr_spurious;
- atomic_long_t intr_mm_lock_failed;
- atomic_long_t call_os;
- atomic_long_t call_os_wait_queue;
- atomic_long_t user_flush_tlb;
- atomic_long_t user_unload_context;
- atomic_long_t user_exception;
- atomic_long_t set_context_option;
- atomic_long_t check_context_retarget_intr;
- atomic_long_t check_context_unload;
- atomic_long_t tlb_dropin;
- atomic_long_t tlb_preload_page;
- atomic_long_t tlb_dropin_fail_no_asid;
- atomic_long_t tlb_dropin_fail_upm;
- atomic_long_t tlb_dropin_fail_invalid;
- atomic_long_t tlb_dropin_fail_range_active;
- atomic_long_t tlb_dropin_fail_idle;
- atomic_long_t tlb_dropin_fail_fmm;
- atomic_long_t tlb_dropin_fail_no_exception;
- atomic_long_t tfh_stale_on_fault;
- atomic_long_t mmu_invalidate_range;
- atomic_long_t mmu_invalidate_page;
- atomic_long_t flush_tlb;
- atomic_long_t flush_tlb_gru;
- atomic_long_t flush_tlb_gru_tgh;
- atomic_long_t flush_tlb_gru_zero_asid;
-
- atomic_long_t copy_gpa;
- atomic_long_t read_gpa;
-
- atomic_long_t mesq_receive;
- atomic_long_t mesq_receive_none;
- atomic_long_t mesq_send;
- atomic_long_t mesq_send_failed;
- atomic_long_t mesq_noop;
- atomic_long_t mesq_send_unexpected_error;
- atomic_long_t mesq_send_lb_overflow;
- atomic_long_t mesq_send_qlimit_reached;
- atomic_long_t mesq_send_amo_nacked;
- atomic_long_t mesq_send_put_nacked;
- atomic_long_t mesq_page_overflow;
- atomic_long_t mesq_qf_locked;
- atomic_long_t mesq_qf_noop_not_full;
- atomic_long_t mesq_qf_switch_head_failed;
- atomic_long_t mesq_qf_unexpected_error;
- atomic_long_t mesq_noop_unexpected_error;
- atomic_long_t mesq_noop_lb_overflow;
- atomic_long_t mesq_noop_qlimit_reached;
- atomic_long_t mesq_noop_amo_nacked;
- atomic_long_t mesq_noop_put_nacked;
- atomic_long_t mesq_noop_page_overflow;
+ atomic_long_unchecked_t vdata_alloc;
+ atomic_long_unchecked_t vdata_free;
+ atomic_long_unchecked_t gts_alloc;
+ atomic_long_unchecked_t gts_free;
+ atomic_long_unchecked_t gms_alloc;
+ atomic_long_unchecked_t gms_free;
+ atomic_long_unchecked_t gts_double_allocate;
+ atomic_long_unchecked_t assign_context;
+ atomic_long_unchecked_t assign_context_failed;
+ atomic_long_unchecked_t free_context;
+ atomic_long_unchecked_t load_user_context;
+ atomic_long_unchecked_t load_kernel_context;
+ atomic_long_unchecked_t lock_kernel_context;
+ atomic_long_unchecked_t unlock_kernel_context;
+ atomic_long_unchecked_t steal_user_context;
+ atomic_long_unchecked_t steal_kernel_context;
+ atomic_long_unchecked_t steal_context_failed;
+ atomic_long_unchecked_t nopfn;
+ atomic_long_unchecked_t asid_new;
+ atomic_long_unchecked_t asid_next;
+ atomic_long_unchecked_t asid_wrap;
+ atomic_long_unchecked_t asid_reuse;
+ atomic_long_unchecked_t intr;
+ atomic_long_unchecked_t intr_cbr;
+ atomic_long_unchecked_t intr_tfh;
+ atomic_long_unchecked_t intr_spurious;
+ atomic_long_unchecked_t intr_mm_lock_failed;
+ atomic_long_unchecked_t call_os;
+ atomic_long_unchecked_t call_os_wait_queue;
+ atomic_long_unchecked_t user_flush_tlb;
+ atomic_long_unchecked_t user_unload_context;
+ atomic_long_unchecked_t user_exception;
+ atomic_long_unchecked_t set_context_option;
+ atomic_long_unchecked_t check_context_retarget_intr;
+ atomic_long_unchecked_t check_context_unload;
+ atomic_long_unchecked_t tlb_dropin;
+ atomic_long_unchecked_t tlb_preload_page;
+ atomic_long_unchecked_t tlb_dropin_fail_no_asid;
+ atomic_long_unchecked_t tlb_dropin_fail_upm;
+ atomic_long_unchecked_t tlb_dropin_fail_invalid;
+ atomic_long_unchecked_t tlb_dropin_fail_range_active;
+ atomic_long_unchecked_t tlb_dropin_fail_idle;
+ atomic_long_unchecked_t tlb_dropin_fail_fmm;
+ atomic_long_unchecked_t tlb_dropin_fail_no_exception;
+ atomic_long_unchecked_t tfh_stale_on_fault;
+ atomic_long_unchecked_t mmu_invalidate_range;
+ atomic_long_unchecked_t mmu_invalidate_page;
+ atomic_long_unchecked_t flush_tlb;
+ atomic_long_unchecked_t flush_tlb_gru;
+ atomic_long_unchecked_t flush_tlb_gru_tgh;
+ atomic_long_unchecked_t flush_tlb_gru_zero_asid;
+
+ atomic_long_unchecked_t copy_gpa;
+ atomic_long_unchecked_t read_gpa;
+
+ atomic_long_unchecked_t mesq_receive;
+ atomic_long_unchecked_t mesq_receive_none;
+ atomic_long_unchecked_t mesq_send;
+ atomic_long_unchecked_t mesq_send_failed;
+ atomic_long_unchecked_t mesq_noop;
+ atomic_long_unchecked_t mesq_send_unexpected_error;
+ atomic_long_unchecked_t mesq_send_lb_overflow;
+ atomic_long_unchecked_t mesq_send_qlimit_reached;
+ atomic_long_unchecked_t mesq_send_amo_nacked;
+ atomic_long_unchecked_t mesq_send_put_nacked;
+ atomic_long_unchecked_t mesq_page_overflow;
+ atomic_long_unchecked_t mesq_qf_locked;
+ atomic_long_unchecked_t mesq_qf_noop_not_full;
+ atomic_long_unchecked_t mesq_qf_switch_head_failed;
+ atomic_long_unchecked_t mesq_qf_unexpected_error;
+ atomic_long_unchecked_t mesq_noop_unexpected_error;
+ atomic_long_unchecked_t mesq_noop_lb_overflow;
+ atomic_long_unchecked_t mesq_noop_qlimit_reached;
+ atomic_long_unchecked_t mesq_noop_amo_nacked;
+ atomic_long_unchecked_t mesq_noop_put_nacked;
+ atomic_long_unchecked_t mesq_noop_page_overflow;
};
@@ -251,8 +251,8 @@ enum mcs_op {cchop_allocate, cchop_start, cchop_interrupt, cchop_interrupt_sync,
tghop_invalidate, mcsop_last};
struct mcs_op_statistic {
- atomic_long_t count;
- atomic_long_t total;
+ atomic_long_unchecked_t count;
+ atomic_long_unchecked_t total;
unsigned long max;
};
@@ -275,7 +275,7 @@ extern struct mcs_op_statistic mcs_op_statistics[mcsop_last];
#define STAT(id) do { \
if (gru_options & OPT_STATS) \
- atomic_long_inc(&gru_stats.id); \
+ atomic_long_inc_unchecked(&gru_stats.id); \
} while (0)
#ifdef CONFIG_SGI_GRU_DEBUG
diff --git a/drivers/net/hyperv/hyperv_net.h b/drivers/net/hyperv/hyperv_net.h
index dd45440..c5f3cae 100644
--- a/drivers/net/hyperv/hyperv_net.h
+++ b/drivers/net/hyperv/hyperv_net.h
@@ -177,7 +177,7 @@ struct rndis_device {
enum rndis_device_state state;
bool link_state;
bool link_change;
- atomic_t new_req_id;
+ atomic_unchecked_t new_req_id;
spinlock_t request_lock;
struct list_head req_list;
diff --git a/drivers/net/hyperv/rndis_filter.c b/drivers/net/hyperv/rndis_filter.c
index 236aeb7..fd695e2 100644
--- a/drivers/net/hyperv/rndis_filter.c
+++ b/drivers/net/hyperv/rndis_filter.c
@@ -101,7 +101,7 @@ static struct rndis_request *get_rndis_request(struct rndis_device *dev,
* template
*/
set = &rndis_msg->msg.set_req;
- set->req_id = atomic_inc_return(&dev->new_req_id);
+ set->req_id = atomic_inc_return_unchecked(&dev->new_req_id);
/* Add to the request list */
spin_lock_irqsave(&dev->request_lock, flags);
@@ -924,7 +924,7 @@ static void rndis_filter_halt_device(struct rndis_device *dev)
/* Setup the rndis set */
halt = &request->request_msg.msg.halt_req;
- halt->req_id = atomic_inc_return(&dev->new_req_id);
+ halt->req_id = atomic_inc_return_unchecked(&dev->new_req_id);
/* Ignore return since this msg is optional. */
rndis_filter_send_request(dev, request);
--
2.5.0
^ permalink raw reply related [flat|nested] 32+ messages in thread* [kernel-hardening][RFC PATCH v2 09/12] drivers: opt out of PAX_REFCOUNT protection (2 of 2)
2015-12-17 14:57 [kernel-hardening][RFC PATCH v2 00/12] Add PAX_REFCOUNT overflow protection David Windsor
` (7 preceding siblings ...)
2015-12-17 14:57 ` [kernel-hardening][RFC PATCH v2 08/12] drivers: opt out of PAX_REFCOUNT protection (1 of 2) David Windsor
@ 2015-12-17 14:57 ` David Windsor
2015-12-17 14:57 ` [kernel-hardening][RFC PATCH v2 10/12] pax: opt out of PAX_REFCOUNT protection David Windsor
` (3 subsequent siblings)
12 siblings, 0 replies; 32+ messages in thread
From: David Windsor @ 2015-12-17 14:57 UTC (permalink / raw)
To: kernel-hardening; +Cc: David Windsor
Signed-off-by: David Windsor <dave@progbits.org>
---
drivers/net/ipvlan/ipvlan_core.c | 2 +-
drivers/net/macvlan.c | 2 +-
drivers/net/usb/sierra_net.c | 4 +--
drivers/net/wireless/rt2x00/rt2x00.h | 2 +-
drivers/net/wireless/rt2x00/rt2x00queue.c | 4 +--
drivers/oprofile/buffer_sync.c | 8 ++---
drivers/oprofile/event_buffer.c | 2 +-
drivers/oprofile/oprof.c | 2 +-
drivers/oprofile/oprofile_stats.c | 10 +++----
drivers/oprofile/oprofile_stats.h | 10 +++----
drivers/oprofile/oprofilefs.c | 6 ++--
drivers/regulator/core.c | 4 +--
drivers/scsi/fcoe/fcoe_sysfs.c | 12 ++++----
drivers/scsi/hosts.c | 4 +--
drivers/scsi/libfc/fc_exch.c | 50 +++++++++++++++----------------
drivers/scsi/lpfc/lpfc.h | 8 ++---
drivers/scsi/lpfc/lpfc_debugfs.c | 18 +++++------
drivers/scsi/lpfc/lpfc_scsi.c | 10 +++----
drivers/scsi/pmcraid.c | 20 ++++++-------
drivers/scsi/pmcraid.h | 8 ++---
drivers/scsi/qla4xxx/ql4_def.h | 2 +-
drivers/scsi/qla4xxx/ql4_os.c | 6 ++--
drivers/scsi/scsi_lib.c | 8 ++---
drivers/scsi/scsi_sysfs.c | 2 +-
drivers/scsi/scsi_transport_fc.c | 6 ++--
drivers/scsi/scsi_transport_iscsi.c | 6 ++--
drivers/scsi/scsi_transport_srp.c | 6 ++--
drivers/scsi/sd.c | 2 +-
drivers/staging/android/timed_output.c | 6 ++--
drivers/staging/octeon/ethernet-rx.c | 12 ++++----
drivers/staging/octeon/ethernet.c | 8 ++---
drivers/target/sbp/sbp_target.c | 4 +--
drivers/target/target_core_device.c | 2 +-
drivers/target/target_core_transport.c | 2 +-
drivers/tty/hvc/hvsi.c | 12 ++++----
drivers/tty/hvc/hvsi_lib.c | 4 +--
drivers/tty/serial/ioc4_serial.c | 6 ++--
drivers/tty/serial/msm_serial.c | 4 +--
drivers/uio/uio.c | 12 ++++----
drivers/usb/atm/usbatm.c | 24 +++++++--------
drivers/usb/core/devices.c | 6 ++--
drivers/usb/core/hcd.c | 4 +--
drivers/usb/core/sysfs.c | 2 +-
drivers/usb/core/usb.c | 2 +-
drivers/usb/host/ehci-hub.c | 4 +--
drivers/usb/misc/appledisplay.c | 4 +--
drivers/usb/usbip/vhci.h | 2 +-
drivers/usb/usbip/vhci_hcd.c | 6 ++--
drivers/usb/usbip/vhci_rx.c | 2 +-
drivers/usb/wusbcore/wa-hc.h | 4 +--
drivers/usb/wusbcore/wa-xfer.c | 2 +-
drivers/video/fbdev/hyperv_fb.c | 4 +--
drivers/video/fbdev/udlfb.c | 32 ++++++++++----------
include/linux/uio_driver.h | 2 +-
include/linux/usb.h | 2 +-
include/media/v4l2-device.h | 2 +-
include/scsi/scsi_device.h | 6 ++--
include/target/target_core_base.h | 2 +-
58 files changed, 204 insertions(+), 204 deletions(-)
diff --git a/drivers/net/ipvlan/ipvlan_core.c b/drivers/net/ipvlan/ipvlan_core.c
index 207f62e..af3f5e5 100644
--- a/drivers/net/ipvlan/ipvlan_core.c
+++ b/drivers/net/ipvlan/ipvlan_core.c
@@ -466,7 +466,7 @@ static void ipvlan_multicast_enqueue(struct ipvl_port *port,
schedule_work(&port->wq);
} else {
spin_unlock(&port->backlog.lock);
- atomic_long_inc(&skb->dev->rx_dropped);
+ atomic_long_inc_unchecked(&skb->dev->rx_dropped);
kfree_skb(skb);
}
}
diff --git a/drivers/net/macvlan.c b/drivers/net/macvlan.c
index 9f59f17..0f9ae3b2 100644
--- a/drivers/net/macvlan.c
+++ b/drivers/net/macvlan.c
@@ -335,7 +335,7 @@ static void macvlan_broadcast_enqueue(struct macvlan_port *port,
free_nskb:
kfree_skb(nskb);
err:
- atomic_long_inc(&skb->dev->rx_dropped);
+ atomic_long_inc_unchecked(&skb->dev->rx_dropped);
}
static void macvlan_flush_sources(struct macvlan_port *port,
diff --git a/drivers/net/usb/sierra_net.c b/drivers/net/usb/sierra_net.c
index a2515887..6d13233 100644
--- a/drivers/net/usb/sierra_net.c
+++ b/drivers/net/usb/sierra_net.c
@@ -51,7 +51,7 @@ static const char driver_name[] = "sierra_net";
/* atomic counter partially included in MAC address to make sure 2 devices
* do not end up with the same MAC - concept breaks in case of > 255 ifaces
*/
-static atomic_t iface_counter = ATOMIC_INIT(0);
+static atomic_unchecked_t iface_counter = ATOMIC_INIT(0);
/*
* SYNC Timer Delay definition used to set the expiry time
@@ -697,7 +697,7 @@ static int sierra_net_bind(struct usbnet *dev, struct usb_interface *intf)
dev->net->netdev_ops = &sierra_net_device_ops;
/* change MAC addr to include, ifacenum, and to be unique */
- dev->net->dev_addr[ETH_ALEN-2] = atomic_inc_return(&iface_counter);
+ dev->net->dev_addr[ETH_ALEN-2] = atomic_inc_return_unchecked(&iface_counter);
dev->net->dev_addr[ETH_ALEN-1] = ifacenum;
/* we will have to manufacture ethernet headers, prepare template */
diff --git a/drivers/net/wireless/rt2x00/rt2x00.h b/drivers/net/wireless/rt2x00/rt2x00.h
index 9bb398b..b0cc047 100644
--- a/drivers/net/wireless/rt2x00/rt2x00.h
+++ b/drivers/net/wireless/rt2x00/rt2x00.h
@@ -375,7 +375,7 @@ struct rt2x00_intf {
* for hardware which doesn't support hardware
* sequence counting.
*/
- atomic_t seqno;
+ atomic_unchecked_t seqno;
};
static inline struct rt2x00_intf* vif_to_intf(struct ieee80211_vif *vif)
diff --git a/drivers/net/wireless/rt2x00/rt2x00queue.c b/drivers/net/wireless/rt2x00/rt2x00queue.c
index 68b620b..92ecd9e 100644
--- a/drivers/net/wireless/rt2x00/rt2x00queue.c
+++ b/drivers/net/wireless/rt2x00/rt2x00queue.c
@@ -224,9 +224,9 @@ static void rt2x00queue_create_tx_descriptor_seq(struct rt2x00_dev *rt2x00dev,
* sequence counter given by mac80211.
*/
if (test_bit(ENTRY_TXD_FIRST_FRAGMENT, &txdesc->flags))
- seqno = atomic_add_return(0x10, &intf->seqno);
+ seqno = atomic_add_return_unchecked(0x10, &intf->seqno);
else
- seqno = atomic_read(&intf->seqno);
+ seqno = atomic_read_unchecked(&intf->seqno);
hdr->seq_ctrl &= cpu_to_le16(IEEE80211_SCTL_FRAG);
hdr->seq_ctrl |= cpu_to_le16(seqno);
diff --git a/drivers/oprofile/buffer_sync.c b/drivers/oprofile/buffer_sync.c
index 82f7000..d6d0447 100644
--- a/drivers/oprofile/buffer_sync.c
+++ b/drivers/oprofile/buffer_sync.c
@@ -345,7 +345,7 @@ static void add_data(struct op_entry *entry, struct mm_struct *mm)
if (cookie == NO_COOKIE)
offset = pc;
if (cookie == INVALID_COOKIE) {
- atomic_inc(&oprofile_stats.sample_lost_no_mapping);
+ atomic_inc_unchecked(&oprofile_stats.sample_lost_no_mapping);
offset = pc;
}
if (cookie != last_cookie) {
@@ -389,14 +389,14 @@ add_sample(struct mm_struct *mm, struct op_sample *s, int in_kernel)
/* add userspace sample */
if (!mm) {
- atomic_inc(&oprofile_stats.sample_lost_no_mm);
+ atomic_inc_unchecked(&oprofile_stats.sample_lost_no_mm);
return 0;
}
cookie = lookup_dcookie(mm, s->eip, &offset);
if (cookie == INVALID_COOKIE) {
- atomic_inc(&oprofile_stats.sample_lost_no_mapping);
+ atomic_inc_unchecked(&oprofile_stats.sample_lost_no_mapping);
return 0;
}
@@ -554,7 +554,7 @@ void sync_buffer(int cpu)
/* ignore backtraces if failed to add a sample */
if (state == sb_bt_start) {
state = sb_bt_ignore;
- atomic_inc(&oprofile_stats.bt_lost_no_mapping);
+ atomic_inc_unchecked(&oprofile_stats.bt_lost_no_mapping);
}
}
release_mm(mm);
diff --git a/drivers/oprofile/event_buffer.c b/drivers/oprofile/event_buffer.c
index c0cc4e7..44d4e54 100644
--- a/drivers/oprofile/event_buffer.c
+++ b/drivers/oprofile/event_buffer.c
@@ -53,7 +53,7 @@ void add_event_entry(unsigned long value)
}
if (buffer_pos == buffer_size) {
- atomic_inc(&oprofile_stats.event_lost_overflow);
+ atomic_inc_unchecked(&oprofile_stats.event_lost_overflow);
return;
}
diff --git a/drivers/oprofile/oprof.c b/drivers/oprofile/oprof.c
index ed2c3ec..deda85a 100644
--- a/drivers/oprofile/oprof.c
+++ b/drivers/oprofile/oprof.c
@@ -110,7 +110,7 @@ static void switch_worker(struct work_struct *work)
if (oprofile_ops.switch_events())
return;
- atomic_inc(&oprofile_stats.multiplex_counter);
+ atomic_inc_unchecked(&oprofile_stats.multiplex_counter);
start_switch_worker();
}
diff --git a/drivers/oprofile/oprofile_stats.c b/drivers/oprofile/oprofile_stats.c
index 59659ce..6c860a0 100644
--- a/drivers/oprofile/oprofile_stats.c
+++ b/drivers/oprofile/oprofile_stats.c
@@ -30,11 +30,11 @@ void oprofile_reset_stats(void)
cpu_buf->sample_invalid_eip = 0;
}
- atomic_set(&oprofile_stats.sample_lost_no_mm, 0);
- atomic_set(&oprofile_stats.sample_lost_no_mapping, 0);
- atomic_set(&oprofile_stats.event_lost_overflow, 0);
- atomic_set(&oprofile_stats.bt_lost_no_mapping, 0);
- atomic_set(&oprofile_stats.multiplex_counter, 0);
+ atomic_set_unchecked(&oprofile_stats.sample_lost_no_mm, 0);
+ atomic_set_unchecked(&oprofile_stats.sample_lost_no_mapping, 0);
+ atomic_set_unchecked(&oprofile_stats.event_lost_overflow, 0);
+ atomic_set_unchecked(&oprofile_stats.bt_lost_no_mapping, 0);
+ atomic_set_unchecked(&oprofile_stats.multiplex_counter, 0);
}
diff --git a/drivers/oprofile/oprofile_stats.h b/drivers/oprofile/oprofile_stats.h
index 1fc622b..8c48fc3 100644
--- a/drivers/oprofile/oprofile_stats.h
+++ b/drivers/oprofile/oprofile_stats.h
@@ -13,11 +13,11 @@
#include <linux/atomic.h>
struct oprofile_stat_struct {
- atomic_t sample_lost_no_mm;
- atomic_t sample_lost_no_mapping;
- atomic_t bt_lost_no_mapping;
- atomic_t event_lost_overflow;
- atomic_t multiplex_counter;
+ atomic_unchecked_t sample_lost_no_mm;
+ atomic_unchecked_t sample_lost_no_mapping;
+ atomic_unchecked_t bt_lost_no_mapping;
+ atomic_unchecked_t event_lost_overflow;
+ atomic_unchecked_t multiplex_counter;
};
extern struct oprofile_stat_struct oprofile_stats;
diff --git a/drivers/oprofile/oprofilefs.c b/drivers/oprofile/oprofilefs.c
index dd92c5e..dfc04b5 100644
--- a/drivers/oprofile/oprofilefs.c
+++ b/drivers/oprofile/oprofilefs.c
@@ -176,8 +176,8 @@ int oprofilefs_create_ro_ulong(struct dentry *root,
static ssize_t atomic_read_file(struct file *file, char __user *buf, size_t count, loff_t *offset)
{
- atomic_t *val = file->private_data;
- return oprofilefs_ulong_to_user(atomic_read(val), buf, count, offset);
+ atomic_unchecked_t *val = file->private_data;
+ return oprofilefs_ulong_to_user(atomic_read_unchecked(val), buf, count, offset);
}
@@ -189,7 +189,7 @@ static const struct file_operations atomic_ro_fops = {
int oprofilefs_create_ro_atomic(struct dentry *root,
- char const *name, atomic_t *val)
+ char const *name, atomic_unchecked_t *val)
{
return __oprofilefs_create_file(root, name,
&atomic_ro_fops, 0444, val);
diff --git a/drivers/regulator/core.c b/drivers/regulator/core.c
index 5081533..794deb2 100644
--- a/drivers/regulator/core.c
+++ b/drivers/regulator/core.c
@@ -3650,7 +3650,7 @@ regulator_register(const struct regulator_desc *regulator_desc,
const struct regulation_constraints *constraints = NULL;
const struct regulator_init_data *init_data;
struct regulator_config *config = NULL;
- static atomic_t regulator_no = ATOMIC_INIT(-1);
+ static atomic_unchecked_t regulator_no = ATOMIC_INIT(-1);
struct regulator_dev *rdev;
struct device *dev;
int ret, i;
@@ -3733,7 +3733,7 @@ regulator_register(const struct regulator_desc *regulator_desc,
rdev->dev.class = ®ulator_class;
rdev->dev.parent = dev;
dev_set_name(&rdev->dev, "regulator.%lu",
- (unsigned long) atomic_inc_return(®ulator_no));
+ (unsigned long) atomic_inc_return_unchecked(®ulator_no));
ret = device_register(&rdev->dev);
if (ret != 0) {
put_device(&rdev->dev);
diff --git a/drivers/scsi/fcoe/fcoe_sysfs.c b/drivers/scsi/fcoe/fcoe_sysfs.c
index 045c4e1..13de803 100644
--- a/drivers/scsi/fcoe/fcoe_sysfs.c
+++ b/drivers/scsi/fcoe/fcoe_sysfs.c
@@ -33,8 +33,8 @@
*/
#include "libfcoe.h"
-static atomic_t ctlr_num;
-static atomic_t fcf_num;
+static atomic_unchecked_t ctlr_num;
+static atomic_unchecked_t fcf_num;
/*
* fcoe_fcf_dev_loss_tmo: the default number of seconds that fcoe sysfs
@@ -685,7 +685,7 @@ struct fcoe_ctlr_device *fcoe_ctlr_device_add(struct device *parent,
if (!ctlr)
goto out;
- ctlr->id = atomic_inc_return(&ctlr_num) - 1;
+ ctlr->id = atomic_inc_return_unchecked(&ctlr_num) - 1;
ctlr->f = f;
ctlr->mode = FIP_CONN_TYPE_FABRIC;
INIT_LIST_HEAD(&ctlr->fcfs);
@@ -902,7 +902,7 @@ struct fcoe_fcf_device *fcoe_fcf_device_add(struct fcoe_ctlr_device *ctlr,
fcf->dev.parent = &ctlr->dev;
fcf->dev.bus = &fcoe_bus_type;
fcf->dev.type = &fcoe_fcf_device_type;
- fcf->id = atomic_inc_return(&fcf_num) - 1;
+ fcf->id = atomic_inc_return_unchecked(&fcf_num) - 1;
fcf->state = FCOE_FCF_STATE_UNKNOWN;
fcf->dev_loss_tmo = ctlr->fcf_dev_loss_tmo;
@@ -938,8 +938,8 @@ int __init fcoe_sysfs_setup(void)
{
int error;
- atomic_set(&ctlr_num, 0);
- atomic_set(&fcf_num, 0);
+ atomic_set_unchecked(&ctlr_num, 0);
+ atomic_set_unchecked(&fcf_num, 0);
error = bus_register(&fcoe_bus_type);
if (error)
diff --git a/drivers/scsi/hosts.c b/drivers/scsi/hosts.c
index 8bb173e..20236b4 100644
--- a/drivers/scsi/hosts.c
+++ b/drivers/scsi/hosts.c
@@ -42,7 +42,7 @@
#include "scsi_logging.h"
-static atomic_t scsi_host_next_hn = ATOMIC_INIT(0); /* host_no for next new host */
+static atomic_unchecked_t scsi_host_next_hn = ATOMIC_INIT(0); /* host_no for next new host */
static void scsi_host_cls_release(struct device *dev)
@@ -392,7 +392,7 @@ struct Scsi_Host *scsi_host_alloc(struct scsi_host_template *sht, int privsize)
* subtract one because we increment first then return, but we need to
* know what the next host number was before increment
*/
- shost->host_no = atomic_inc_return(&scsi_host_next_hn) - 1;
+ shost->host_no = atomic_inc_return_unchecked(&scsi_host_next_hn) - 1;
shost->dma_channel = 0xff;
/* These three are default values which can be overridden */
diff --git a/drivers/scsi/libfc/fc_exch.c b/drivers/scsi/libfc/fc_exch.c
index 30f9ef0..a1e29ac 100644
--- a/drivers/scsi/libfc/fc_exch.c
+++ b/drivers/scsi/libfc/fc_exch.c
@@ -101,12 +101,12 @@ struct fc_exch_mgr {
u16 pool_max_index;
struct {
- atomic_t no_free_exch;
- atomic_t no_free_exch_xid;
- atomic_t xid_not_found;
- atomic_t xid_busy;
- atomic_t seq_not_found;
- atomic_t non_bls_resp;
+ atomic_unchecked_t no_free_exch;
+ atomic_unchecked_t no_free_exch_xid;
+ atomic_unchecked_t xid_not_found;
+ atomic_unchecked_t xid_busy;
+ atomic_unchecked_t seq_not_found;
+ atomic_unchecked_t non_bls_resp;
} stats;
};
@@ -809,7 +809,7 @@ static struct fc_exch *fc_exch_em_alloc(struct fc_lport *lport,
/* allocate memory for exchange */
ep = mempool_alloc(mp->ep_pool, GFP_ATOMIC);
if (!ep) {
- atomic_inc(&mp->stats.no_free_exch);
+ atomic_inc_unchecked(&mp->stats.no_free_exch);
goto out;
}
memset(ep, 0, sizeof(*ep));
@@ -872,7 +872,7 @@ out:
return ep;
err:
spin_unlock_bh(&pool->lock);
- atomic_inc(&mp->stats.no_free_exch_xid);
+ atomic_inc_unchecked(&mp->stats.no_free_exch_xid);
mempool_free(ep, mp->ep_pool);
return NULL;
}
@@ -1021,7 +1021,7 @@ static enum fc_pf_rjt_reason fc_seq_lookup_recip(struct fc_lport *lport,
xid = ntohs(fh->fh_ox_id); /* we originated exch */
ep = fc_exch_find(mp, xid);
if (!ep) {
- atomic_inc(&mp->stats.xid_not_found);
+ atomic_inc_unchecked(&mp->stats.xid_not_found);
reject = FC_RJT_OX_ID;
goto out;
}
@@ -1051,7 +1051,7 @@ static enum fc_pf_rjt_reason fc_seq_lookup_recip(struct fc_lport *lport,
ep = fc_exch_find(mp, xid);
if ((f_ctl & FC_FC_FIRST_SEQ) && fc_sof_is_init(fr_sof(fp))) {
if (ep) {
- atomic_inc(&mp->stats.xid_busy);
+ atomic_inc_unchecked(&mp->stats.xid_busy);
reject = FC_RJT_RX_ID;
goto rel;
}
@@ -1062,7 +1062,7 @@ static enum fc_pf_rjt_reason fc_seq_lookup_recip(struct fc_lport *lport,
}
xid = ep->xid; /* get our XID */
} else if (!ep) {
- atomic_inc(&mp->stats.xid_not_found);
+ atomic_inc_unchecked(&mp->stats.xid_not_found);
reject = FC_RJT_RX_ID; /* XID not found */
goto out;
}
@@ -1080,7 +1080,7 @@ static enum fc_pf_rjt_reason fc_seq_lookup_recip(struct fc_lport *lport,
} else {
sp = &ep->seq;
if (sp->id != fh->fh_seq_id) {
- atomic_inc(&mp->stats.seq_not_found);
+ atomic_inc_unchecked(&mp->stats.seq_not_found);
if (f_ctl & FC_FC_END_SEQ) {
/*
* Update sequence_id based on incoming last
@@ -1531,22 +1531,22 @@ static void fc_exch_recv_seq_resp(struct fc_exch_mgr *mp, struct fc_frame *fp)
ep = fc_exch_find(mp, ntohs(fh->fh_ox_id));
if (!ep) {
- atomic_inc(&mp->stats.xid_not_found);
+ atomic_inc_unchecked(&mp->stats.xid_not_found);
goto out;
}
if (ep->esb_stat & ESB_ST_COMPLETE) {
- atomic_inc(&mp->stats.xid_not_found);
+ atomic_inc_unchecked(&mp->stats.xid_not_found);
goto rel;
}
if (ep->rxid == FC_XID_UNKNOWN)
ep->rxid = ntohs(fh->fh_rx_id);
if (ep->sid != 0 && ep->sid != ntoh24(fh->fh_d_id)) {
- atomic_inc(&mp->stats.xid_not_found);
+ atomic_inc_unchecked(&mp->stats.xid_not_found);
goto rel;
}
if (ep->did != ntoh24(fh->fh_s_id) &&
ep->did != FC_FID_FLOGI) {
- atomic_inc(&mp->stats.xid_not_found);
+ atomic_inc_unchecked(&mp->stats.xid_not_found);
goto rel;
}
sof = fr_sof(fp);
@@ -1555,7 +1555,7 @@ static void fc_exch_recv_seq_resp(struct fc_exch_mgr *mp, struct fc_frame *fp)
sp->ssb_stat |= SSB_ST_RESP;
sp->id = fh->fh_seq_id;
} else if (sp->id != fh->fh_seq_id) {
- atomic_inc(&mp->stats.seq_not_found);
+ atomic_inc_unchecked(&mp->stats.seq_not_found);
goto rel;
}
@@ -1618,9 +1618,9 @@ static void fc_exch_recv_resp(struct fc_exch_mgr *mp, struct fc_frame *fp)
sp = fc_seq_lookup_orig(mp, fp); /* doesn't hold sequence */
if (!sp)
- atomic_inc(&mp->stats.xid_not_found);
+ atomic_inc_unchecked(&mp->stats.xid_not_found);
else
- atomic_inc(&mp->stats.non_bls_resp);
+ atomic_inc_unchecked(&mp->stats.non_bls_resp);
fc_frame_free(fp);
}
@@ -2261,13 +2261,13 @@ void fc_exch_update_stats(struct fc_lport *lport)
list_for_each_entry(ema, &lport->ema_list, ema_list) {
mp = ema->mp;
- st->fc_no_free_exch += atomic_read(&mp->stats.no_free_exch);
+ st->fc_no_free_exch += atomic_read_unchecked(&mp->stats.no_free_exch);
st->fc_no_free_exch_xid +=
- atomic_read(&mp->stats.no_free_exch_xid);
- st->fc_xid_not_found += atomic_read(&mp->stats.xid_not_found);
- st->fc_xid_busy += atomic_read(&mp->stats.xid_busy);
- st->fc_seq_not_found += atomic_read(&mp->stats.seq_not_found);
- st->fc_non_bls_resp += atomic_read(&mp->stats.non_bls_resp);
+ atomic_read_unchecked(&mp->stats.no_free_exch_xid);
+ st->fc_xid_not_found += atomic_read_unchecked(&mp->stats.xid_not_found);
+ st->fc_xid_busy += atomic_read_unchecked(&mp->stats.xid_busy);
+ st->fc_seq_not_found += atomic_read_unchecked(&mp->stats.seq_not_found);
+ st->fc_non_bls_resp += atomic_read_unchecked(&mp->stats.non_bls_resp);
}
}
EXPORT_SYMBOL(fc_exch_update_stats);
diff --git a/drivers/scsi/lpfc/lpfc.h b/drivers/scsi/lpfc/lpfc.h
index a5a56fa..43499fd 100644
--- a/drivers/scsi/lpfc/lpfc.h
+++ b/drivers/scsi/lpfc/lpfc.h
@@ -435,7 +435,7 @@ struct lpfc_vport {
struct dentry *debug_nodelist;
struct dentry *vport_debugfs_root;
struct lpfc_debugfs_trc *disc_trc;
- atomic_t disc_trc_cnt;
+ atomic_unchecked_t disc_trc_cnt;
#endif
uint8_t stat_data_enabled;
uint8_t stat_data_blocked;
@@ -885,8 +885,8 @@ struct lpfc_hba {
struct timer_list fabric_block_timer;
unsigned long bit_flags;
#define FABRIC_COMANDS_BLOCKED 0
- atomic_t num_rsrc_err;
- atomic_t num_cmd_success;
+ atomic_unchecked_t num_rsrc_err;
+ atomic_unchecked_t num_cmd_success;
unsigned long last_rsrc_error_time;
unsigned long last_ramp_down_time;
#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
@@ -921,7 +921,7 @@ struct lpfc_hba {
struct dentry *debug_slow_ring_trc;
struct lpfc_debugfs_trc *slow_ring_trc;
- atomic_t slow_ring_trc_cnt;
+ atomic_unchecked_t slow_ring_trc_cnt;
/* iDiag debugfs sub-directory */
struct dentry *idiag_root;
struct dentry *idiag_pci_cfg;
diff --git a/drivers/scsi/lpfc/lpfc_debugfs.c b/drivers/scsi/lpfc/lpfc_debugfs.c
index 25aa9b9..d700a65 100644
--- a/drivers/scsi/lpfc/lpfc_debugfs.c
+++ b/drivers/scsi/lpfc/lpfc_debugfs.c
@@ -106,7 +106,7 @@ MODULE_PARM_DESC(lpfc_debugfs_mask_disc_trc,
#include <linux/debugfs.h>
-static atomic_t lpfc_debugfs_seq_trc_cnt = ATOMIC_INIT(0);
+static atomic_unchecked_t lpfc_debugfs_seq_trc_cnt = ATOMIC_INIT(0);
static unsigned long lpfc_debugfs_start_time = 0L;
/* iDiag */
@@ -147,7 +147,7 @@ lpfc_debugfs_disc_trc_data(struct lpfc_vport *vport, char *buf, int size)
lpfc_debugfs_enable = 0;
len = 0;
- index = (atomic_read(&vport->disc_trc_cnt) + 1) &
+ index = (atomic_read_unchecked(&vport->disc_trc_cnt) + 1) &
(lpfc_debugfs_max_disc_trc - 1);
for (i = index; i < lpfc_debugfs_max_disc_trc; i++) {
dtp = vport->disc_trc + i;
@@ -213,7 +213,7 @@ lpfc_debugfs_slow_ring_trc_data(struct lpfc_hba *phba, char *buf, int size)
lpfc_debugfs_enable = 0;
len = 0;
- index = (atomic_read(&phba->slow_ring_trc_cnt) + 1) &
+ index = (atomic_read_unchecked(&phba->slow_ring_trc_cnt) + 1) &
(lpfc_debugfs_max_slow_ring_trc - 1);
for (i = index; i < lpfc_debugfs_max_slow_ring_trc; i++) {
dtp = phba->slow_ring_trc + i;
@@ -646,14 +646,14 @@ lpfc_debugfs_disc_trc(struct lpfc_vport *vport, int mask, char *fmt,
!vport || !vport->disc_trc)
return;
- index = atomic_inc_return(&vport->disc_trc_cnt) &
+ index = atomic_inc_return_unchecked(&vport->disc_trc_cnt) &
(lpfc_debugfs_max_disc_trc - 1);
dtp = vport->disc_trc + index;
dtp->fmt = fmt;
dtp->data1 = data1;
dtp->data2 = data2;
dtp->data3 = data3;
- dtp->seq_cnt = atomic_inc_return(&lpfc_debugfs_seq_trc_cnt);
+ dtp->seq_cnt = atomic_inc_return_unchecked(&lpfc_debugfs_seq_trc_cnt);
dtp->jif = jiffies;
#endif
return;
@@ -684,14 +684,14 @@ lpfc_debugfs_slow_ring_trc(struct lpfc_hba *phba, char *fmt,
!phba || !phba->slow_ring_trc)
return;
- index = atomic_inc_return(&phba->slow_ring_trc_cnt) &
+ index = atomic_inc_return_unchecked(&phba->slow_ring_trc_cnt) &
(lpfc_debugfs_max_slow_ring_trc - 1);
dtp = phba->slow_ring_trc + index;
dtp->fmt = fmt;
dtp->data1 = data1;
dtp->data2 = data2;
dtp->data3 = data3;
- dtp->seq_cnt = atomic_inc_return(&lpfc_debugfs_seq_trc_cnt);
+ dtp->seq_cnt = atomic_inc_return_unchecked(&lpfc_debugfs_seq_trc_cnt);
dtp->jif = jiffies;
#endif
return;
@@ -4268,7 +4268,7 @@ lpfc_debugfs_initialize(struct lpfc_vport *vport)
"slow_ring buffer\n");
goto debug_failed;
}
- atomic_set(&phba->slow_ring_trc_cnt, 0);
+ atomic_set_unchecked(&phba->slow_ring_trc_cnt, 0);
memset(phba->slow_ring_trc, 0,
(sizeof(struct lpfc_debugfs_trc) *
lpfc_debugfs_max_slow_ring_trc));
@@ -4314,7 +4314,7 @@ lpfc_debugfs_initialize(struct lpfc_vport *vport)
"buffer\n");
goto debug_failed;
}
- atomic_set(&vport->disc_trc_cnt, 0);
+ atomic_set_unchecked(&vport->disc_trc_cnt, 0);
snprintf(name, sizeof(name), "discovery_trace");
vport->debug_disc_trc =
diff --git a/drivers/scsi/lpfc/lpfc_scsi.c b/drivers/scsi/lpfc/lpfc_scsi.c
index e5eb40d..056dcd4 100644
--- a/drivers/scsi/lpfc/lpfc_scsi.c
+++ b/drivers/scsi/lpfc/lpfc_scsi.c
@@ -261,7 +261,7 @@ lpfc_rampdown_queue_depth(struct lpfc_hba *phba)
unsigned long expires;
spin_lock_irqsave(&phba->hbalock, flags);
- atomic_inc(&phba->num_rsrc_err);
+ atomic_inc_unchecked(&phba->num_rsrc_err);
phba->last_rsrc_error_time = jiffies;
expires = phba->last_ramp_down_time + QUEUE_RAMP_DOWN_INTERVAL;
@@ -303,8 +303,8 @@ lpfc_ramp_down_queue_handler(struct lpfc_hba *phba)
unsigned long num_rsrc_err, num_cmd_success;
int i;
- num_rsrc_err = atomic_read(&phba->num_rsrc_err);
- num_cmd_success = atomic_read(&phba->num_cmd_success);
+ num_rsrc_err = atomic_read_unchecked(&phba->num_rsrc_err);
+ num_cmd_success = atomic_read_unchecked(&phba->num_cmd_success);
/*
* The error and success command counters are global per
@@ -331,8 +331,8 @@ lpfc_ramp_down_queue_handler(struct lpfc_hba *phba)
}
}
lpfc_destroy_vport_work_array(phba, vports);
- atomic_set(&phba->num_rsrc_err, 0);
- atomic_set(&phba->num_cmd_success, 0);
+ atomic_set_unchecked(&phba->num_rsrc_err, 0);
+ atomic_set_unchecked(&phba->num_cmd_success, 0);
}
/**
diff --git a/drivers/scsi/pmcraid.c b/drivers/scsi/pmcraid.c
index ed31d8c..ab856b3 100644
--- a/drivers/scsi/pmcraid.c
+++ b/drivers/scsi/pmcraid.c
@@ -200,8 +200,8 @@ static int pmcraid_slave_alloc(struct scsi_device *scsi_dev)
res->scsi_dev = scsi_dev;
scsi_dev->hostdata = res;
res->change_detected = 0;
- atomic_set(&res->read_failures, 0);
- atomic_set(&res->write_failures, 0);
+ atomic_set_unchecked(&res->read_failures, 0);
+ atomic_set_unchecked(&res->write_failures, 0);
rc = 0;
}
spin_unlock_irqrestore(&pinstance->resource_lock, lock_flags);
@@ -2640,9 +2640,9 @@ static int pmcraid_error_handler(struct pmcraid_cmd *cmd)
/* If this was a SCSI read/write command keep count of errors */
if (SCSI_CMD_TYPE(scsi_cmd->cmnd[0]) == SCSI_READ_CMD)
- atomic_inc(&res->read_failures);
+ atomic_inc_unchecked(&res->read_failures);
else if (SCSI_CMD_TYPE(scsi_cmd->cmnd[0]) == SCSI_WRITE_CMD)
- atomic_inc(&res->write_failures);
+ atomic_inc_unchecked(&res->write_failures);
if (!RES_IS_GSCSI(res->cfg_entry) &&
masked_ioasc != PMCRAID_IOASC_HW_DEVICE_BUS_STATUS_ERROR) {
@@ -3468,7 +3468,7 @@ static int pmcraid_queuecommand_lck(
* block of scsi_cmd which is re-used (e.g. cancel/abort), which uses
* hrrq_id assigned here in queuecommand
*/
- ioarcb->hrrq_id = atomic_add_return(1, &(pinstance->last_message_id)) %
+ ioarcb->hrrq_id = atomic_add_return_unchecked(1, &(pinstance->last_message_id)) %
pinstance->num_hrrq;
cmd->cmd_done = pmcraid_io_done;
@@ -3782,7 +3782,7 @@ static long pmcraid_ioctl_passthrough(
* block of scsi_cmd which is re-used (e.g. cancel/abort), which uses
* hrrq_id assigned here in queuecommand
*/
- ioarcb->hrrq_id = atomic_add_return(1, &(pinstance->last_message_id)) %
+ ioarcb->hrrq_id = atomic_add_return_unchecked(1, &(pinstance->last_message_id)) %
pinstance->num_hrrq;
if (request_size) {
@@ -4420,7 +4420,7 @@ static void pmcraid_worker_function(struct work_struct *workp)
pinstance = container_of(workp, struct pmcraid_instance, worker_q);
/* add resources only after host is added into system */
- if (!atomic_read(&pinstance->expose_resources))
+ if (!atomic_read_unchecked(&pinstance->expose_resources))
return;
fw_version = be16_to_cpu(pinstance->inq_data->fw_version);
@@ -5237,8 +5237,8 @@ static int pmcraid_init_instance(struct pci_dev *pdev, struct Scsi_Host *host,
init_waitqueue_head(&pinstance->reset_wait_q);
atomic_set(&pinstance->outstanding_cmds, 0);
- atomic_set(&pinstance->last_message_id, 0);
- atomic_set(&pinstance->expose_resources, 0);
+ atomic_set_unchecked(&pinstance->last_message_id, 0);
+ atomic_set_unchecked(&pinstance->expose_resources, 0);
INIT_LIST_HEAD(&pinstance->free_res_q);
INIT_LIST_HEAD(&pinstance->used_res_q);
@@ -5951,7 +5951,7 @@ static int pmcraid_probe(struct pci_dev *pdev,
/* Schedule worker thread to handle CCN and take care of adding and
* removing devices to OS
*/
- atomic_set(&pinstance->expose_resources, 1);
+ atomic_set_unchecked(&pinstance->expose_resources, 1);
schedule_work(&pinstance->worker_q);
return rc;
diff --git a/drivers/scsi/pmcraid.h b/drivers/scsi/pmcraid.h
index e1d150f..6c6df44 100644
--- a/drivers/scsi/pmcraid.h
+++ b/drivers/scsi/pmcraid.h
@@ -748,7 +748,7 @@ struct pmcraid_instance {
struct pmcraid_isr_param hrrq_vector[PMCRAID_NUM_MSIX_VECTORS];
/* Message id as filled in last fired IOARCB, used to identify HRRQ */
- atomic_t last_message_id;
+ atomic_unchecked_t last_message_id;
/* configuration table */
struct pmcraid_config_table *cfg_table;
@@ -777,7 +777,7 @@ struct pmcraid_instance {
atomic_t outstanding_cmds;
/* should add/delete resources to mid-layer now ?*/
- atomic_t expose_resources;
+ atomic_unchecked_t expose_resources;
@@ -813,8 +813,8 @@ struct pmcraid_resource_entry {
struct pmcraid_config_table_entry_ext cfg_entry_ext;
};
struct scsi_device *scsi_dev; /* Link scsi_device structure */
- atomic_t read_failures; /* count of failed READ commands */
- atomic_t write_failures; /* count of failed WRITE commands */
+ atomic_unchecked_t read_failures; /* count of failed READ commands */
+ atomic_unchecked_t write_failures; /* count of failed WRITE commands */
/* To indicate add/delete/modify during CCN */
u8 change_detected;
diff --git a/drivers/scsi/qla4xxx/ql4_def.h b/drivers/scsi/qla4xxx/ql4_def.h
index a7cfc27..151f483 100644
--- a/drivers/scsi/qla4xxx/ql4_def.h
+++ b/drivers/scsi/qla4xxx/ql4_def.h
@@ -306,7 +306,7 @@ struct ddb_entry {
* (4000 only) */
atomic_t relogin_timer; /* Max Time to wait for
* relogin to complete */
- atomic_t relogin_retry_count; /* Num of times relogin has been
+ atomic_unchecked_t relogin_retry_count; /* Num of times relogin has been
* retried */
uint32_t default_time2wait; /* Default Min time between
* relogins (+aens) */
diff --git a/drivers/scsi/qla4xxx/ql4_os.c b/drivers/scsi/qla4xxx/ql4_os.c
index 6d25879..3031a9f 100644
--- a/drivers/scsi/qla4xxx/ql4_os.c
+++ b/drivers/scsi/qla4xxx/ql4_os.c
@@ -4491,12 +4491,12 @@ static void qla4xxx_check_relogin_flash_ddb(struct iscsi_cls_session *cls_sess)
*/
if (!iscsi_is_session_online(cls_sess)) {
/* Reset retry relogin timer */
- atomic_inc(&ddb_entry->relogin_retry_count);
+ atomic_inc_unchecked(&ddb_entry->relogin_retry_count);
DEBUG2(ql4_printk(KERN_INFO, ha,
"%s: index[%d] relogin timed out-retrying"
" relogin (%d), retry (%d)\n", __func__,
ddb_entry->fw_ddb_index,
- atomic_read(&ddb_entry->relogin_retry_count),
+ atomic_read_unchecked(&ddb_entry->relogin_retry_count),
ddb_entry->default_time2wait + 4));
set_bit(DPC_RELOGIN_DEVICE, &ha->dpc_flags);
atomic_set(&ddb_entry->retry_relogin_timer,
@@ -6604,7 +6604,7 @@ static void qla4xxx_setup_flash_ddb_entry(struct scsi_qla_host *ha,
atomic_set(&ddb_entry->retry_relogin_timer, INVALID_ENTRY);
atomic_set(&ddb_entry->relogin_timer, 0);
- atomic_set(&ddb_entry->relogin_retry_count, 0);
+ atomic_set_unchecked(&ddb_entry->relogin_retry_count, 0);
def_timeout = le16_to_cpu(ddb_entry->fw_ddb_entry.def_timeout);
ddb_entry->default_relogin_timeout =
(def_timeout > LOGIN_TOV) && (def_timeout < LOGIN_TOV * 10) ?
diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c
index 448ebda..9bd345f 100644
--- a/drivers/scsi/scsi_lib.c
+++ b/drivers/scsi/scsi_lib.c
@@ -1597,7 +1597,7 @@ static void scsi_kill_request(struct request *req, struct request_queue *q)
shost = sdev->host;
scsi_init_cmd_errh(cmd);
cmd->result = DID_NO_CONNECT << 16;
- atomic_inc(&cmd->device->iorequest_cnt);
+ atomic_inc_unchecked(&cmd->device->iorequest_cnt);
/*
* SCSI request completion path will do scsi_device_unbusy(),
@@ -1620,9 +1620,9 @@ static void scsi_softirq_done(struct request *rq)
INIT_LIST_HEAD(&cmd->eh_entry);
- atomic_inc(&cmd->device->iodone_cnt);
+ atomic_inc_unchecked(&cmd->device->iodone_cnt);
if (cmd->result)
- atomic_inc(&cmd->device->ioerr_cnt);
+ atomic_inc_unchecked(&cmd->device->ioerr_cnt);
disposition = scsi_decide_disposition(cmd);
if (disposition != SUCCESS &&
@@ -1663,7 +1663,7 @@ static int scsi_dispatch_cmd(struct scsi_cmnd *cmd)
struct Scsi_Host *host = cmd->device->host;
int rtn = 0;
- atomic_inc(&cmd->device->iorequest_cnt);
+ atomic_inc_unchecked(&cmd->device->iorequest_cnt);
/* check if the device is still usable */
if (unlikely(cmd->device->sdev_state == SDEV_DEL)) {
diff --git a/drivers/scsi/scsi_sysfs.c b/drivers/scsi/scsi_sysfs.c
index 9ad4116..4e736fc 100644
--- a/drivers/scsi/scsi_sysfs.c
+++ b/drivers/scsi/scsi_sysfs.c
@@ -788,7 +788,7 @@ show_iostat_##field(struct device *dev, struct device_attribute *attr, \
char *buf) \
{ \
struct scsi_device *sdev = to_scsi_device(dev); \
- unsigned long long count = atomic_read(&sdev->field); \
+ unsigned long long count = atomic_read_unchecked(&sdev->field); \
return snprintf(buf, 20, "0x%llx\n", count); \
} \
static DEVICE_ATTR(field, S_IRUGO, show_iostat_##field, NULL)
diff --git a/drivers/scsi/scsi_transport_fc.c b/drivers/scsi/scsi_transport_fc.c
index 24eaaf6..0e8cb69 100644
--- a/drivers/scsi/scsi_transport_fc.c
+++ b/drivers/scsi/scsi_transport_fc.c
@@ -502,7 +502,7 @@ static DECLARE_TRANSPORT_CLASS(fc_vport_class,
* Netlink Infrastructure
*/
-static atomic_t fc_event_seq;
+static atomic_unchecked_t fc_event_seq;
/**
* fc_get_event_number - Obtain the next sequential FC event number
@@ -515,7 +515,7 @@ static atomic_t fc_event_seq;
u32
fc_get_event_number(void)
{
- return atomic_add_return(1, &fc_event_seq);
+ return atomic_add_return_unchecked(1, &fc_event_seq);
}
EXPORT_SYMBOL(fc_get_event_number);
@@ -659,7 +659,7 @@ static __init int fc_transport_init(void)
{
int error;
- atomic_set(&fc_event_seq, 0);
+ atomic_set_unchecked(&fc_event_seq, 0);
error = transport_class_register(&fc_host_class);
if (error)
diff --git a/drivers/scsi/scsi_transport_iscsi.c b/drivers/scsi/scsi_transport_iscsi.c
index 55647aa..b647d05 100644
--- a/drivers/scsi/scsi_transport_iscsi.c
+++ b/drivers/scsi/scsi_transport_iscsi.c
@@ -79,7 +79,7 @@ struct iscsi_internal {
struct transport_container session_cont;
};
-static atomic_t iscsi_session_nr; /* sysfs session id for next new session */
+static atomic_unchecked_t iscsi_session_nr; /* sysfs session id for next new session */
static struct workqueue_struct *iscsi_eh_timer_workq;
static DEFINE_IDA(iscsi_sess_ida);
@@ -2073,7 +2073,7 @@ int iscsi_add_session(struct iscsi_cls_session *session, unsigned int target_id)
int err;
ihost = shost->shost_data;
- session->sid = atomic_add_return(1, &iscsi_session_nr);
+ session->sid = atomic_add_return_unchecked(1, &iscsi_session_nr);
if (target_id == ISCSI_MAX_TARGET) {
id = ida_simple_get(&iscsi_sess_ida, 0, 0, GFP_KERNEL);
@@ -4517,7 +4517,7 @@ static __init int iscsi_transport_init(void)
printk(KERN_INFO "Loading iSCSI transport class v%s.\n",
ISCSI_TRANSPORT_VERSION);
- atomic_set(&iscsi_session_nr, 0);
+ atomic_set_unchecked(&iscsi_session_nr, 0);
err = class_register(&iscsi_transport_class);
if (err)
diff --git a/drivers/scsi/scsi_transport_srp.c b/drivers/scsi/scsi_transport_srp.c
index e3cd3ec..00560ec 100644
--- a/drivers/scsi/scsi_transport_srp.c
+++ b/drivers/scsi/scsi_transport_srp.c
@@ -35,7 +35,7 @@
#include "scsi_priv.h"
struct srp_host_attrs {
- atomic_t next_port_id;
+ atomic_unchecked_t next_port_id;
};
#define to_srp_host_attrs(host) ((struct srp_host_attrs *)(host)->shost_data)
@@ -105,7 +105,7 @@ static int srp_host_setup(struct transport_container *tc, struct device *dev,
struct Scsi_Host *shost = dev_to_shost(dev);
struct srp_host_attrs *srp_host = to_srp_host_attrs(shost);
- atomic_set(&srp_host->next_port_id, 0);
+ atomic_set_unchecked(&srp_host->next_port_id, 0);
return 0;
}
@@ -752,7 +752,7 @@ struct srp_rport *srp_rport_add(struct Scsi_Host *shost,
rport_fast_io_fail_timedout);
INIT_DELAYED_WORK(&rport->dev_loss_work, rport_dev_loss_timedout);
- id = atomic_inc_return(&to_srp_host_attrs(shost)->next_port_id);
+ id = atomic_inc_return_unchecked(&to_srp_host_attrs(shost)->next_port_id);
dev_set_name(&rport->dev, "port-%d:%d", shost->host_no, id);
transport_setup_device(&rport->dev);
diff --git a/drivers/scsi/sd.c b/drivers/scsi/sd.c
index a20da8c..d01312b 100644
--- a/drivers/scsi/sd.c
+++ b/drivers/scsi/sd.c
@@ -2973,7 +2973,7 @@ static int sd_probe(struct device *dev)
sdkp->disk = gd;
sdkp->index = index;
atomic_set(&sdkp->openers, 0);
- atomic_set(&sdkp->device->ioerr_cnt, 0);
+ atomic_set_unchecked(&sdkp->device->ioerr_cnt, 0);
if (!sdp->request_queue->rq_timeout) {
if (sdp->type != TYPE_MOD)
diff --git a/drivers/staging/android/timed_output.c b/drivers/staging/android/timed_output.c
index b41429f..2de5373 100644
--- a/drivers/staging/android/timed_output.c
+++ b/drivers/staging/android/timed_output.c
@@ -25,7 +25,7 @@
#include "timed_output.h"
static struct class *timed_output_class;
-static atomic_t device_count;
+static atomic_unchecked_t device_count;
static ssize_t enable_show(struct device *dev, struct device_attribute *attr,
char *buf)
@@ -65,7 +65,7 @@ static int create_timed_output_class(void)
timed_output_class = class_create(THIS_MODULE, "timed_output");
if (IS_ERR(timed_output_class))
return PTR_ERR(timed_output_class);
- atomic_set(&device_count, 0);
+ atomic_set_unchecked(&device_count, 0);
timed_output_class->dev_groups = timed_output_groups;
}
@@ -83,7 +83,7 @@ int timed_output_dev_register(struct timed_output_dev *tdev)
if (ret < 0)
return ret;
- tdev->index = atomic_inc_return(&device_count);
+ tdev->index = atomic_inc_return_unchecked(&device_count);
tdev->dev = device_create(timed_output_class, NULL,
MKDEV(0, tdev->index), NULL, "%s", tdev->name);
if (IS_ERR(tdev->dev))
diff --git a/drivers/staging/octeon/ethernet-rx.c b/drivers/staging/octeon/ethernet-rx.c
index 22853d3..cfa3c49 100644
--- a/drivers/staging/octeon/ethernet-rx.c
+++ b/drivers/staging/octeon/ethernet-rx.c
@@ -335,14 +335,14 @@ static int cvm_oct_napi_poll(struct napi_struct *napi, int budget)
/* Increment RX stats for virtual ports */
if (work->ipprt >= CVMX_PIP_NUM_INPUT_PORTS) {
#ifdef CONFIG_64BIT
- atomic64_add(1,
+ atomic64_add_unchecked(1,
(atomic64_t *)&priv->stats.rx_packets);
- atomic64_add(skb->len,
+ atomic64_add_unchecked(skb->len,
(atomic64_t *)&priv->stats.rx_bytes);
#else
- atomic_add(1,
+ atomic_add_unchecked(1,
(atomic_t *)&priv->stats.rx_packets);
- atomic_add(skb->len,
+ atomic_add_unchecked(skb->len,
(atomic_t *)&priv->stats.rx_bytes);
#endif
}
@@ -354,10 +354,10 @@ static int cvm_oct_napi_poll(struct napi_struct *napi, int budget)
dev->name);
*/
#ifdef CONFIG_64BIT
- atomic64_add(1,
+ atomic64_add_unchecked(1,
(atomic64_t *)&priv->stats.rx_dropped);
#else
- atomic_add(1,
+ atomic_add_unchecked(1,
(atomic_t *)&priv->stats.rx_dropped);
#endif
dev_kfree_skb_irq(skb);
diff --git a/drivers/staging/octeon/ethernet.c b/drivers/staging/octeon/ethernet.c
index f9dba23..7bc0ef3 100644
--- a/drivers/staging/octeon/ethernet.c
+++ b/drivers/staging/octeon/ethernet.c
@@ -231,11 +231,11 @@ static struct net_device_stats *cvm_oct_common_get_stats(struct net_device *dev)
* since the RX tasklet also increments it.
*/
#ifdef CONFIG_64BIT
- atomic64_add(rx_status.dropped_packets,
- (atomic64_t *)&priv->stats.rx_dropped);
+ atomic64_add_unchecked(rx_status.dropped_packets,
+ (atomic64_unchecked_t *)&priv->stats.rx_dropped);
#else
- atomic_add(rx_status.dropped_packets,
- (atomic_t *)&priv->stats.rx_dropped);
+ atomic_add_unchecked(rx_status.dropped_packets,
+ (atomic_unchecked_t *)&priv->stats.rx_dropped);
#endif
}
diff --git a/drivers/target/sbp/sbp_target.c b/drivers/target/sbp/sbp_target.c
index 0edf320..49afe95 100644
--- a/drivers/target/sbp/sbp_target.c
+++ b/drivers/target/sbp/sbp_target.c
@@ -60,7 +60,7 @@ static const u32 sbp_unit_directory_template[] = {
#define SESSION_MAINTENANCE_INTERVAL HZ
-static atomic_t login_id = ATOMIC_INIT(0);
+static atomic_unchecked_t login_id = ATOMIC_INIT(0);
static void session_maintenance_work(struct work_struct *);
static int sbp_run_transaction(struct fw_card *, int, int, int, int,
@@ -441,7 +441,7 @@ static void sbp_management_request_login(
login->login_lun = unpacked_lun;
login->status_fifo_addr = sbp2_pointer_to_addr(&req->orb.status_fifo);
login->exclusive = LOGIN_ORB_EXCLUSIVE(be32_to_cpu(req->orb.misc));
- login->login_id = atomic_inc_return(&login_id);
+ login->login_id = atomic_inc_return_unchecked(&login_id);
login->tgt_agt = sbp_target_agent_register(login);
if (IS_ERR(login->tgt_agt)) {
diff --git a/drivers/target/target_core_device.c b/drivers/target/target_core_device.c
index 8f1cd19..ba7a8f1 100644
--- a/drivers/target/target_core_device.c
+++ b/drivers/target/target_core_device.c
@@ -772,7 +772,7 @@ struct se_device *target_alloc_device(struct se_hba *hba, const char *name)
spin_lock_init(&dev->se_tmr_lock);
spin_lock_init(&dev->qf_cmd_lock);
sema_init(&dev->caw_sem, 1);
- atomic_set(&dev->dev_ordered_id, 0);
+ atomic_set_unchecked(&dev->dev_ordered_id, 0);
INIT_LIST_HEAD(&dev->t10_wwn.t10_vpd_list);
spin_lock_init(&dev->t10_wwn.t10_vpd_lock);
INIT_LIST_HEAD(&dev->t10_pr.registration_list);
diff --git a/drivers/target/target_core_transport.c b/drivers/target/target_core_transport.c
index ce8574b..98d6199 100644
--- a/drivers/target/target_core_transport.c
+++ b/drivers/target/target_core_transport.c
@@ -1181,7 +1181,7 @@ transport_check_alloc_task_attr(struct se_cmd *cmd)
* Used to determine when ORDERED commands should go from
* Dormant to Active status.
*/
- cmd->se_ordered_id = atomic_inc_return(&dev->dev_ordered_id);
+ cmd->se_ordered_id = atomic_inc_return_unchecked(&dev->dev_ordered_id);
pr_debug("Allocated se_ordered_id: %u for Task Attr: 0x%02x on %s\n",
cmd->se_ordered_id, cmd->sam_task_attr,
dev->transport->name);
diff --git a/drivers/tty/hvc/hvsi.c b/drivers/tty/hvc/hvsi.c
index 4190199..48f2920 100644
--- a/drivers/tty/hvc/hvsi.c
+++ b/drivers/tty/hvc/hvsi.c
@@ -85,7 +85,7 @@ struct hvsi_struct {
int n_outbuf;
uint32_t vtermno;
uint32_t virq;
- atomic_t seqno; /* HVSI packet sequence number */
+ atomic_unchecked_t seqno; /* HVSI packet sequence number */
uint16_t mctrl;
uint8_t state; /* HVSI protocol state */
uint8_t flags;
@@ -295,7 +295,7 @@ static int hvsi_version_respond(struct hvsi_struct *hp, uint16_t query_seqno)
packet.hdr.type = VS_QUERY_RESPONSE_PACKET_HEADER;
packet.hdr.len = sizeof(struct hvsi_query_response);
- packet.hdr.seqno = atomic_inc_return(&hp->seqno);
+ packet.hdr.seqno = atomic_inc_return_unchecked(&hp->seqno);
packet.verb = VSV_SEND_VERSION_NUMBER;
packet.u.version = HVSI_VERSION;
packet.query_seqno = query_seqno+1;
@@ -555,7 +555,7 @@ static int hvsi_query(struct hvsi_struct *hp, uint16_t verb)
packet.hdr.type = VS_QUERY_PACKET_HEADER;
packet.hdr.len = sizeof(struct hvsi_query);
- packet.hdr.seqno = atomic_inc_return(&hp->seqno);
+ packet.hdr.seqno = atomic_inc_return_unchecked(&hp->seqno);
packet.verb = verb;
pr_debug("%s: sending %i bytes\n", __func__, packet.hdr.len);
@@ -597,7 +597,7 @@ static int hvsi_set_mctrl(struct hvsi_struct *hp, uint16_t mctrl)
int wrote;
packet.hdr.type = VS_CONTROL_PACKET_HEADER,
- packet.hdr.seqno = atomic_inc_return(&hp->seqno);
+ packet.hdr.seqno = atomic_inc_return_unchecked(&hp->seqno);
packet.hdr.len = sizeof(struct hvsi_control);
packet.verb = VSV_SET_MODEM_CTL;
packet.mask = HVSI_TSDTR;
@@ -680,7 +680,7 @@ static int hvsi_put_chars(struct hvsi_struct *hp, const char *buf, int count)
BUG_ON(count > HVSI_MAX_OUTGOING_DATA);
packet.hdr.type = VS_DATA_PACKET_HEADER;
- packet.hdr.seqno = atomic_inc_return(&hp->seqno);
+ packet.hdr.seqno = atomic_inc_return_unchecked(&hp->seqno);
packet.hdr.len = count + sizeof(struct hvsi_header);
memcpy(&packet.data, buf, count);
@@ -697,7 +697,7 @@ static void hvsi_close_protocol(struct hvsi_struct *hp)
struct hvsi_control packet __ALIGNED__;
packet.hdr.type = VS_CONTROL_PACKET_HEADER;
- packet.hdr.seqno = atomic_inc_return(&hp->seqno);
+ packet.hdr.seqno = atomic_inc_return_unchecked(&hp->seqno);
packet.hdr.len = 6;
packet.verb = VSV_CLOSE_PROTOCOL;
diff --git a/drivers/tty/hvc/hvsi_lib.c b/drivers/tty/hvc/hvsi_lib.c
index a270f04..7c77b5d 100644
--- a/drivers/tty/hvc/hvsi_lib.c
+++ b/drivers/tty/hvc/hvsi_lib.c
@@ -8,7 +8,7 @@
static int hvsi_send_packet(struct hvsi_priv *pv, struct hvsi_header *packet)
{
- packet->seqno = cpu_to_be16(atomic_inc_return(&pv->seqno));
+ packet->seqno = cpu_to_be16(atomic_inc_return_unchecked(&pv->seqno));
/* Assumes that always succeeds, works in practice */
return pv->put_chars(pv->termno, (char *)packet, packet->len);
@@ -20,7 +20,7 @@ static void hvsi_start_handshake(struct hvsi_priv *pv)
/* Reset state */
pv->established = 0;
- atomic_set(&pv->seqno, 0);
+ atomic_set_unchecked(&pv->seqno, 0);
pr_devel("HVSI@%x: Handshaking started\n", pv->termno);
diff --git a/drivers/tty/serial/ioc4_serial.c b/drivers/tty/serial/ioc4_serial.c
index e5c42fe..f091b02 100644
--- a/drivers/tty/serial/ioc4_serial.c
+++ b/drivers/tty/serial/ioc4_serial.c
@@ -437,7 +437,7 @@ struct ioc4_soft {
} is_intr_info[MAX_IOC4_INTR_ENTS];
/* Number of entries active in the above array */
- atomic_t is_num_intrs;
+ atomic_unchecked_t is_num_intrs;
} is_intr_type[IOC4_NUM_INTR_TYPES];
/* is_ir_lock must be held while
@@ -974,7 +974,7 @@ intr_connect(struct ioc4_soft *soft, int type,
BUG_ON(!((type == IOC4_SIO_INTR_TYPE)
|| (type == IOC4_OTHER_INTR_TYPE)));
- i = atomic_inc_return(&soft-> is_intr_type[type].is_num_intrs) - 1;
+ i = atomic_inc_return_unchecked(&soft-> is_intr_type[type].is_num_intrs) - 1;
BUG_ON(!(i < MAX_IOC4_INTR_ENTS || (printk("i %d\n", i), 0)));
/* Save off the lower level interrupt handler */
@@ -1001,7 +1001,7 @@ static irqreturn_t ioc4_intr(int irq, void *arg)
soft = arg;
for (intr_type = 0; intr_type < IOC4_NUM_INTR_TYPES; intr_type++) {
- num_intrs = (int)atomic_read(
+ num_intrs = (int)atomic_read_unchecked(
&soft->is_intr_type[intr_type].is_num_intrs);
this_mir = this_ir = pending_intrs(soft, intr_type);
diff --git a/drivers/tty/serial/msm_serial.c b/drivers/tty/serial/msm_serial.c
index b73889c..9f74f0a 100644
--- a/drivers/tty/serial/msm_serial.c
+++ b/drivers/tty/serial/msm_serial.c
@@ -1012,7 +1012,7 @@ static struct uart_driver msm_uart_driver = {
.cons = MSM_CONSOLE,
};
-static atomic_t msm_uart_next_id = ATOMIC_INIT(0);
+static atomic_unchecked_t msm_uart_next_id = ATOMIC_INIT(0);
static const struct of_device_id msm_uartdm_table[] = {
{ .compatible = "qcom,msm-uartdm-v1.1", .data = (void *)UARTDM_1P1 },
@@ -1036,7 +1036,7 @@ static int msm_serial_probe(struct platform_device *pdev)
line = pdev->id;
if (line < 0)
- line = atomic_inc_return(&msm_uart_next_id) - 1;
+ line = atomic_inc_return_unchecked(&msm_uart_next_id) - 1;
if (unlikely(line < 0 || line >= UART_NR))
return -ENXIO;
diff --git a/drivers/uio/uio.c b/drivers/uio/uio.c
index 3257d42..16f1d37 100644
--- a/drivers/uio/uio.c
+++ b/drivers/uio/uio.c
@@ -231,7 +231,7 @@ static ssize_t event_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct uio_device *idev = dev_get_drvdata(dev);
- return sprintf(buf, "%u\n", (unsigned int)atomic_read(&idev->event));
+ return sprintf(buf, "%u\n", (unsigned int)atomic_read_unchecked(&idev->event));
}
static DEVICE_ATTR_RO(event);
@@ -393,7 +393,7 @@ void uio_event_notify(struct uio_info *info)
{
struct uio_device *idev = info->uio_dev;
- atomic_inc(&idev->event);
+ atomic_inc_unchecked(&idev->event);
wake_up_interruptible(&idev->wait);
kill_fasync(&idev->async_queue, SIGIO, POLL_IN);
}
@@ -446,7 +446,7 @@ static int uio_open(struct inode *inode, struct file *filep)
}
listener->dev = idev;
- listener->event_count = atomic_read(&idev->event);
+ listener->event_count = atomic_read_unchecked(&idev->event);
filep->private_data = listener;
if (idev->info->open) {
@@ -497,7 +497,7 @@ static unsigned int uio_poll(struct file *filep, poll_table *wait)
return -EIO;
poll_wait(filep, &idev->wait, wait);
- if (listener->event_count != atomic_read(&idev->event))
+ if (listener->event_count != atomic_read_unchecked(&idev->event))
return POLLIN | POLLRDNORM;
return 0;
}
@@ -522,7 +522,7 @@ static ssize_t uio_read(struct file *filep, char __user *buf,
do {
set_current_state(TASK_INTERRUPTIBLE);
- event_count = atomic_read(&idev->event);
+ event_count = atomic_read_unchecked(&idev->event);
if (event_count != listener->event_count) {
if (copy_to_user(buf, &event_count, count))
retval = -EFAULT;
@@ -813,7 +813,7 @@ int __uio_register_device(struct module *owner,
idev->owner = owner;
idev->info = info;
init_waitqueue_head(&idev->wait);
- atomic_set(&idev->event, 0);
+ atomic_set_unchecked(&idev->event, 0);
ret = uio_get_minor(idev);
if (ret)
diff --git a/drivers/usb/atm/usbatm.c b/drivers/usb/atm/usbatm.c
index db322d9..f0f4bc1 100644
--- a/drivers/usb/atm/usbatm.c
+++ b/drivers/usb/atm/usbatm.c
@@ -331,7 +331,7 @@ static void usbatm_extract_one_cell(struct usbatm_data *instance, unsigned char
if (printk_ratelimit())
atm_warn(instance, "%s: OAM not supported (vpi %d, vci %d)!\n",
__func__, vpi, vci);
- atomic_inc(&vcc->stats->rx_err);
+ atomic_inc_unchecked(&vcc->stats->rx_err);
return;
}
@@ -358,7 +358,7 @@ static void usbatm_extract_one_cell(struct usbatm_data *instance, unsigned char
if (length > ATM_MAX_AAL5_PDU) {
atm_rldbg(instance, "%s: bogus length %u (vcc: 0x%p)!\n",
__func__, length, vcc);
- atomic_inc(&vcc->stats->rx_err);
+ atomic_inc_unchecked(&vcc->stats->rx_err);
goto out;
}
@@ -367,14 +367,14 @@ static void usbatm_extract_one_cell(struct usbatm_data *instance, unsigned char
if (sarb->len < pdu_length) {
atm_rldbg(instance, "%s: bogus pdu_length %u (sarb->len: %u, vcc: 0x%p)!\n",
__func__, pdu_length, sarb->len, vcc);
- atomic_inc(&vcc->stats->rx_err);
+ atomic_inc_unchecked(&vcc->stats->rx_err);
goto out;
}
if (crc32_be(~0, skb_tail_pointer(sarb) - pdu_length, pdu_length) != 0xc704dd7b) {
atm_rldbg(instance, "%s: packet failed crc check (vcc: 0x%p)!\n",
__func__, vcc);
- atomic_inc(&vcc->stats->rx_err);
+ atomic_inc_unchecked(&vcc->stats->rx_err);
goto out;
}
@@ -387,7 +387,7 @@ static void usbatm_extract_one_cell(struct usbatm_data *instance, unsigned char
if (printk_ratelimit())
atm_err(instance, "%s: no memory for skb (length: %u)!\n",
__func__, length);
- atomic_inc(&vcc->stats->rx_drop);
+ atomic_inc_unchecked(&vcc->stats->rx_drop);
goto out;
}
@@ -415,7 +415,7 @@ static void usbatm_extract_one_cell(struct usbatm_data *instance, unsigned char
vcc->push(vcc, skb);
- atomic_inc(&vcc->stats->rx);
+ atomic_inc_unchecked(&vcc->stats->rx);
out:
skb_trim(sarb, 0);
}
@@ -613,7 +613,7 @@ static void usbatm_tx_process(unsigned long data)
struct atm_vcc *vcc = UDSL_SKB(skb)->atm.vcc;
usbatm_pop(vcc, skb);
- atomic_inc(&vcc->stats->tx);
+ atomic_inc_unchecked(&vcc->stats->tx);
skb = skb_dequeue(&instance->sndqueue);
}
@@ -757,11 +757,11 @@ static int usbatm_atm_proc_read(struct atm_dev *atm_dev, loff_t *pos, char *page
if (!left--)
return sprintf(page,
"AAL5: tx %d ( %d err ), rx %d ( %d err, %d drop )\n",
- atomic_read(&atm_dev->stats.aal5.tx),
- atomic_read(&atm_dev->stats.aal5.tx_err),
- atomic_read(&atm_dev->stats.aal5.rx),
- atomic_read(&atm_dev->stats.aal5.rx_err),
- atomic_read(&atm_dev->stats.aal5.rx_drop));
+ atomic_read_unchecked(&atm_dev->stats.aal5.tx),
+ atomic_read_unchecked(&atm_dev->stats.aal5.tx_err),
+ atomic_read_unchecked(&atm_dev->stats.aal5.rx),
+ atomic_read_unchecked(&atm_dev->stats.aal5.rx_err),
+ atomic_read_unchecked(&atm_dev->stats.aal5.rx_drop));
if (!left--) {
if (instance->disconnected)
diff --git a/drivers/usb/core/devices.c b/drivers/usb/core/devices.c
index 2a3bbdf..91d72cf 100644
--- a/drivers/usb/core/devices.c
+++ b/drivers/usb/core/devices.c
@@ -126,7 +126,7 @@ static const char format_endpt[] =
* time it gets called.
*/
static struct device_connect_event {
- atomic_t count;
+ atomic_unchecked_t count;
wait_queue_head_t wait;
} device_event = {
.count = ATOMIC_INIT(1),
@@ -164,7 +164,7 @@ static const struct class_info clas_info[] = {
void usbfs_conn_disc_event(void)
{
- atomic_add(2, &device_event.count);
+ atomic_add_unchecked(2, &device_event.count);
wake_up(&device_event.wait);
}
@@ -652,7 +652,7 @@ static unsigned int usb_device_poll(struct file *file,
poll_wait(file, &device_event.wait, wait);
- event_count = atomic_read(&device_event.count);
+ event_count = atomic_read_unchecked(&device_event.count);
if (file->f_version != event_count) {
file->f_version = event_count;
return POLLIN | POLLRDNORM;
diff --git a/drivers/usb/core/hcd.c b/drivers/usb/core/hcd.c
index cbcd092..e783f87 100644
--- a/drivers/usb/core/hcd.c
+++ b/drivers/usb/core/hcd.c
@@ -1554,7 +1554,7 @@ int usb_hcd_submit_urb (struct urb *urb, gfp_t mem_flags)
*/
usb_get_urb(urb);
atomic_inc(&urb->use_count);
- atomic_inc(&urb->dev->urbnum);
+ atomic_inc_unchecked(&urb->dev->urbnum);
usbmon_urb_submit(&hcd->self, urb);
/* NOTE requirements on root-hub callers (usbfs and the hub
@@ -1581,7 +1581,7 @@ int usb_hcd_submit_urb (struct urb *urb, gfp_t mem_flags)
urb->hcpriv = NULL;
INIT_LIST_HEAD(&urb->urb_list);
atomic_dec(&urb->use_count);
- atomic_dec(&urb->dev->urbnum);
+ atomic_dec_unchecked(&urb->dev->urbnum);
if (atomic_read(&urb->reject))
wake_up(&usb_kill_urb_queue);
usb_put_urb(urb);
diff --git a/drivers/usb/core/sysfs.c b/drivers/usb/core/sysfs.c
index d269738..7340cd7 100644
--- a/drivers/usb/core/sysfs.c
+++ b/drivers/usb/core/sysfs.c
@@ -244,7 +244,7 @@ static ssize_t urbnum_show(struct device *dev, struct device_attribute *attr,
struct usb_device *udev;
udev = to_usb_device(dev);
- return sprintf(buf, "%d\n", atomic_read(&udev->urbnum));
+ return sprintf(buf, "%d\n", atomic_read_unchecked(&udev->urbnum));
}
static DEVICE_ATTR_RO(urbnum);
diff --git a/drivers/usb/core/usb.c b/drivers/usb/core/usb.c
index 8d5b2f4..3896940 100644
--- a/drivers/usb/core/usb.c
+++ b/drivers/usb/core/usb.c
@@ -447,7 +447,7 @@ struct usb_device *usb_alloc_dev(struct usb_device *parent,
set_dev_node(&dev->dev, dev_to_node(bus->controller));
dev->state = USB_STATE_ATTACHED;
dev->lpm_disable_count = 1;
- atomic_set(&dev->urbnum, 0);
+ atomic_set_unchecked(&dev->urbnum, 0);
INIT_LIST_HEAD(&dev->ep0.urb_list);
dev->ep0.desc.bLength = USB_DT_ENDPOINT_SIZE;
diff --git a/drivers/usb/host/ehci-hub.c b/drivers/usb/host/ehci-hub.c
index 22abb68..50b7b84 100644
--- a/drivers/usb/host/ehci-hub.c
+++ b/drivers/usb/host/ehci-hub.c
@@ -773,7 +773,7 @@ static struct urb *request_single_step_set_feature_urb(
urb->transfer_flags = URB_DIR_IN;
usb_get_urb(urb);
atomic_inc(&urb->use_count);
- atomic_inc(&urb->dev->urbnum);
+ atomic_inc_unchecked(&urb->dev->urbnum);
urb->setup_dma = dma_map_single(
hcd->self.controller,
urb->setup_packet,
@@ -840,7 +840,7 @@ static int ehset_single_step_set_feature(struct usb_hcd *hcd, int port)
urb->status = -EINPROGRESS;
usb_get_urb(urb);
atomic_inc(&urb->use_count);
- atomic_inc(&urb->dev->urbnum);
+ atomic_inc_unchecked(&urb->dev->urbnum);
retval = submit_single_step_set_feature(hcd, urb, 0);
if (!retval && !wait_for_completion_timeout(&done,
msecs_to_jiffies(2000))) {
diff --git a/drivers/usb/misc/appledisplay.c b/drivers/usb/misc/appledisplay.c
index a0a3827..d7ec10b 100644
--- a/drivers/usb/misc/appledisplay.c
+++ b/drivers/usb/misc/appledisplay.c
@@ -84,7 +84,7 @@ struct appledisplay {
struct mutex sysfslock; /* concurrent read and write */
};
-static atomic_t count_displays = ATOMIC_INIT(0);
+static atomic_unchecked_t count_displays = ATOMIC_INIT(0);
static struct workqueue_struct *wq;
static void appledisplay_complete(struct urb *urb)
@@ -288,7 +288,7 @@ static int appledisplay_probe(struct usb_interface *iface,
/* Register backlight device */
snprintf(bl_name, sizeof(bl_name), "appledisplay%d",
- atomic_inc_return(&count_displays) - 1);
+ atomic_inc_return_unchecked(&count_displays) - 1);
memset(&props, 0, sizeof(struct backlight_properties));
props.type = BACKLIGHT_RAW;
props.max_brightness = 0xff;
diff --git a/drivers/usb/usbip/vhci.h b/drivers/usb/usbip/vhci.h
index a863a98..d272795 100644
--- a/drivers/usb/usbip/vhci.h
+++ b/drivers/usb/usbip/vhci.h
@@ -83,7 +83,7 @@ struct vhci_hcd {
unsigned resuming:1;
unsigned long re_timeout;
- atomic_t seqnum;
+ atomic_unchecked_t seqnum;
/*
* NOTE:
diff --git a/drivers/usb/usbip/vhci_hcd.c b/drivers/usb/usbip/vhci_hcd.c
index e9ef1ec..c3a0b04 100644
--- a/drivers/usb/usbip/vhci_hcd.c
+++ b/drivers/usb/usbip/vhci_hcd.c
@@ -440,7 +440,7 @@ static void vhci_tx_urb(struct urb *urb)
spin_lock(&vdev->priv_lock);
- priv->seqnum = atomic_inc_return(&the_controller->seqnum);
+ priv->seqnum = atomic_inc_return_unchecked(&the_controller->seqnum);
if (priv->seqnum == 0xffff)
dev_info(&urb->dev->dev, "seqnum max\n");
@@ -685,7 +685,7 @@ static int vhci_urb_dequeue(struct usb_hcd *hcd, struct urb *urb, int status)
return -ENOMEM;
}
- unlink->seqnum = atomic_inc_return(&the_controller->seqnum);
+ unlink->seqnum = atomic_inc_return_unchecked(&the_controller->seqnum);
if (unlink->seqnum == 0xffff)
pr_info("seqnum max\n");
@@ -889,7 +889,7 @@ static int vhci_start(struct usb_hcd *hcd)
vdev->rhport = rhport;
}
- atomic_set(&vhci->seqnum, 0);
+ atomic_set_unchecked(&vhci->seqnum, 0);
spin_lock_init(&vhci->lock);
hcd->power_budget = 0; /* no limit */
diff --git a/drivers/usb/usbip/vhci_rx.c b/drivers/usb/usbip/vhci_rx.c
index 00e4a54..d676f85 100644
--- a/drivers/usb/usbip/vhci_rx.c
+++ b/drivers/usb/usbip/vhci_rx.c
@@ -80,7 +80,7 @@ static void vhci_recv_ret_submit(struct vhci_device *vdev,
if (!urb) {
pr_err("cannot find a urb of seqnum %u\n", pdu->base.seqnum);
pr_info("max seqnum %d\n",
- atomic_read(&the_controller->seqnum));
+ atomic_read_unchecked(&the_controller->seqnum));
usbip_event_add(ud, VDEV_EVENT_ERROR_TCP);
return;
}
diff --git a/drivers/usb/wusbcore/wa-hc.h b/drivers/usb/wusbcore/wa-hc.h
index edc7267..9f65ce2 100644
--- a/drivers/usb/wusbcore/wa-hc.h
+++ b/drivers/usb/wusbcore/wa-hc.h
@@ -240,7 +240,7 @@ struct wahc {
spinlock_t xfer_list_lock;
struct work_struct xfer_enqueue_work;
struct work_struct xfer_error_work;
- atomic_t xfer_id_count;
+ atomic_unchecked_t xfer_id_count;
kernel_ulong_t quirks;
};
@@ -305,7 +305,7 @@ static inline void wa_init(struct wahc *wa)
INIT_WORK(&wa->xfer_enqueue_work, wa_urb_enqueue_run);
INIT_WORK(&wa->xfer_error_work, wa_process_errored_transfers_run);
wa->dto_in_use = 0;
- atomic_set(&wa->xfer_id_count, 1);
+ atomic_set_unchecked(&wa->xfer_id_count, 1);
/* init the buf in URBs */
for (index = 0; index < WA_MAX_BUF_IN_URBS; ++index)
usb_init_urb(&(wa->buf_in_urbs[index]));
diff --git a/drivers/usb/wusbcore/wa-xfer.c b/drivers/usb/wusbcore/wa-xfer.c
index 69af4fd..da390d7 100644
--- a/drivers/usb/wusbcore/wa-xfer.c
+++ b/drivers/usb/wusbcore/wa-xfer.c
@@ -314,7 +314,7 @@ static void wa_xfer_completion(struct wa_xfer *xfer)
*/
static void wa_xfer_id_init(struct wa_xfer *xfer)
{
- xfer->id = atomic_add_return(1, &xfer->wa->xfer_id_count);
+ xfer->id = atomic_add_return_unchecked(1, &xfer->wa->xfer_id_count);
}
/* Return the xfer's ID. */
diff --git a/drivers/video/fbdev/hyperv_fb.c b/drivers/video/fbdev/hyperv_fb.c
index 807ee22..7814cd6 100644
--- a/drivers/video/fbdev/hyperv_fb.c
+++ b/drivers/video/fbdev/hyperv_fb.c
@@ -240,7 +240,7 @@ static uint screen_fb_size;
static inline int synthvid_send(struct hv_device *hdev,
struct synthvid_msg *msg)
{
- static atomic64_t request_id = ATOMIC64_INIT(0);
+ static atomic64_unchecked_t request_id = ATOMIC64_INIT(0);
int ret;
msg->pipe_hdr.type = PIPE_MSG_DATA;
@@ -248,7 +248,7 @@ static inline int synthvid_send(struct hv_device *hdev,
ret = vmbus_sendpacket(hdev->channel, msg,
msg->vid_hdr.size + sizeof(struct pipe_msg_hdr),
- atomic64_inc_return(&request_id),
+ atomic64_inc_return_unchecked(&request_id),
VM_PKT_DATA_INBAND, 0);
if (ret)
diff --git a/drivers/video/fbdev/udlfb.c b/drivers/video/fbdev/udlfb.c
index ff2b873..5ba574d 100644
--- a/drivers/video/fbdev/udlfb.c
+++ b/drivers/video/fbdev/udlfb.c
@@ -623,11 +623,11 @@ static int dlfb_handle_damage(struct dlfb_data *dev, int x, int y,
dlfb_urb_completion(urb);
error:
- atomic_add(bytes_sent, &dev->bytes_sent);
- atomic_add(bytes_identical, &dev->bytes_identical);
- atomic_add(width*height*2, &dev->bytes_rendered);
+ atomic_add_unchecked(bytes_sent, &dev->bytes_sent);
+ atomic_add_unchecked(bytes_identical, &dev->bytes_identical);
+ atomic_add_unchecked(width*height*2, &dev->bytes_rendered);
end_cycles = get_cycles();
- atomic_add(((unsigned int) ((end_cycles - start_cycles)
+ atomic_add_unchecked(((unsigned int) ((end_cycles - start_cycles)
>> 10)), /* Kcycles */
&dev->cpu_kcycles_used);
@@ -748,11 +748,11 @@ static void dlfb_dpy_deferred_io(struct fb_info *info,
dlfb_urb_completion(urb);
error:
- atomic_add(bytes_sent, &dev->bytes_sent);
- atomic_add(bytes_identical, &dev->bytes_identical);
- atomic_add(bytes_rendered, &dev->bytes_rendered);
+ atomic_add_unchecked(bytes_sent, &dev->bytes_sent);
+ atomic_add_unchecked(bytes_identical, &dev->bytes_identical);
+ atomic_add_unchecked(bytes_rendered, &dev->bytes_rendered);
end_cycles = get_cycles();
- atomic_add(((unsigned int) ((end_cycles - start_cycles)
+ atomic_add_unchecked(((unsigned int) ((end_cycles - start_cycles)
>> 10)), /* Kcycles */
&dev->cpu_kcycles_used);
}
@@ -1373,7 +1373,7 @@ static ssize_t metrics_bytes_rendered_show(struct device *fbdev,
struct fb_info *fb_info = dev_get_drvdata(fbdev);
struct dlfb_data *dev = fb_info->par;
return snprintf(buf, PAGE_SIZE, "%u\n",
- atomic_read(&dev->bytes_rendered));
+ atomic_read_unchecked(&dev->bytes_rendered));
}
static ssize_t metrics_bytes_identical_show(struct device *fbdev,
@@ -1381,7 +1381,7 @@ static ssize_t metrics_bytes_identical_show(struct device *fbdev,
struct fb_info *fb_info = dev_get_drvdata(fbdev);
struct dlfb_data *dev = fb_info->par;
return snprintf(buf, PAGE_SIZE, "%u\n",
- atomic_read(&dev->bytes_identical));
+ atomic_read_unchecked(&dev->bytes_identical));
}
static ssize_t metrics_bytes_sent_show(struct device *fbdev,
@@ -1389,7 +1389,7 @@ static ssize_t metrics_bytes_sent_show(struct device *fbdev,
struct fb_info *fb_info = dev_get_drvdata(fbdev);
struct dlfb_data *dev = fb_info->par;
return snprintf(buf, PAGE_SIZE, "%u\n",
- atomic_read(&dev->bytes_sent));
+ atomic_read_unchecked(&dev->bytes_sent));
}
static ssize_t metrics_cpu_kcycles_used_show(struct device *fbdev,
@@ -1397,7 +1397,7 @@ static ssize_t metrics_cpu_kcycles_used_show(struct device *fbdev,
struct fb_info *fb_info = dev_get_drvdata(fbdev);
struct dlfb_data *dev = fb_info->par;
return snprintf(buf, PAGE_SIZE, "%u\n",
- atomic_read(&dev->cpu_kcycles_used));
+ atomic_read_unchecked(&dev->cpu_kcycles_used));
}
static ssize_t edid_show(
@@ -1457,10 +1457,10 @@ static ssize_t metrics_reset_store(struct device *fbdev,
struct fb_info *fb_info = dev_get_drvdata(fbdev);
struct dlfb_data *dev = fb_info->par;
- atomic_set(&dev->bytes_rendered, 0);
- atomic_set(&dev->bytes_identical, 0);
- atomic_set(&dev->bytes_sent, 0);
- atomic_set(&dev->cpu_kcycles_used, 0);
+ atomic_set_unchecked(&dev->bytes_rendered, 0);
+ atomic_set_unchecked(&dev->bytes_identical, 0);
+ atomic_set_unchecked(&dev->bytes_sent, 0);
+ atomic_set_unchecked(&dev->cpu_kcycles_used, 0);
return count;
}
diff --git a/include/linux/uio_driver.h b/include/linux/uio_driver.h
index 32c0e83..671eb35 100644
--- a/include/linux/uio_driver.h
+++ b/include/linux/uio_driver.h
@@ -67,7 +67,7 @@ struct uio_device {
struct module *owner;
struct device *dev;
int minor;
- atomic_t event;
+ atomic_unchecked_t event;
struct fasync_struct *async_queue;
wait_queue_head_t wait;
struct uio_info *info;
diff --git a/include/linux/usb.h b/include/linux/usb.h
index 447fe29..ff424fd 100644
--- a/include/linux/usb.h
+++ b/include/linux/usb.h
@@ -592,7 +592,7 @@ struct usb_device {
int maxchild;
u32 quirks;
- atomic_t urbnum;
+ atomic_unchecked_t urbnum;
unsigned long active_duration;
diff --git a/include/media/v4l2-device.h b/include/media/v4l2-device.h
index 9c58157..d86ebf5 100644
--- a/include/media/v4l2-device.h
+++ b/include/media/v4l2-device.h
@@ -93,7 +93,7 @@ int __must_check v4l2_device_register(struct device *dev, struct v4l2_device *v4
this function returns 0. If the name ends with a digit (e.g. cx18),
then the name will be set to cx18-0 since cx180 looks really odd. */
int v4l2_device_set_name(struct v4l2_device *v4l2_dev, const char *basename,
- atomic_t *instance);
+ atomic_unchecked_t *instance);
/* Set v4l2_dev->dev to NULL. Call when the USB parent disconnects.
Since the parent disappears this ensures that v4l2_dev doesn't have an
diff --git a/include/scsi/scsi_device.h b/include/scsi/scsi_device.h
index ae84b22..7954097 100644
--- a/include/scsi/scsi_device.h
+++ b/include/scsi/scsi_device.h
@@ -185,9 +185,9 @@ struct scsi_device {
unsigned int max_device_blocked; /* what device_blocked counts down from */
#define SCSI_DEFAULT_DEVICE_BLOCKED 3
- atomic_t iorequest_cnt;
- atomic_t iodone_cnt;
- atomic_t ioerr_cnt;
+ atomic_unchecked_t iorequest_cnt;
+ atomic_unchecked_t iodone_cnt;
+ atomic_unchecked_t ioerr_cnt;
struct device sdev_gendev,
sdev_dev;
diff --git a/include/target/target_core_base.h b/include/target/target_core_base.h
index 17ae2d6..2c06382 100644
--- a/include/target/target_core_base.h
+++ b/include/target/target_core_base.h
@@ -751,7 +751,7 @@ struct se_device {
atomic_long_t write_bytes;
/* Active commands on this virtual SE device */
atomic_t simple_cmds;
- atomic_t dev_ordered_id;
+ atomic_unchecked_t dev_ordered_id;
atomic_t dev_ordered_sync;
atomic_t dev_qf_count;
u32 export_count;
--
2.5.0
^ permalink raw reply related [flat|nested] 32+ messages in thread* [kernel-hardening][RFC PATCH v2 10/12] pax: opt out of PAX_REFCOUNT protection
2015-12-17 14:57 [kernel-hardening][RFC PATCH v2 00/12] Add PAX_REFCOUNT overflow protection David Windsor
` (8 preceding siblings ...)
2015-12-17 14:57 ` [kernel-hardening][RFC PATCH v2 09/12] drivers: opt out of PAX_REFCOUNT protection (2 " David Windsor
@ 2015-12-17 14:57 ` David Windsor
2015-12-17 14:57 ` [kernel-hardening][RFC PATCH v2 11/12] security: add Kconfig options for PAX_REFCOUNT David Windsor
` (2 subsequent siblings)
12 siblings, 0 replies; 32+ messages in thread
From: David Windsor @ 2015-12-17 14:57 UTC (permalink / raw)
To: kernel-hardening; +Cc: David Windsor
These changes are those that did not fit categorically into any other of the
preceding "opt-out" patches.
Signed-off-by: David Windsor <dave@progbits.org>
---
include/linux/genhd.h | 2 +-
lib/show_mem.c | 2 +-
sound/pci/ymfpci/ymfpci.h | 2 +-
sound/pci/ymfpci/ymfpci_main.c | 12 ++++++------
virt/kvm/kvm_main.c | 8 ++++----
5 files changed, 13 insertions(+), 13 deletions(-)
diff --git a/include/linux/genhd.h b/include/linux/genhd.h
index ec274e0..c183f10 100644
--- a/include/linux/genhd.h
+++ b/include/linux/genhd.h
@@ -194,7 +194,7 @@ struct gendisk {
struct kobject *slave_dir;
struct timer_rand_state *random;
- atomic_t sync_io; /* RAID */
+ atomic_unchecked_t sync_io; /* RAID */
struct disk_events *ev;
#ifdef CONFIG_BLK_DEV_INTEGRITY
struct blk_integrity *integrity;
diff --git a/lib/show_mem.c b/lib/show_mem.c
index adc98e18..0ce83c2 100644
--- a/lib/show_mem.c
+++ b/lib/show_mem.c
@@ -49,6 +49,6 @@ void show_mem(unsigned int filter)
quicklist_total_size());
#endif
#ifdef CONFIG_MEMORY_FAILURE
- printk("%lu pages hwpoisoned\n", atomic_long_read(&num_poisoned_pages));
+ printk("%lu pages hwpoisoned\n", atomic_long_read_unchecked(&num_poisoned_pages));
#endif
}
diff --git a/sound/pci/ymfpci/ymfpci.h b/sound/pci/ymfpci/ymfpci.h
index 149d4cb..7784769 100644
--- a/sound/pci/ymfpci/ymfpci.h
+++ b/sound/pci/ymfpci/ymfpci.h
@@ -358,7 +358,7 @@ struct snd_ymfpci {
spinlock_t reg_lock;
spinlock_t voice_lock;
wait_queue_head_t interrupt_sleep;
- atomic_t interrupt_sleep_count;
+ atomic_unchecked_t interrupt_sleep_count;
struct snd_info_entry *proc_entry;
const struct firmware *dsp_microcode;
const struct firmware *controller_microcode;
diff --git a/sound/pci/ymfpci/ymfpci_main.c b/sound/pci/ymfpci/ymfpci_main.c
index 4c26076..a13f370 100644
--- a/sound/pci/ymfpci/ymfpci_main.c
+++ b/sound/pci/ymfpci/ymfpci_main.c
@@ -204,8 +204,8 @@ static void snd_ymfpci_hw_stop(struct snd_ymfpci *chip)
if ((snd_ymfpci_readl(chip, YDSXGR_STATUS) & 2) == 0)
break;
}
- if (atomic_read(&chip->interrupt_sleep_count)) {
- atomic_set(&chip->interrupt_sleep_count, 0);
+ if (atomic_read_unchecked(&chip->interrupt_sleep_count)) {
+ atomic_set_unchecked(&chip->interrupt_sleep_count, 0);
wake_up(&chip->interrupt_sleep);
}
__end:
@@ -789,7 +789,7 @@ static void snd_ymfpci_irq_wait(struct snd_ymfpci *chip)
continue;
init_waitqueue_entry(&wait, current);
add_wait_queue(&chip->interrupt_sleep, &wait);
- atomic_inc(&chip->interrupt_sleep_count);
+ atomic_inc_unchecked(&chip->interrupt_sleep_count);
schedule_timeout_uninterruptible(msecs_to_jiffies(50));
remove_wait_queue(&chip->interrupt_sleep, &wait);
}
@@ -827,8 +827,8 @@ static irqreturn_t snd_ymfpci_interrupt(int irq, void *dev_id)
snd_ymfpci_writel(chip, YDSXGR_MODE, mode);
spin_unlock(&chip->reg_lock);
- if (atomic_read(&chip->interrupt_sleep_count)) {
- atomic_set(&chip->interrupt_sleep_count, 0);
+ if (atomic_read_unchecked(&chip->interrupt_sleep_count)) {
+ atomic_set_unchecked(&chip->interrupt_sleep_count, 0);
wake_up(&chip->interrupt_sleep);
}
}
@@ -2384,7 +2384,7 @@ int snd_ymfpci_create(struct snd_card *card,
spin_lock_init(&chip->reg_lock);
spin_lock_init(&chip->voice_lock);
init_waitqueue_head(&chip->interrupt_sleep);
- atomic_set(&chip->interrupt_sleep_count, 0);
+ atomic_set_unchecked(&chip->interrupt_sleep_count, 0);
chip->card = card;
chip->pci = pci;
chip->irq = -1;
diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
index 5a2a78a..f868724 100644
--- a/virt/kvm/kvm_main.c
+++ b/virt/kvm/kvm_main.c
@@ -81,7 +81,7 @@ LIST_HEAD(vm_list);
static cpumask_var_t cpus_hardware_enabled;
static int kvm_usage_count;
-static atomic_t hardware_enable_failed;
+static atomic_unchecked_t hardware_enable_failed;
struct kmem_cache *kvm_vcpu_cache;
EXPORT_SYMBOL_GPL(kvm_vcpu_cache);
@@ -2961,7 +2961,7 @@ static void hardware_enable_nolock(void *junk)
if (r) {
cpumask_clear_cpu(cpu, cpus_hardware_enabled);
- atomic_inc(&hardware_enable_failed);
+ atomic_inc_unchecked(&hardware_enable_failed);
pr_info("kvm: enabling virtualization on CPU%d failed\n", cpu);
}
}
@@ -3016,10 +3016,10 @@ static int hardware_enable_all(void)
kvm_usage_count++;
if (kvm_usage_count == 1) {
- atomic_set(&hardware_enable_failed, 0);
+ atomic_set_unchecked(&hardware_enable_failed, 0);
on_each_cpu(hardware_enable_nolock, NULL, 1);
- if (atomic_read(&hardware_enable_failed)) {
+ if (atomic_read_unchecked(&hardware_enable_failed)) {
hardware_disable_all_nolock();
r = -EBUSY;
}
--
2.5.0
^ permalink raw reply related [flat|nested] 32+ messages in thread* [kernel-hardening][RFC PATCH v2 11/12] security: add Kconfig options for PAX_REFCOUNT
2015-12-17 14:57 [kernel-hardening][RFC PATCH v2 00/12] Add PAX_REFCOUNT overflow protection David Windsor
` (9 preceding siblings ...)
2015-12-17 14:57 ` [kernel-hardening][RFC PATCH v2 10/12] pax: opt out of PAX_REFCOUNT protection David Windsor
@ 2015-12-17 14:57 ` David Windsor
2015-12-17 20:53 ` Kees Cook
2015-12-17 14:57 ` [kernel-hardening][RFC PATCH v2 12/12] lkdtm: add test for atomic_t underflow/overflow David Windsor
2015-12-17 20:55 ` [kernel-hardening][RFC PATCH v2 00/12] Add PAX_REFCOUNT overflow protection Kees Cook
12 siblings, 1 reply; 32+ messages in thread
From: David Windsor @ 2015-12-17 14:57 UTC (permalink / raw)
To: kernel-hardening; +Cc: David Windsor
Currently the Kconfig options for this feature retain the original
naming from Pax/grsecurity.
Signed-off-by: David Windsor <dave@progbits.org>
---
security/Kconfig | 72 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 72 insertions(+)
diff --git a/security/Kconfig b/security/Kconfig
index bf4ec46..cf08034 100644
--- a/security/Kconfig
+++ b/security/Kconfig
@@ -4,6 +4,78 @@
menu "Security options"
+menu "Grsecurity"
+
+config GRKERNSEC
+ bool "Grsecurity"
+ help
+ If you say Y here, you will be able to configure many features
+ that will enhance the security of your system. It is highly
+ recommended that you say Y here and read through the help
+ for each option so that you fully understand the features and
+ can evaluate their usefulness for your machine.
+
+choice
+ prompt "Configuration Method"
+ depends on GRKERNSEC
+ default GRKERNSEC_CONFIG_CUSTOM
+ help
+
+config GRKERNSEC_CONFIG_AUTO
+ bool "Automatic"
+ help
+ If you choose this configuration method, you'll be able to answer a small
+ number of simple questions about how you plan to use this kernel.
+ The settings of grsecurity and PaX will be automatically configured for
+ the highest commonly-used settings within the provided constraints.
+
+ If you require additional configuration, custom changes can still be made
+ from the "custom configuration" menu.
+
+config GRKERNSEC_CONFIG_CUSTOM
+ bool "Custom"
+ help
+ If you choose this configuration method, you'll be able to configure all
+ grsecurity and PaX settings manually. Via this method, no options are
+ automatically enabled.
+
+ Take note that if menuconfig is exited with this configuration method
+ chosen, you will not be able to use the automatic configuration methods
+ without starting again with a kernel configuration with no grsecurity
+ or PaX options specified inside.
+
+endchoice
+
+menu "PaX"
+
+menu "Miscellaneous hardening features"
+
+config PAX_REFCOUNT
+ bool "Prevent various kernel object reference counter overflows"
+ default y if GRKERNSEC_CONFIG_AUTO
+ depends on GRKERNSEC && ((ARM && (CPU_V6 || CPU_V6K || CPU_V7)) || MIPS || PPC || SPARC64 || X86)
+ help
+ By saying Y here the kernel will detect and prevent overflowing
+ various (but not all) kinds of object reference counters. Such
+ overflows can normally occur due to bugs only and are often, if
+ not always, exploitable.
+
+ The tradeoff is that data structures protected by an overflowed
+ refcount will never be freed and therefore will leak memory. Note
+ that this leak also happens even without this protection but in
+ that case the overflow can eventually trigger the freeing of the
+ data structure while it is still being used elsewhere, resulting
+ in the exploitable situation that this feature prevents.
+
+ Since this has a negligible performance impact, you should enable
+ this feature.
+
+endmenu
+
+endmenu
+
+endmenu
+
source security/keys/Kconfig
config SECURITY_DMESG_RESTRICT
--
2.5.0
^ permalink raw reply related [flat|nested] 32+ messages in thread* Re: [kernel-hardening][RFC PATCH v2 11/12] security: add Kconfig options for PAX_REFCOUNT
2015-12-17 14:57 ` [kernel-hardening][RFC PATCH v2 11/12] security: add Kconfig options for PAX_REFCOUNT David Windsor
@ 2015-12-17 20:53 ` Kees Cook
0 siblings, 0 replies; 32+ messages in thread
From: Kees Cook @ 2015-12-17 20:53 UTC (permalink / raw)
To: kernel-hardening@lists.openwall.com; +Cc: David Windsor
On Thu, Dec 17, 2015 at 6:57 AM, David Windsor <dave@progbits.org> wrote:
> Currently the Kconfig options for this feature retain the original
> naming from Pax/grsecurity.
>
> Signed-off-by: David Windsor <dave@progbits.org>
> ---
> security/Kconfig | 72 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++
> 1 file changed, 72 insertions(+)
>
> diff --git a/security/Kconfig b/security/Kconfig
> index bf4ec46..cf08034 100644
> --- a/security/Kconfig
> +++ b/security/Kconfig
> @@ -4,6 +4,78 @@
>
> menu "Security options"
>
> +menu "Grsecurity"
Let's start our bikeshedding! :) As requested during the kernel
summit, people would like a "hardening" menu, so let's call this
"Kernel hardening" for now.
> +config GRKERNSEC
> + bool "Grsecurity"
> + help
> + If you say Y here, you will be able to configure many features
> + that will enhance the security of your system. It is highly
> + recommended that you say Y here and read through the help
> + for each option so that you fully understand the features and
> + can evaluate their usefulness for your machine.
> +
> +choice
> + prompt "Configuration Method"
> + depends on GRKERNSEC
> + default GRKERNSEC_CONFIG_CUSTOM
> + help
> +
> +config GRKERNSEC_CONFIG_AUTO
> + bool "Automatic"
> + help
> + If you choose this configuration method, you'll be able to answer a small
> + number of simple questions about how you plan to use this kernel.
> + The settings of grsecurity and PaX will be automatically configured for
> + the highest commonly-used settings within the provided constraints.
> +
> + If you require additional configuration, custom changes can still be made
> + from the "custom configuration" menu.
> +
> +config GRKERNSEC_CONFIG_CUSTOM
> + bool "Custom"
> + help
> + If you choose this configuration method, you'll be able to configure all
> + grsecurity and PaX settings manually. Via this method, no options are
> + automatically enabled.
> +
> + Take note that if menuconfig is exited with this configuration method
> + chosen, you will not be able to use the automatic configuration methods
> + without starting again with a kernel configuration with no grsecurity
> + or PaX options specified inside.
> +
> +endchoice
And we'll drop these.
> +menu "PaX"
> +
> +menu "Miscellaneous hardening features"
> +
> +config PAX_REFCOUNT
I would propose "STRICT_REFCOUNT" maybe?
> + bool "Prevent various kernel object reference counter overflows"
> + default y if GRKERNSEC_CONFIG_AUTO
> + depends on GRKERNSEC && ((ARM && (CPU_V6 || CPU_V6K || CPU_V7)) || MIPS || PPC || SPARC64 || X86)
As I detail in the other email, I think architectures should opt in to
this, so a
depends on HAVE_ARCH_STRICT_REFCOUNT
here might be best.
> + help
> + By saying Y here the kernel will detect and prevent overflowing
> + various (but not all) kinds of object reference counters. Such
> + overflows can normally occur due to bugs only and are often, if
> + not always, exploitable.
> +
> + The tradeoff is that data structures protected by an overflowed
> + refcount will never be freed and therefore will leak memory. Note
> + that this leak also happens even without this protection but in
> + that case the overflow can eventually trigger the freeing of the
> + data structure while it is still being used elsewhere, resulting
> + in the exploitable situation that this feature prevents.
> +
> + Since this has a negligible performance impact, you should enable
> + this feature.
> +
> +endmenu
> +
> +endmenu
> +
> +endmenu
> +
> source security/keys/Kconfig
>
> config SECURITY_DMESG_RESTRICT
> --
> 2.5.0
>
>
I also think part of the commit message here should include links to
example exploits that this kills.
-Kees
--
Kees Cook
Chrome OS & Brillo Security
^ permalink raw reply [flat|nested] 32+ messages in thread
* [kernel-hardening][RFC PATCH v2 12/12] lkdtm: add test for atomic_t underflow/overflow
2015-12-17 14:57 [kernel-hardening][RFC PATCH v2 00/12] Add PAX_REFCOUNT overflow protection David Windsor
` (10 preceding siblings ...)
2015-12-17 14:57 ` [kernel-hardening][RFC PATCH v2 11/12] security: add Kconfig options for PAX_REFCOUNT David Windsor
@ 2015-12-17 14:57 ` David Windsor
2015-12-17 20:55 ` [kernel-hardening][RFC PATCH v2 00/12] Add PAX_REFCOUNT overflow protection Kees Cook
12 siblings, 0 replies; 32+ messages in thread
From: David Windsor @ 2015-12-17 14:57 UTC (permalink / raw)
To: kernel-hardening; +Cc: David Windsor
dmesg output of running this LKDTM test:
[187095.475573] lkdtm: No crash points registered, enable through debugfs
[187118.020257] lkdtm: Performing direct entry WRAP_ATOMIC
[187118.030045] lkdtm: attempting atomic underflow
[187118.030929] PAX: refcount overflow detected in: bash:1790, uid/euid: 0/0
[187118.071667] PAX: refcount overflow occured at: lkdtm_do_action+0x19e/0x400 [lkdtm]
[187118.081423] CPU: 3 PID: 1790 Comm: bash Not tainted 4.2.6-pax-refcount-split+ #2
[187118.083403] Hardware name: innotek GmbH VirtualBox/VirtualBox, BIOS VirtualBox 12/01/2006
[187118.102596] task: ffff8800da8de040 ti: ffff8800da8e4000 task.ti: ffff8800da8e4000
[187118.111321] RIP: 0010:[<ffffffffc00fc2fe>] [<ffffffffc00fc2fe>] lkdtm_do_action+0x19e/0x400 [lkdtm]
...
[187118.128074] lkdtm: attempting atomic overflow
[187118.128080] PAX: refcount overflow detected in: bash:1790, uid/euid: 0/0
[187118.128082] PAX: refcount overflow occured at: lkdtm_do_action+0x1b6/0x400 [lkdtm]
[187118.128085] CPU: 3 PID: 1790 Comm: bash Not tainted 4.2.6-pax-refcount-split+ #2
[187118.128086] Hardware name: innotek GmbH VirtualBox/VirtualBox, BIOS VirtualBox 12/01/2006
[187118.128088] task: ffff8800da8de040 ti: ffff8800da8e4000 task.ti: ffff8800da8e4000
[187118.128092] RIP: 0010:[<ffffffffc00fc316>] [<ffffffffc00fc316>] lkdtm_do_action+0x1b6/0x400 [lkdtm]
Signed-off-by: David Windsor <dave@progbits.org>
---
drivers/misc/lkdtm.c | 13 +++++++++++++
1 file changed, 13 insertions(+)
diff --git a/drivers/misc/lkdtm.c b/drivers/misc/lkdtm.c
index b5abe34..5002747 100644
--- a/drivers/misc/lkdtm.c
+++ b/drivers/misc/lkdtm.c
@@ -104,6 +104,7 @@ enum ctype {
CT_ACCESS_USERSPACE,
CT_WRITE_RO,
CT_WRITE_KERN,
+ CT_WRAP_ATOMIC
};
static char* cp_name[] = {
@@ -141,6 +142,7 @@ static char* cp_type[] = {
"ACCESS_USERSPACE",
"WRITE_RO",
"WRITE_KERN",
+ "WRAP_ATOMIC"
};
static struct jprobe lkdtm;
@@ -522,6 +524,17 @@ static void lkdtm_do_action(enum ctype which)
do_overwritten();
break;
}
+ case CT_WRAP_ATOMIC: {
+ atomic_t under = ATOMIC_INIT(INT_MIN);
+ atomic_t over = ATOMIC_INIT(INT_MAX);
+
+ pr_info("attempting atomic underflow\n");
+ atomic_dec(&under);
+ pr_info("attempting atomic overflow\n");
+ atomic_inc(&over);
+
+ return;
+ }
case CT_NONE:
default:
break;
--
2.5.0
^ permalink raw reply related [flat|nested] 32+ messages in thread* Re: [kernel-hardening][RFC PATCH v2 00/12] Add PAX_REFCOUNT overflow protection
2015-12-17 14:57 [kernel-hardening][RFC PATCH v2 00/12] Add PAX_REFCOUNT overflow protection David Windsor
` (11 preceding siblings ...)
2015-12-17 14:57 ` [kernel-hardening][RFC PATCH v2 12/12] lkdtm: add test for atomic_t underflow/overflow David Windsor
@ 2015-12-17 20:55 ` Kees Cook
2016-01-19 19:07 ` Kees Cook
12 siblings, 1 reply; 32+ messages in thread
From: Kees Cook @ 2015-12-17 20:55 UTC (permalink / raw)
To: kernel-hardening@lists.openwall.com; +Cc: David Windsor
On Thu, Dec 17, 2015 at 6:57 AM, David Windsor <dave@progbits.org> wrote:
> NOTE: This is a v2 submission because patch 3/5 in v1 was too large to sent
> to kernel-hardening. Taking that as a sign that the patch needed to be split,
> I'm sending this version of the patchset, with the patches split more or less
> on a per-maintainer basis (except for those in drivers/).
>
> These patches are in support of the kernel self-protection effort recently
> proposed on kernel-hardening.
>
> The Linux kernel can be better protected against use-after-free exploits.
> This patch adds functionality originally found in the PaX project [1] to
> protect the kernel against integer overflows affecting kernel reference
> counters, a class of bugs that can lead to use-after-free exploits.
>
> Problems arise when reference counters get overflowed. The existence of a
> kernel path in which a reference counter is incremented more than it is
> decremented leads to an overflow bug. This buggy path can be executed until
> INT_MAX is reached, at which point further increments will cause the reference
> counter to wrap around to 0. At this point, the kernel will free the counted
> object while it still has legitimate users, creating conditions for a
> use-after-free exploit to occur.
>
>
> Patch Details
> ----------------
>
> These patches were taken from the PaX project's PAX_REFCOUNT feature. It
> applies cleanly to linux-stable/v4.2.6 [2]. I am in the process of rebasing
> these patches to linux-next, but wanted to submit them now for general
> comments, bikeshedding, etc.
>
> Most of the changes involve users of atomic_t opting out of overflow protection:
> many atomic_t users use the type not as a reference counter, but as some type
> of statistical counter (packet counters, etc.).
>
>
> Issues
> ---------
>
> * The patch is currently based on linux-stable/v4.2.6. I am in the process of
> rebasing it to linux-next and will submit a follow-up patchset soon.
>
> * PAX_REFCOUNT naming: I picked this naming scheme by default. It will likely
> need to change before going upstream.
>
> * When an overflow is detected, SIGKILL is sent to the offending process.
> This may be too drastic for an initial upstream submission. WARN_ON may be
> more appropriate until distros have some time to absorb it and report any
> unaddressed overflows.
>
> [1] https://forums.grsecurity.net/viewtopic.php?f=7&t=4173
> [2] git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable.git
>
>
> David Windsor (12):
> x86: add PAX_REFCOUNT support
Since this feature needs per-architecture support, the CONFIG stuff
around this will likely need to use something like how
HAVE_ARCH_SECCOMP_FILTER works in arch/Kconfig, and once an
architecture supports it, they can select HAVE_ARCH_STRICT_REFCOUNT or
whatever we'd like to name this.
> percpu_ref: decrease per-CPU refcount bias
> kernel: opt out of PAX_REFCOUNT protection
> mm: opt out of PAX_REFCOUNT protection
> fs: opt out of PAX_REFCOUNT protection
> net: opt out of PAX_REFCOUNT protection
> security: opt out of PAX_REFCOUNT protection
> drivers: opt out of PAX_REFCOUNT protection (1 of 2)
> drivers: opt out of PAX_REFCOUNT protection (2 of 2)
> pax: opt out of PAX_REFCOUNT protection
While there are lot of these, I think this is correct. We want
atomic_t to be protected by default, and non-reference counters should
be explicitly named as such.
> security: add Kconfig options for PAX_REFCOUNT
As above, I think I would add the CONFIG first, and then add per-arch
support after that.
> lkdtm: add test for atomic_t underflow/overflow
Thanks for fixing my blind attempt at this test. :)
-Kees
>
> arch/x86/include/asm/atomic.h | 263 ++++++++++++++++++++--
> arch/x86/include/asm/atomic64_32.h | 100 +++++++++
> arch/x86/include/asm/atomic64_64.h | 164 +++++++++++++-
> arch/x86/include/asm/bitops.h | 6 +-
> arch/x86/include/asm/cmpxchg.h | 39 ++++
> arch/x86/include/asm/hw_irq.h | 4 +-
> arch/x86/include/asm/local.h | 106 ++++++++-
> arch/x86/include/asm/preempt.h | 2 +-
> arch/x86/include/asm/rmwcc.h | 84 +++++++-
> arch/x86/include/asm/rwsem.h | 60 +++++-
> arch/x86/kernel/apic/apic.c | 2 +-
> arch/x86/kernel/apic/io_apic.c | 4 +-
> arch/x86/kernel/cpu/mcheck/mce.c | 12 +-
> arch/x86/kernel/i8259.c | 2 +-
> arch/x86/kernel/irq.c | 8 +-
> arch/x86/kernel/kgdb.c | 6 +-
> arch/x86/kernel/pvclock.c | 8 +-
> arch/x86/kernel/tboot.c | 8 +-
> arch/x86/kernel/traps.c | 6 +
> arch/x86/lib/atomic64_386_32.S | 164 ++++++++++++++
> arch/x86/lib/atomic64_cx8_32.S | 94 +++++++-
> arch/x86/mm/mmio-mod.c | 4 +-
> drivers/acpi/apei/ghes.c | 4 +-
> drivers/ata/libata-core.c | 4 +-
> drivers/ata/libata-scsi.c | 2 +-
> drivers/ata/libata.h | 2 +-
> drivers/atm/adummy.c | 2 +-
> drivers/atm/ambassador.c | 8 +-
> drivers/atm/atmtcp.c | 14 +-
> drivers/atm/eni.c | 10 +-
> drivers/atm/firestream.c | 8 +-
> drivers/atm/fore200e.c | 14 +-
> drivers/atm/he.c | 18 +-
> drivers/atm/horizon.c | 4 +-
> drivers/atm/idt77252.c | 36 ++--
> drivers/atm/iphase.c | 34 +--
> drivers/atm/lanai.c | 12 +-
> drivers/atm/nicstar.c | 46 ++--
> drivers/atm/solos-pci.c | 4 +-
> drivers/atm/suni.c | 4 +-
> drivers/atm/uPD98402.c | 16 +-
> drivers/atm/zatm.c | 6 +-
> drivers/base/power/wakeup.c | 8 +-
> drivers/block/drbd/drbd_bitmap.c | 2 +-
> drivers/block/drbd/drbd_int.h | 8 +-
> drivers/block/drbd/drbd_main.c | 12 +-
> drivers/block/drbd/drbd_nl.c | 4 +-
> drivers/block/drbd/drbd_receiver.c | 30 +--
> drivers/block/drbd/drbd_worker.c | 8 +-
> drivers/char/ipmi/ipmi_msghandler.c | 8 +-
> drivers/char/ipmi/ipmi_si_intf.c | 8 +-
> drivers/crypto/hifn_795x.c | 4 +-
> drivers/edac/edac_device.c | 4 +-
> drivers/edac/edac_pci.c | 4 +-
> drivers/edac/edac_pci_sysfs.c | 20 +-
> drivers/firewire/core-card.c | 4 +-
> drivers/firmware/efi/cper.c | 8 +-
> drivers/gpio/gpio-vr41xx.c | 2 +-
> drivers/gpu/drm/i810/i810_drv.h | 4 +-
> drivers/gpu/drm/mga/mga_drv.h | 4 +-
> drivers/gpu/drm/mga/mga_irq.c | 8 +-
> drivers/gpu/drm/qxl/qxl_cmd.c | 12 +-
> drivers/gpu/drm/qxl/qxl_debugfs.c | 8 +-
> drivers/gpu/drm/qxl/qxl_drv.h | 8 +-
> drivers/gpu/drm/qxl/qxl_irq.c | 16 +-
> drivers/gpu/drm/r128/r128_cce.c | 2 +-
> drivers/gpu/drm/r128/r128_drv.h | 4 +-
> drivers/gpu/drm/r128/r128_irq.c | 4 +-
> drivers/gpu/drm/r128/r128_state.c | 4 +-
> drivers/gpu/drm/radeon/radeon_drv.h | 2 +-
> drivers/gpu/drm/radeon/radeon_irq.c | 6 +-
> drivers/gpu/drm/via/via_drv.h | 4 +-
> drivers/gpu/drm/via/via_irq.c | 18 +-
> drivers/gpu/drm/vmwgfx/vmwgfx_drv.h | 2 +-
> drivers/gpu/drm/vmwgfx/vmwgfx_fifo.c | 6 +-
> drivers/gpu/drm/vmwgfx/vmwgfx_irq.c | 4 +-
> drivers/gpu/drm/vmwgfx/vmwgfx_marker.c | 2 +-
> drivers/hid/hid-core.c | 4 +-
> drivers/hv/channel.c | 2 +-
> drivers/hv/hv_balloon.c | 18 +-
> drivers/hv/hyperv_vmbus.h | 2 +-
> drivers/hwmon/sht15.c | 12 +-
> drivers/infiniband/core/cm.c | 32 +--
> drivers/infiniband/core/fmr_pool.c | 20 +-
> drivers/infiniband/hw/cxgb4/mem.c | 4 +-
> drivers/infiniband/hw/ipath/ipath_rc.c | 6 +-
> drivers/infiniband/hw/ipath/ipath_ruc.c | 6 +-
> drivers/infiniband/hw/mlx4/mad.c | 2 +-
> drivers/infiniband/hw/mlx4/mcg.c | 2 +-
> drivers/infiniband/hw/mlx4/mlx4_ib.h | 2 +-
> drivers/infiniband/hw/nes/nes.c | 4 +-
> drivers/infiniband/hw/nes/nes.h | 40 ++--
> drivers/infiniband/hw/nes/nes_cm.c | 62 +++---
> drivers/infiniband/hw/nes/nes_mgt.c | 8 +-
> drivers/infiniband/hw/nes/nes_nic.c | 40 ++--
> drivers/infiniband/hw/nes/nes_verbs.c | 10 +-
> drivers/input/gameport/gameport.c | 4 +-
> drivers/input/input.c | 4 +-
> drivers/input/joystick/xpad.c | 4 +-
> drivers/input/misc/ims-pcu.c | 4 +-
> drivers/input/serio/serio.c | 4 +-
> drivers/input/serio/serio_raw.c | 4 +-
> drivers/isdn/capi/capi.c | 10 +-
> drivers/md/dm-raid1.c | 18 +-
> drivers/md/dm-stripe.c | 10 +-
> drivers/md/dm.c | 16 +-
> drivers/md/md.c | 26 +--
> drivers/md/md.h | 6 +-
> drivers/md/raid1.c | 4 +-
> drivers/md/raid10.c | 16 +-
> drivers/md/raid5.c | 10 +-
> drivers/media/pci/ivtv/ivtv-driver.c | 2 +-
> drivers/media/pci/solo6x10/solo6x10-p2m.c | 2 +-
> drivers/media/pci/solo6x10/solo6x10.h | 2 +-
> drivers/media/pci/tw68/tw68-core.c | 2 +-
> drivers/media/radio/radio-maxiradio.c | 2 +-
> drivers/media/radio/radio-shark.c | 2 +-
> drivers/media/radio/radio-shark2.c | 2 +-
> drivers/media/radio/radio-si476x.c | 2 +-
> drivers/media/v4l2-core/v4l2-device.c | 4 +-
> drivers/misc/lis3lv02d/lis3lv02d.c | 8 +-
> drivers/misc/lis3lv02d/lis3lv02d.h | 2 +-
> drivers/misc/lkdtm.c | 13 ++
> drivers/misc/sgi-gru/gruhandles.c | 4 +-
> drivers/misc/sgi-gru/gruprocfs.c | 8 +-
> drivers/misc/sgi-gru/grutables.h | 158 +++++++-------
> drivers/net/hyperv/hyperv_net.h | 2 +-
> drivers/net/hyperv/rndis_filter.c | 4 +-
> drivers/net/ipvlan/ipvlan_core.c | 2 +-
> drivers/net/macvlan.c | 2 +-
> drivers/net/usb/sierra_net.c | 4 +-
> drivers/net/wireless/rt2x00/rt2x00.h | 2 +-
> drivers/net/wireless/rt2x00/rt2x00queue.c | 4 +-
> drivers/oprofile/buffer_sync.c | 8 +-
> drivers/oprofile/event_buffer.c | 2 +-
> drivers/oprofile/oprof.c | 2 +-
> drivers/oprofile/oprofile_stats.c | 10 +-
> drivers/oprofile/oprofile_stats.h | 10 +-
> drivers/oprofile/oprofilefs.c | 6 +-
> drivers/regulator/core.c | 4 +-
> drivers/scsi/fcoe/fcoe_sysfs.c | 12 +-
> drivers/scsi/hosts.c | 4 +-
> drivers/scsi/libfc/fc_exch.c | 50 ++---
> drivers/scsi/lpfc/lpfc.h | 8 +-
> drivers/scsi/lpfc/lpfc_debugfs.c | 18 +-
> drivers/scsi/lpfc/lpfc_scsi.c | 10 +-
> drivers/scsi/pmcraid.c | 20 +-
> drivers/scsi/pmcraid.h | 8 +-
> drivers/scsi/qla4xxx/ql4_def.h | 2 +-
> drivers/scsi/qla4xxx/ql4_os.c | 6 +-
> drivers/scsi/scsi_lib.c | 8 +-
> drivers/scsi/scsi_sysfs.c | 2 +-
> drivers/scsi/scsi_transport_fc.c | 6 +-
> drivers/scsi/scsi_transport_iscsi.c | 6 +-
> drivers/scsi/scsi_transport_srp.c | 6 +-
> drivers/scsi/sd.c | 2 +-
> drivers/staging/android/timed_output.c | 6 +-
> drivers/staging/octeon/ethernet-rx.c | 12 +-
> drivers/staging/octeon/ethernet.c | 8 +-
> drivers/target/sbp/sbp_target.c | 4 +-
> drivers/target/target_core_device.c | 2 +-
> drivers/target/target_core_transport.c | 2 +-
> drivers/tty/hvc/hvsi.c | 12 +-
> drivers/tty/hvc/hvsi_lib.c | 4 +-
> drivers/tty/serial/ioc4_serial.c | 6 +-
> drivers/tty/serial/msm_serial.c | 4 +-
> drivers/uio/uio.c | 12 +-
> drivers/usb/atm/usbatm.c | 24 +--
> drivers/usb/core/devices.c | 6 +-
> drivers/usb/core/hcd.c | 4 +-
> drivers/usb/core/sysfs.c | 2 +-
> drivers/usb/core/usb.c | 2 +-
> drivers/usb/host/ehci-hub.c | 4 +-
> drivers/usb/misc/appledisplay.c | 4 +-
> drivers/usb/usbip/vhci.h | 2 +-
> drivers/usb/usbip/vhci_hcd.c | 6 +-
> drivers/usb/usbip/vhci_rx.c | 2 +-
> drivers/usb/wusbcore/wa-hc.h | 4 +-
> drivers/usb/wusbcore/wa-xfer.c | 2 +-
> drivers/video/fbdev/hyperv_fb.c | 4 +-
> drivers/video/fbdev/udlfb.c | 32 +--
> fs/afs/inode.c | 4 +-
> fs/btrfs/delayed-inode.c | 6 +-
> fs/btrfs/delayed-inode.h | 4 +-
> fs/cachefiles/internal.h | 12 +-
> fs/cachefiles/namei.c | 2 +-
> fs/cachefiles/proc.c | 12 +-
> fs/ceph/super.c | 4 +-
> fs/cifs/cifs_debug.c | 12 +-
> fs/cifs/cifsfs.c | 4 +-
> fs/cifs/cifsglob.h | 54 ++---
> fs/cifs/misc.c | 4 +-
> fs/cifs/smb1ops.c | 80 +++----
> fs/cifs/smb2ops.c | 84 ++++----
> fs/coda/cache.c | 10 +-
> fs/coredump.c | 6 +-
> fs/exec.c | 24 +++
> fs/ext4/ext4.h | 20 +-
> fs/ext4/mballoc.c | 44 ++--
> fs/fscache/cookie.c | 40 ++--
> fs/fscache/internal.h | 202 ++++++++---------
> fs/fscache/object.c | 26 +--
> fs/fscache/operation.c | 38 ++--
> fs/fscache/page.c | 110 +++++-----
> fs/fscache/stats.c | 348 +++++++++++++++---------------
> fs/inode.c | 4 +-
> fs/kernfs/file.c | 12 +-
> fs/lockd/clntproc.c | 4 +-
> fs/namespace.c | 4 +-
> fs/nfs/inode.c | 6 +-
> fs/notify/notification.c | 4 +-
> fs/ocfs2/localalloc.c | 2 +-
> fs/ocfs2/ocfs2.h | 10 +-
> fs/ocfs2/suballoc.c | 12 +-
> fs/ocfs2/super.c | 20 +-
> fs/proc/meminfo.c | 2 +-
> fs/quota/netlink.c | 4 +-
> fs/reiserfs/do_balan.c | 2 +-
> fs/reiserfs/procfs.c | 2 +-
> fs/reiserfs/reiserfs.h | 4 +-
> include/asm-generic/atomic-long.h | 210 ++++++++++++++++++
> include/asm-generic/atomic64.h | 12 ++
> include/asm-generic/local.h | 13 ++
> include/linux/atmdev.h | 2 +-
> include/linux/blktrace_api.h | 2 +-
> include/linux/fscache-cache.h | 2 +-
> include/linux/genhd.h | 2 +-
> include/linux/irqdesc.h | 2 +-
> include/linux/kgdb.h | 2 +-
> include/linux/mm.h | 2 +-
> include/linux/mmzone.h | 2 +-
> include/linux/netdevice.h | 6 +-
> include/linux/oprofile.h | 4 +-
> include/linux/padata.h | 2 +-
> include/linux/perf_event.h | 8 +-
> include/linux/sched.h | 15 +-
> include/linux/slab_def.h | 8 +-
> include/linux/sonet.h | 2 +-
> include/linux/sunrpc/svc_rdma.h | 18 +-
> include/linux/types.h | 16 ++
> include/linux/uio_driver.h | 2 +-
> include/linux/usb.h | 2 +-
> include/linux/vmstat.h | 20 +-
> include/media/v4l2-device.h | 2 +-
> include/net/bonding.h | 2 +-
> include/net/caif/cfctrl.h | 4 +-
> include/net/flow.h | 2 +-
> include/net/gro_cells.h | 2 +-
> include/net/inetpeer.h | 2 +-
> include/net/ip_fib.h | 2 +-
> include/net/ip_vs.h | 4 +-
> include/net/iucv/af_iucv.h | 2 +-
> include/net/net_namespace.h | 12 +-
> include/net/netns/ipv4.h | 4 +-
> include/net/netns/ipv6.h | 4 +-
> include/net/netns/xfrm.h | 2 +-
> include/net/sock.h | 6 +-
> include/net/xfrm.h | 2 +-
> include/scsi/scsi_device.h | 6 +-
> include/target/target_core_base.h | 2 +-
> include/video/udlfb.h | 8 +-
> kernel/audit.c | 8 +-
> kernel/auditsc.c | 4 +-
> kernel/debug/debug_core.c | 16 +-
> kernel/events/core.c | 22 +-
> kernel/irq/manage.c | 2 +-
> kernel/irq/spurious.c | 2 +-
> kernel/locking/lockdep.c | 2 +-
> kernel/locking/rtmutex-tester.c | 24 +--
> kernel/padata.c | 4 +-
> kernel/profile.c | 14 +-
> kernel/rcu/rcutorture.c | 60 +++---
> kernel/rcu/tree.c | 64 +++---
> kernel/rcu/tree.h | 26 +--
> kernel/rcu/tree_plugin.h | 12 +-
> kernel/rcu/tree_trace.c | 22 +-
> kernel/sched/auto_group.c | 4 +-
> kernel/time/timer_stats.c | 10 +-
> kernel/trace/blktrace.c | 6 +-
> kernel/trace/ftrace.c | 4 +-
> kernel/trace/ring_buffer.c | 96 ++++-----
> kernel/trace/trace_clock.c | 4 +-
> kernel/trace/trace_functions_graph.c | 4 +-
> kernel/trace/trace_mmiotrace.c | 8 +-
> lib/percpu-refcount.c | 2 +-
> lib/show_mem.c | 2 +-
> mm/backing-dev.c | 4 +-
> mm/memory-failure.c | 30 +--
> mm/page_alloc.c | 6 +-
> mm/slab.c | 20 +-
> mm/sparse.c | 2 +-
> mm/swapfile.c | 12 +-
> mm/vmstat.c | 12 +-
> net/atm/atm_misc.c | 8 +-
> net/atm/proc.c | 6 +-
> net/atm/resources.c | 4 +-
> net/batman-adv/bat_iv_ogm.c | 8 +-
> net/batman-adv/fragmentation.c | 2 +-
> net/batman-adv/soft-interface.c | 6 +-
> net/batman-adv/types.h | 6 +-
> net/caif/cfctrl.c | 10 +-
> net/ceph/messenger.c | 4 +-
> net/core/datagram.c | 2 +-
> net/core/dev.c | 12 +-
> net/core/flow.c | 6 +-
> net/core/net-sysfs.c | 2 +-
> net/core/netpoll.c | 4 +-
> net/core/rtnetlink.c | 2 +-
> net/core/sock.c | 12 +-
> net/core/sock_diag.c | 6 +-
> net/ipv4/devinet.c | 4 +-
> net/ipv4/fib_frontend.c | 6 +-
> net/ipv4/fib_semantics.c | 2 +-
> net/ipv4/inet_connection_sock.c | 4 +-
> net/ipv4/inet_timewait_sock.c | 2 +-
> net/ipv4/inetpeer.c | 2 +-
> net/ipv4/ip_fragment.c | 2 +-
> net/ipv4/ping.c | 2 +-
> net/ipv4/raw.c | 4 +-
> net/ipv4/route.c | 10 +-
> net/ipv4/tcp_input.c | 2 +-
> net/ipv4/udp.c | 10 +-
> net/ipv6/addrconf.c | 6 +-
> net/ipv6/af_inet6.c | 2 +-
> net/ipv6/datagram.c | 2 +-
> net/ipv6/ip6_fib.c | 4 +-
> net/ipv6/raw.c | 6 +-
> net/ipv6/udp.c | 6 +-
> net/iucv/af_iucv.c | 4 +-
> net/key/af_key.c | 4 +-
> net/l2tp/l2tp_eth.c | 38 ++--
> net/netfilter/ipvs/ip_vs_conn.c | 6 +-
> net/netfilter/ipvs/ip_vs_core.c | 4 +-
> net/netfilter/ipvs/ip_vs_ctl.c | 10 +-
> net/netfilter/ipvs/ip_vs_sync.c | 6 +-
> net/netfilter/ipvs/ip_vs_xmit.c | 4 +-
> net/netfilter/nfnetlink_log.c | 4 +-
> net/netfilter/xt_statistic.c | 8 +-
> net/netlink/af_netlink.c | 4 +-
> net/openvswitch/vport.c | 16 +-
> net/openvswitch/vport.h | 8 +-
> net/packet/af_packet.c | 4 +-
> net/phonet/pep.c | 6 +-
> net/phonet/socket.c | 2 +-
> net/rds/cong.c | 6 +-
> net/rds/ib.h | 2 +-
> net/rds/ib_cm.c | 2 +-
> net/rds/ib_recv.c | 4 +-
> net/rds/iw.h | 2 +-
> net/rds/iw_cm.c | 2 +-
> net/rds/iw_recv.c | 4 +-
> net/rxrpc/af_rxrpc.c | 2 +-
> net/rxrpc/ar-ack.c | 14 +-
> net/rxrpc/ar-call.c | 2 +-
> net/rxrpc/ar-connection.c | 2 +-
> net/rxrpc/ar-connevent.c | 2 +-
> net/rxrpc/ar-input.c | 4 +-
> net/rxrpc/ar-internal.h | 8 +-
> net/rxrpc/ar-local.c | 2 +-
> net/rxrpc/ar-output.c | 4 +-
> net/rxrpc/ar-peer.c | 2 +-
> net/rxrpc/ar-proc.c | 4 +-
> net/rxrpc/ar-transport.c | 2 +-
> net/rxrpc/rxkad.c | 4 +-
> net/sched/sch_generic.c | 4 +-
> net/sunrpc/auth_gss/svcauth_gss.c | 4 +-
> net/sunrpc/sched.c | 4 +-
> net/sunrpc/xprtrdma/svc_rdma.c | 36 ++--
> net/sunrpc/xprtrdma/svc_rdma_recvfrom.c | 8 +-
> net/sunrpc/xprtrdma/svc_rdma_sendto.c | 2 +-
> net/sunrpc/xprtrdma/svc_rdma_transport.c | 10 +-
> net/xfrm/xfrm_policy.c | 10 +-
> net/xfrm/xfrm_state.c | 4 +-
> security/Kconfig | 72 +++++++
> security/integrity/ima/ima.h | 4 +-
> security/integrity/ima/ima_api.c | 2 +-
> security/integrity/ima/ima_fs.c | 4 +-
> security/integrity/ima/ima_queue.c | 2 +-
> security/selinux/avc.c | 6 +-
> security/selinux/include/xfrm.h | 2 +-
> sound/pci/ymfpci/ymfpci.h | 2 +-
> sound/pci/ymfpci/ymfpci_main.c | 12 +-
> virt/kvm/kvm_main.c | 8 +-
> 383 files changed, 3387 insertions(+), 2034 deletions(-)
>
> --
> 2.5.0
>
>
--
Kees Cook
Chrome OS & Brillo Security
^ permalink raw reply [flat|nested] 32+ messages in thread* Re: [kernel-hardening][RFC PATCH v2 00/12] Add PAX_REFCOUNT overflow protection
2015-12-17 20:55 ` [kernel-hardening][RFC PATCH v2 00/12] Add PAX_REFCOUNT overflow protection Kees Cook
@ 2016-01-19 19:07 ` Kees Cook
2016-01-20 13:11 ` David Windsor
2016-01-20 20:57 ` [kernel-hardening][RFC PATCH v2 00/12] " Yves-Alexis Perez
0 siblings, 2 replies; 32+ messages in thread
From: Kees Cook @ 2016-01-19 19:07 UTC (permalink / raw)
To: kernel-hardening@lists.openwall.com; +Cc: David Windsor
Hi David,
On Thu, Dec 17, 2015 at 12:55 PM, Kees Cook <keescook@chromium.org> wrote:
> On Thu, Dec 17, 2015 at 6:57 AM, David Windsor <dave@progbits.org> wrote:
>> NOTE: This is a v2 submission because patch 3/5 in v1 was too large to sent
>> to kernel-hardening. Taking that as a sign that the patch needed to be split,
>> I'm sending this version of the patchset, with the patches split more or less
>> on a per-maintainer basis (except for those in drivers/).
How's the next spin coming? It looks like we have some new real-world
examples of exploits that would have been blocked by this protection:
http://perception-point.io/2016/01/14/analysis-and-exploitation-of-a-linux-kernel-vulnerability-cve-2016-0728/
:)
-Kees
--
Kees Cook
Chrome OS & Brillo Security
^ permalink raw reply [flat|nested] 32+ messages in thread
* Re: [kernel-hardening][RFC PATCH v2 00/12] Add PAX_REFCOUNT overflow protection
2016-01-19 19:07 ` Kees Cook
@ 2016-01-20 13:11 ` David Windsor
2016-02-02 11:33 ` David Windsor
2016-01-20 20:57 ` [kernel-hardening][RFC PATCH v2 00/12] " Yves-Alexis Perez
1 sibling, 1 reply; 32+ messages in thread
From: David Windsor @ 2016-01-20 13:11 UTC (permalink / raw)
To: Kees Cook; +Cc: kernel-hardening@lists.openwall.com
Hi Kees,
On Tue, Jan 19, 2016 at 2:07 PM, Kees Cook <keescook@chromium.org> wrote:
> Hi David,
>
> On Thu, Dec 17, 2015 at 12:55 PM, Kees Cook <keescook@chromium.org> wrote:
>> On Thu, Dec 17, 2015 at 6:57 AM, David Windsor <dave@progbits.org> wrote:
>>> NOTE: This is a v2 submission because patch 3/5 in v1 was too large to sent
>>> to kernel-hardening. Taking that as a sign that the patch needed to be split,
>>> I'm sending this version of the patchset, with the patches split more or less
>>> on a per-maintainer basis (except for those in drivers/).
>
> How's the next spin coming? It looks like we have some new real-world
> examples of exploits that would have been blocked by this protection:
>
> http://perception-point.io/2016/01/14/analysis-and-exploitation-of-a-linux-kernel-vulnerability-cve-2016-0728/
>
> :)
>
I'm currently working on v3 of this patchset: porting the patches to
linux-next and incorporating suggested changes to v2. I'm fairly well
along, but need just a bit more time I'm a bit busy at the moment, so
I expect realistically to have time to finish this at the beginning of
February.
Thanks,
David
> -Kees
>
> --
> Kees Cook
> Chrome OS & Brillo Security
^ permalink raw reply [flat|nested] 32+ messages in thread
* Re: [kernel-hardening][RFC PATCH v2 00/12] Add PAX_REFCOUNT overflow protection
2016-01-20 13:11 ` David Windsor
@ 2016-02-02 11:33 ` David Windsor
2016-02-04 18:19 ` Kees Cook
0 siblings, 1 reply; 32+ messages in thread
From: David Windsor @ 2016-02-02 11:33 UTC (permalink / raw)
To: Kees Cook, kernel-hardening
FYI, I now have time to work on this again.
Currently, I'm rebasing v2 atop linux-next. I've already incorporated
the following changes suggested during the on-list review of v2:
* s/PAX_REFCOUNT/STRICT_REFCOUNT
* Reordering the patchset in a more sane manner (per Greg KH)
* Creation of the "Kernel Hardening" menu in Kconfig
* Creation of per-architecture Kconfig option for opting in to STRICT_REFCOUNT
* Whitespace fixes
v3 is forthcoming and will be posted here as soon as I have the
patchset rebased to linux-next.
Thanks,
David
On Wed, Jan 20, 2016 at 8:11 AM, David Windsor <dave@progbits.org> wrote:
> Hi Kees,
>
> On Tue, Jan 19, 2016 at 2:07 PM, Kees Cook <keescook@chromium.org> wrote:
>> Hi David,
>>
>> On Thu, Dec 17, 2015 at 12:55 PM, Kees Cook <keescook@chromium.org> wrote:
>>> On Thu, Dec 17, 2015 at 6:57 AM, David Windsor <dave@progbits.org> wrote:
>>>> NOTE: This is a v2 submission because patch 3/5 in v1 was too large to sent
>>>> to kernel-hardening. Taking that as a sign that the patch needed to be split,
>>>> I'm sending this version of the patchset, with the patches split more or less
>>>> on a per-maintainer basis (except for those in drivers/).
>>
>> How's the next spin coming? It looks like we have some new real-world
>> examples of exploits that would have been blocked by this protection:
>>
>> http://perception-point.io/2016/01/14/analysis-and-exploitation-of-a-linux-kernel-vulnerability-cve-2016-0728/
>>
>> :)
>>
>
> I'm currently working on v3 of this patchset: porting the patches to
> linux-next and incorporating suggested changes to v2. I'm fairly well
> along, but need just a bit more time I'm a bit busy at the moment, so
> I expect realistically to have time to finish this at the beginning of
> February.
>
> Thanks,
> David
>
>> -Kees
>>
>> --
>> Kees Cook
>> Chrome OS & Brillo Security
^ permalink raw reply [flat|nested] 32+ messages in thread
* Re: [kernel-hardening][RFC PATCH v2 00/12] Add PAX_REFCOUNT overflow protection
2016-02-02 11:33 ` David Windsor
@ 2016-02-04 18:19 ` Kees Cook
2016-04-21 20:37 ` Kees Cook
0 siblings, 1 reply; 32+ messages in thread
From: Kees Cook @ 2016-02-04 18:19 UTC (permalink / raw)
To: David Windsor; +Cc: kernel-hardening@lists.openwall.com
On Tue, Feb 2, 2016 at 3:33 AM, David Windsor <dave@progbits.org> wrote:
> FYI, I now have time to work on this again.
>
> Currently, I'm rebasing v2 atop linux-next. I've already incorporated
> the following changes suggested during the on-list review of v2:
>
> * s/PAX_REFCOUNT/STRICT_REFCOUNT
> * Reordering the patchset in a more sane manner (per Greg KH)
> * Creation of the "Kernel Hardening" menu in Kconfig
> * Creation of per-architecture Kconfig option for opting in to STRICT_REFCOUNT
> * Whitespace fixes
>
> v3 is forthcoming and will be posted here as soon as I have the
> patchset rebased to linux-next.
Thanks for the update!
It may be helpful to mention in the changelog the two recent refcount
overflow bugs that would have been stopped by this mitigiation:
CVE-2014-2851 https://cyseclabs.com/page?n=02012016
CVE-2016-0728 http://perception-point.io/2016/01/14/analysis-and-exploitation-of-a-linux-kernel-vulnerability-cve-2016-0728/
-Kees
--
Kees Cook
Chrome OS & Brillo Security
^ permalink raw reply [flat|nested] 32+ messages in thread
* Re: [kernel-hardening][RFC PATCH v2 00/12] Add PAX_REFCOUNT overflow protection
2016-02-04 18:19 ` Kees Cook
@ 2016-04-21 20:37 ` Kees Cook
2016-04-28 12:16 ` [kernel-hardening][RFC PATCH 0/5] " David Windsor
0 siblings, 1 reply; 32+ messages in thread
From: Kees Cook @ 2016-04-21 20:37 UTC (permalink / raw)
To: David Windsor; +Cc: kernel-hardening@lists.openwall.com
Hi David,
On Thu, Feb 4, 2016 at 10:19 AM, Kees Cook <keescook@chromium.org> wrote:
> On Tue, Feb 2, 2016 at 3:33 AM, David Windsor <dave@progbits.org> wrote:
>> FYI, I now have time to work on this again.
>>
>> Currently, I'm rebasing v2 atop linux-next. I've already incorporated
>> the following changes suggested during the on-list review of v2:
>>
>> * s/PAX_REFCOUNT/STRICT_REFCOUNT
>> * Reordering the patchset in a more sane manner (per Greg KH)
>> * Creation of the "Kernel Hardening" menu in Kconfig
>> * Creation of per-architecture Kconfig option for opting in to STRICT_REFCOUNT
>> * Whitespace fixes
>>
>> v3 is forthcoming and will be posted here as soon as I have the
>> patchset rebased to linux-next.
>
> Thanks for the update!
>
> It may be helpful to mention in the changelog the two recent refcount
> overflow bugs that would have been stopped by this mitigiation:
>
> CVE-2014-2851 https://cyseclabs.com/page?n=02012016
> CVE-2016-0728 http://perception-point.io/2016/01/14/analysis-and-exploitation-of-a-linux-kernel-vulnerability-cve-2016-0728/
Any news on a v3 series? I'd love to see what you've got so far.
Thanks!
-Kees
--
Kees Cook
Chrome OS & Brillo Security
^ permalink raw reply [flat|nested] 32+ messages in thread
* Re: [kernel-hardening][RFC PATCH 0/5] Add PAX_REFCOUNT overflow protection
2016-04-21 20:37 ` Kees Cook
@ 2016-04-28 12:16 ` David Windsor
0 siblings, 0 replies; 32+ messages in thread
From: David Windsor @ 2016-04-28 12:16 UTC (permalink / raw)
To: Kees Cook; +Cc: kernel-hardening@lists.openwall.com
[-- Attachment #1: Type: text/plain, Size: 1617 bytes --]
Hi,
Unfortunately, I've had much less time to work on this than originally
anticipated and have gotten basically nothing done towards v3. I still
plan on working on it, just can't give you an ETA I can realistically
commit to.
Thanks,
David
On Thursday, April 21, 2016, Kees Cook <keescook@chromium.org> wrote:
> Hi David,
>
> On Thu, Feb 4, 2016 at 10:19 AM, Kees Cook <keescook@chromium.org
> <javascript:;>> wrote:
> > On Tue, Feb 2, 2016 at 3:33 AM, David Windsor <dave@progbits.org
> <javascript:;>> wrote:
> >> FYI, I now have time to work on this again.
> >>
> >> Currently, I'm rebasing v2 atop linux-next. I've already incorporated
> >> the following changes suggested during the on-list review of v2:
> >>
> >> * s/PAX_REFCOUNT/STRICT_REFCOUNT
> >> * Reordering the patchset in a more sane manner (per Greg KH)
> >> * Creation of the "Kernel Hardening" menu in Kconfig
> >> * Creation of per-architecture Kconfig option for opting in to
> STRICT_REFCOUNT
> >> * Whitespace fixes
> >>
> >> v3 is forthcoming and will be posted here as soon as I have the
> >> patchset rebased to linux-next.
> >
> > Thanks for the update!
> >
> > It may be helpful to mention in the changelog the two recent refcount
> > overflow bugs that would have been stopped by this mitigiation:
> >
> > CVE-2014-2851 https://cyseclabs.com/page?n=02012016
> > CVE-2016-0728
> http://perception-point.io/2016/01/14/analysis-and-exploitation-of-a-linux-kernel-vulnerability-cve-2016-0728/
>
> Any news on a v3 series? I'd love to see what you've got so far.
>
> Thanks!
>
> -Kees
>
> --
> Kees Cook
> Chrome OS & Brillo Security
>
[-- Attachment #2: Type: text/html, Size: 2494 bytes --]
^ permalink raw reply [flat|nested] 32+ messages in thread
* Re: [kernel-hardening][RFC PATCH v2 00/12] Add PAX_REFCOUNT overflow protection
2016-01-19 19:07 ` Kees Cook
2016-01-20 13:11 ` David Windsor
@ 2016-01-20 20:57 ` Yves-Alexis Perez
2016-01-20 23:39 ` Kees Cook
2016-01-21 1:01 ` Daniel Micay
1 sibling, 2 replies; 32+ messages in thread
From: Yves-Alexis Perez @ 2016-01-20 20:57 UTC (permalink / raw)
To: kernel-hardening, Kees Cook; +Cc: David Windsor
[-- Attachment #1: Type: text/plain, Size: 1332 bytes --]
On mar., 2016-01-19 at 11:07 -0800, Kees Cook wrote:
> Hi David,
>
> On Thu, Dec 17, 2015 at 12:55 PM, Kees Cook <keescook@chromium.org> wrote:
> > On Thu, Dec 17, 2015 at 6:57 AM, David Windsor <dave@progbits.org> wrote:
> > > NOTE: This is a v2 submission because patch 3/5 in v1 was too large to
> > > sent
> > > to kernel-hardening. Taking that as a sign that the patch needed to be
> > > split,
> > > I'm sending this version of the patchset, with the patches split more or
> > > less
> > > on a per-maintainer basis (except for those in drivers/).
>
> How's the next spin coming? It looks like we have some new real-world
> examples of exploits that would have been blocked by this protection:
>
> http://perception-point.io/2016/01/14/analysis-and-exploitation-of-a-linux-k
> ernel-vulnerability-cve-2016-0728/
>
>
One thing which is surprising (I have to admit I'm not really an expert on how
SLAB works) is how easy it apparently is to have multiple allocations end up
at the same place. You don't even have to *know* the exact address.
Wouldn't it be possible to at least have some randomization here, so new
object are not at the same place as the not-freed-ones, somehow preventing the
use-after-free and forcing an attacker to do some heap massaging?
Regards,
--
Yves-Alexis
[-- Attachment #2: This is a digitally signed message part --]
[-- Type: application/pgp-signature, Size: 473 bytes --]
^ permalink raw reply [flat|nested] 32+ messages in thread
* Re: [kernel-hardening][RFC PATCH v2 00/12] Add PAX_REFCOUNT overflow protection
2016-01-20 20:57 ` [kernel-hardening][RFC PATCH v2 00/12] " Yves-Alexis Perez
@ 2016-01-20 23:39 ` Kees Cook
2016-01-21 1:01 ` Daniel Micay
1 sibling, 0 replies; 32+ messages in thread
From: Kees Cook @ 2016-01-20 23:39 UTC (permalink / raw)
To: Yves-Alexis Perez; +Cc: kernel-hardening@lists.openwall.com, David Windsor
On Wed, Jan 20, 2016 at 12:57 PM, Yves-Alexis Perez <corsac@debian.org> wrote:
> On mar., 2016-01-19 at 11:07 -0800, Kees Cook wrote:
>> Hi David,
>>
>> On Thu, Dec 17, 2015 at 12:55 PM, Kees Cook <keescook@chromium.org> wrote:
>> > On Thu, Dec 17, 2015 at 6:57 AM, David Windsor <dave@progbits.org> wrote:
>> > > NOTE: This is a v2 submission because patch 3/5 in v1 was too large to
>> > > sent
>> > > to kernel-hardening. Taking that as a sign that the patch needed to be
>> > > split,
>> > > I'm sending this version of the patchset, with the patches split more or
>> > > less
>> > > on a per-maintainer basis (except for those in drivers/).
>>
>> How's the next spin coming? It looks like we have some new real-world
>> examples of exploits that would have been blocked by this protection:
>>
>> http://perception-point.io/2016/01/14/analysis-and-exploitation-of-a-linux-k
>> ernel-vulnerability-cve-2016-0728/
>>
>>
> One thing which is surprising (I have to admit I'm not really an expert on how
> SLAB works) is how easy it apparently is to have multiple allocations end up
> at the same place. You don't even have to *know* the exact address.
>
> Wouldn't it be possible to at least have some randomization here, so new
> object are not at the same place as the not-freed-ones, somehow preventing the
> use-after-free and forcing an attacker to do some heap massaging?
Yeah, it could be worth doing. Maybe Laura will want to take a stab at
this after the sanitization series is done?
-Kees
--
Kees Cook
Chrome OS & Brillo Security
^ permalink raw reply [flat|nested] 32+ messages in thread
* Re: [kernel-hardening][RFC PATCH v2 00/12] Add PAX_REFCOUNT overflow protection
2016-01-20 20:57 ` [kernel-hardening][RFC PATCH v2 00/12] " Yves-Alexis Perez
2016-01-20 23:39 ` Kees Cook
@ 2016-01-21 1:01 ` Daniel Micay
2016-01-21 12:45 ` Yves-Alexis Perez
1 sibling, 1 reply; 32+ messages in thread
From: Daniel Micay @ 2016-01-21 1:01 UTC (permalink / raw)
To: kernel-hardening, Kees Cook; +Cc: David Windsor
[-- Attachment #1: Type: text/plain, Size: 2669 bytes --]
On Wed, 2016-01-20 at 21:57 +0100, Yves-Alexis Perez wrote:
> On mar., 2016-01-19 at 11:07 -0800, Kees Cook wrote:
> > Hi David,
> >
> > On Thu, Dec 17, 2015 at 12:55 PM, Kees Cook <keescook@chromium.org>
> > wrote:
> > > On Thu, Dec 17, 2015 at 6:57 AM, David Windsor <dave@progbits.org>
> > > wrote:
> > > > NOTE: This is a v2 submission because patch 3/5 in v1 was too
> > > > large to
> > > > sent
> > > > to kernel-hardening. Taking that as a sign that the patch
> > > > needed to be
> > > > split,
> > > > I'm sending this version of the patchset, with the patches split
> > > > more or
> > > > less
> > > > on a per-maintainer basis (except for those in drivers/).
> >
> > How's the next spin coming? It looks like we have some new real-
> > world
> > examples of exploits that would have been blocked by this
> > protection:
> >
> > http://perception-point.io/2016/01/14/analysis-and-exploitation-of-a
> > -linux-k
> > ernel-vulnerability-cve-2016-0728/
> >
> >
> One thing which is surprising (I have to admit I'm not really an
> expert on how
> SLAB works) is how easy it apparently is to have multiple allocations
> end up
> at the same place. You don't even have to *know* the exact address.
>
> Wouldn't it be possible to at least have some randomization here, so
> new
> object are not at the same place as the not-freed-ones, somehow
> preventing the
> use-after-free and forcing an attacker to do some heap massaging?
>
> Regards,
It could do some fine-grained randomization. It does have a measurable
performance cost since it makes allocations colder and there isn't much
room for adding entropy but it does add up over time. Depending on the
locking design it could also hurt there too.
Take a look at OpenBSD malloc if you're interested in it. It has a few
forms of randomization and they do add up:
* small array of quarantined pointers, free(ptr) randomly swaps with one
and frees that (wastes some memory, makes allocations colder) - can also
have a feature checking that junk filled for sanitization is still in
place when moving out of the quarantine at a further performance cost
(which I recently upstreamed for OpenBSD malloc)
* picking a random slot within a slab (makes allocations colder and
slows down the allocator a bit)
* picking a random slab list to satisfy each allocation (significantly
increases worst-case memory usage by spreading stuff out)
* page cache randomization (i.e. randomization of the layer below slabs
which in userspace means the allocator cache and fine-grained mmap rnd,
but it still applies to the kernel's allocation of the slab pages)
[-- Attachment #2: This is a digitally signed message part --]
[-- Type: application/pgp-signature, Size: 819 bytes --]
^ permalink raw reply [flat|nested] 32+ messages in thread
* Re: [kernel-hardening][RFC PATCH v2 00/12] Add PAX_REFCOUNT overflow protection
2016-01-21 1:01 ` Daniel Micay
@ 2016-01-21 12:45 ` Yves-Alexis Perez
0 siblings, 0 replies; 32+ messages in thread
From: Yves-Alexis Perez @ 2016-01-21 12:45 UTC (permalink / raw)
To: kernel-hardening, Kees Cook; +Cc: David Windsor
[-- Attachment #1: Type: text/plain, Size: 1350 bytes --]
On mer., 2016-01-20 at 20:01 -0500, Daniel Micay wrote:
> It could do some fine-grained randomization. It does have a measurable
> performance cost since it makes allocations colder and there isn't much
> room for adding entropy but it does add up over time. Depending on the
> locking design it could also hurt there too.
>
> Take a look at OpenBSD malloc if you're interested in it. It has a few
> forms of randomization and they do add
I also checked in the PaX patch to see if there was something done to harden
slab, and while it's not randomisation, there's some hardening done as part of
PAX_USERCOPY (heap hardening for copy_{from,to}_user stuff).
The patch adds a GFP_USERCOPY/SLAB_USERCOPY flag for allocation, and then use
it for “tagging” allocations of pages which will end up in a
copy_{from,to}_user() call (as far as I can tell, it's a manual approach). The
tagged allocations are then using a separate slab cache from the kernel
itself, so it shouldn't be possible to exploit an UAF: the memory originating
from userland would not be dereferenced as a kernel allocation.
4.5 kernel will apparently include a SLAB_ACCOUNT flag which is use from
accounting. I wonder if it'd be possible to also use it to tag and isolate
memory allocated from different privilege levels.
Regards,
--
Yves-Alexis
[-- Attachment #2: This is a digitally signed message part --]
[-- Type: application/pgp-signature, Size: 473 bytes --]
^ permalink raw reply [flat|nested] 32+ messages in thread