add support for HTM lock elision for x86

add support for HTM lock elision for x86

[Roman Dementiev]

This series of patches adds methods that use hardware memory transactions (HTM)
on fast-path for DPDK locks (a.k.a. lock elision). Here the methods are implemented 
for x86 using Restricted Transactional Memory instructions (Intel(r) Transactional 
Synchronization Extensions). The implementation fall-backs to the normal DPDK lock
if HTM is not available or memory transactions fail.
This is not a replacement for lock usages since not all critical sections protected
by locks are friendly to HTM.

Roman Dementiev (3):
 spinlock: add support for HTM lock elision for x86 
 rwlock: add support for HTM lock elision for x86
 test scaling of HTM lock elision protecting rte_hash

 app/test/Makefile                                  |   1 +
 app/test/test_hash_scaling.c                       | 223 +++++++++++++++++++++
 lib/librte_eal/common/Makefile                     |   4 +-
 .../common/include/arch/ppc_64/rte_rwlock.h        |  38 ++++
 .../common/include/arch/ppc_64/rte_spinlock.h      |  41 ++++
 lib/librte_eal/common/include/arch/x86/rte_rtm.h   |  73 +++++++
 .../common/include/arch/x86/rte_rwlock.h           |  82 ++++++++
 .../common/include/arch/x86/rte_spinlock.h         | 107 ++++++++++
 lib/librte_eal/common/include/generic/rte_rwlock.h | 194 ++++++++++++++++++
 .../common/include/generic/rte_spinlock.h          |  75 +++++++
 lib/librte_eal/common/include/rte_rwlock.h         | 158 ---------------
 11 files changed, 836 insertions(+), 160 deletions(-)

Intel GmbH
Dornacher Strasse 1
85622 Feldkirchen/Muenchen, Deutschland
Sitz der Gesellschaft: Feldkirchen bei Muenchen
Geschaeftsfuehrer: Christian Lamprechter, Hannes Schwaderer, Douglas Lusk
Registergericht: Muenchen HRB 47456
Ust.-IdNr./VAT Registration No.: DE129385895
Citibank Frankfurt a.M. (BLZ 502 109 00) 600119052

[PATCH 1/3] spinlock: add support for HTM lock elision for x86

[Roman Dementiev]

This patch adds methods that use hardware memory transactions (HTM)
on fast-path for spinlocks (a.k.a. lock elision). Here the methods
are implemented for x86 using Restricted Transactional Memory
instructions (Intel(r) Transactional Synchronization Extensions).
The implementation fall-backs to the normal spinlock if HTM is not
available or memory transactions fail.
This is not a replacement for all spinlock usages since not all
critical sections protected by spinlocks are friendly to HTM.

Signed-off-by: Roman Dementiev <roman.dementiev@intel.com>
---
 .../common/include/arch/ppc_64/rte_spinlock.h      |  41 ++++++++
 lib/librte_eal/common/include/arch/x86/rte_rtm.h   |  73 ++++++++++++++
 .../common/include/arch/x86/rte_spinlock.h         | 107 +++++++++++++++++++++
 .../common/include/generic/rte_spinlock.h          |  75 +++++++++++++++
 4 files changed, 296 insertions(+)
 create mode 100644 lib/librte_eal/common/include/arch/x86/rte_rtm.h

diff --git a/lib/librte_eal/common/include/arch/ppc_64/rte_spinlock.h b/lib/librte_eal/common/include/arch/ppc_64/rte_spinlock.h
index cf8b81a..3336435 100644
--- a/lib/librte_eal/common/include/arch/ppc_64/rte_spinlock.h
+++ b/lib/librte_eal/common/include/arch/ppc_64/rte_spinlock.h
@@ -66,6 +66,47 @@ rte_spinlock_trylock(rte_spinlock_t *sl)
 
 #endif
 
+static inline int rte_tm_supported(void)
+{
+	return 0;
+}
+
+static inline void
+rte_spinlock_lock_tm(rte_spinlock_t *sl)
+{
+	rte_spinlock_lock(sl); /* fall-back */
+}
+
+static inline int
+rte_spinlock_trylock_tm(rte_spinlock_t *sl)
+{
+	return rte_spinlock_trylock(sl);
+}
+
+static inline void
+rte_spinlock_unlock_tm(rte_spinlock_t *sl)
+{
+	rte_spinlock_unlock(sl);
+}
+
+static inline void
+rte_spinlock_recursive_lock_tm(rte_spinlock_recursive_t *slr)
+{
+	rte_spinlock_recursive_lock(slr); /* fall-back */
+}
+
+static inline void
+rte_spinlock_recursive_unlock_tm(rte_spinlock_recursive_t *slr)
+{
+	rte_spinlock_recursive_unlock(slr);
+}
+
+static inline int
+rte_spinlock_recursive_trylock_tm(rte_spinlock_recursive_t *slr)
+{
+	return rte_spinlock_recursive_trylock(slr);
+}
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/librte_eal/common/include/arch/x86/rte_rtm.h b/lib/librte_eal/common/include/arch/x86/rte_rtm.h
new file mode 100644
index 0000000..d935641
--- /dev/null
+++ b/lib/librte_eal/common/include/arch/x86/rte_rtm.h
@@ -0,0 +1,73 @@
+#ifndef _RTE_RTM_H_
+#define _RTE_RTM_H_ 1
+
+/*
+ * Copyright (c) 2012,2013 Intel Corporation
+ * Author: Andi Kleen
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that: (1) source code distributions
+ * retain the above copyright notice and this paragraph in its entirety, (2)
+ * distributions including binary code include the above copyright notice and
+ * this paragraph in its entirety in the documentation or other materials
+ * provided with the distribution
+ *
+ * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.
+ */
+
+/* Official RTM intrinsics interface matching gcc/icc, but works
+   on older gcc compatible compilers and binutils. */
+
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+#define RTE_XBEGIN_STARTED		(~0u)
+#define RTE_XABORT_EXPLICIT		(1 << 0)
+#define RTE_XABORT_RETRY		(1 << 1)
+#define RTE_XABORT_CONFLICT		(1 << 2)
+#define RTE_XABORT_CAPACITY		(1 << 3)
+#define RTE_XABORT_DEBUG		(1 << 4)
+#define RTE_XABORT_NESTED		(1 << 5)
+#define RTE_XABORT_CODE(x)		(((x) >> 24) & 0xff)
+
+static __attribute__((__always_inline__)) inline
+unsigned int rte_xbegin(void)
+{
+	unsigned int ret = RTE_XBEGIN_STARTED;
+
+	asm volatile(".byte 0xc7,0xf8 ; .long 0" : "+a" (ret) :: "memory");
+	return ret;
+}
+
+static __attribute__((__always_inline__)) inline
+void rte_xend(void)
+{
+	 asm volatile(".byte 0x0f,0x01,0xd5" ::: "memory");
+}
+
+static __attribute__((__always_inline__)) inline
+void rte_xabort(const unsigned int status)
+{
+	asm volatile(".byte 0xc6,0xf8,%P0" :: "i" (status) : "memory");
+}
+
+static __attribute__((__always_inline__)) inline
+int rte_xtest(void)
+{
+	unsigned char out;
+
+	asm volatile(".byte 0x0f,0x01,0xd6 ; setnz %0" :
+		"=r" (out) :: "memory");
+	return out;
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_RTM_H_ */
diff --git a/lib/librte_eal/common/include/arch/x86/rte_spinlock.h b/lib/librte_eal/common/include/arch/x86/rte_spinlock.h
index 54fba95..136f25a 100644
--- a/lib/librte_eal/common/include/arch/x86/rte_spinlock.h
+++ b/lib/librte_eal/common/include/arch/x86/rte_spinlock.h
@@ -39,6 +39,13 @@ extern "C" {
 #endif
 
 #include "generic/rte_spinlock.h"
+#include "rte_rtm.h"
+#include "rte_cpuflags.h"
+#include "rte_branch_prediction.h"
+#include <rte_common.h>
+
+#define RTE_RTM_MAX_RETRIES (10)
+#define RTE_XABORT_LOCK_BUSY (0xff)
 
 #ifndef RTE_FORCE_INTRINSICS
 static inline void
@@ -87,6 +94,106 @@ rte_spinlock_trylock (rte_spinlock_t *sl)
 }
 #endif
 
+static uint8_t rtm_supported; /* cache the flag to avoid the overhead
+				 of the rte_cpu_get_flag_enabled function */
+
+static inline void __attribute__((constructor))
+rte_rtm_init(void)
+{
+	rtm_supported = rte_cpu_get_flag_enabled(RTE_CPUFLAG_RTM);
+}
+
+static inline int rte_tm_supported(void)
+{
+	return rtm_supported;
+}
+
+static inline int
+rte_try_tm(volatile int *lock)
+{
+	if (!rtm_supported)
+		return 0;
+
+	int retries = RTE_RTM_MAX_RETRIES;
+
+	while (likely(retries--)) {
+
+		unsigned int status = rte_xbegin();
+
+		if (likely(RTE_XBEGIN_STARTED == status)) {
+			if (unlikely(*lock))
+				rte_xabort(RTE_XABORT_LOCK_BUSY);
+			else
+				return 1;
+		}
+		while (*lock)
+			rte_pause();
+
+		if ((status & RTE_XABORT_EXPLICIT) &&
+			(RTE_XABORT_CODE(status) == RTE_XABORT_LOCK_BUSY))
+			continue;
+
+		if ((status & RTE_XABORT_RETRY) == 0) /* do not retry */
+			break;
+	}
+	return 0;
+}
+
+static inline void
+rte_spinlock_lock_tm(rte_spinlock_t *sl)
+{
+	if (likely(rte_try_tm(&sl->locked)))
+		return;
+
+	rte_spinlock_lock(sl); /* fall-back */
+}
+
+static inline int
+rte_spinlock_trylock_tm(rte_spinlock_t *sl)
+{
+	if (likely(rte_try_tm(&sl->locked)))
+		return 1;
+
+	return rte_spinlock_trylock(sl);
+}
+
+static inline void
+rte_spinlock_unlock_tm(rte_spinlock_t *sl)
+{
+	if (unlikely(sl->locked))
+		rte_spinlock_unlock(sl);
+	else
+		rte_xend();
+}
+
+static inline void
+rte_spinlock_recursive_lock_tm(rte_spinlock_recursive_t *slr)
+{
+	if (likely(rte_try_tm(&slr->sl.locked)))
+		return;
+
+	rte_spinlock_recursive_lock(slr); /* fall-back */
+}
+
+static inline void
+rte_spinlock_recursive_unlock_tm(rte_spinlock_recursive_t *slr)
+{
+	if (unlikely(slr->sl.locked))
+		rte_spinlock_recursive_unlock(slr);
+	else
+		rte_xend();
+}
+
+static inline int
+rte_spinlock_recursive_trylock_tm(rte_spinlock_recursive_t *slr)
+{
+	if (likely(rte_try_tm(&slr->sl.locked)))
+		return 1;
+
+	return rte_spinlock_recursive_trylock(slr);
+}
+
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/librte_eal/common/include/generic/rte_spinlock.h b/lib/librte_eal/common/include/generic/rte_spinlock.h
index c7fb0df..ddb79bf 100644
--- a/lib/librte_eal/common/include/generic/rte_spinlock.h
+++ b/lib/librte_eal/common/include/generic/rte_spinlock.h
@@ -145,6 +145,47 @@ static inline int rte_spinlock_is_locked (rte_spinlock_t *sl)
 }
 
 /**
+ * Test if hardware transactional memory (lock elision) is supported
+ *
+ * @return
+ *   1 if the hardware transactional memory is supported; 0 otherwise.
+ */
+static inline int rte_tm_supported(void);
+
+/**
+ * Try to execute critical section in a hardware memory transaction,
+ * if it fails or not available take the spinlock.
+ *
+ * @param sl
+ *   A pointer to the spinlock.
+ */
+static inline void
+rte_spinlock_lock_tm(rte_spinlock_t *sl);
+
+/**
+ * Commit hardware memory transaction or release the spinlock if
+ * the spinlock is used as a fall-back
+ *
+ * @param sl
+ *   A pointer to the spinlock.
+ */
+static inline void
+rte_spinlock_unlock_tm(rte_spinlock_t *sl);
+
+/**
+ * Try to execute critical section in a hardware memory transaction,
+ * if it fails or not available try to take the lock.
+ *
+ * @param sl
+ *   A pointer to the spinlock.
+ * @return
+ *   1 if the hardware memory transaction is successfully started
+ *   or lock is successfully taken; 0 otherwise.
+ */
+static inline int
+rte_spinlock_trylock_tm(rte_spinlock_t *sl);
+
+/**
  * The rte_spinlock_recursive_t type.
  */
 typedef struct {
@@ -223,4 +264,38 @@ static inline int rte_spinlock_recursive_trylock(rte_spinlock_recursive_t *slr)
 	return 1;
 }
 
+
+/**
+ * Try to execute critical section in a hardware memory transaction,
+ * if it fails or not available take the recursive spinlocks
+ *
+ * @param slr
+ *   A pointer to the recursive spinlock.
+ */
+static inline void rte_spinlock_recursive_lock_tm(
+	rte_spinlock_recursive_t *slr);
+
+/**
+ * Commit hardware memory transaction or release the recursive spinlock
+ * if the recursive spinlock is used as a fall-back
+ *
+ * @param slr
+ *   A pointer to the recursive spinlock.
+ */
+static inline void rte_spinlock_recursive_unlock_tm(
+	rte_spinlock_recursive_t *slr);
+
+/**
+ * Try to execute critical section in a hardware memory transaction,
+ * if it fails or not available try to take the recursive lock
+ *
+ * @param slr
+ *   A pointer to the recursive spinlock.
+ * @return
+ *   1 if the hardware memory transaction is successfully started
+ *   or lock is successfully taken; 0 otherwise.
+ */
+static inline int rte_spinlock_recursive_trylock_tm(
+	rte_spinlock_recursive_t *slr);
+
 #endif /* _RTE_SPINLOCK_H_ */
-- 
1.9.5.msysgit.0

Intel GmbH
Dornacher Strasse 1
85622 Feldkirchen/Muenchen, Deutschland
Sitz der Gesellschaft: Feldkirchen bei Muenchen
Geschaeftsfuehrer: Christian Lamprechter, Hannes Schwaderer, Douglas Lusk
Registergericht: Muenchen HRB 47456
Ust.-IdNr./VAT Registration No.: DE129385895
Citibank Frankfurt a.M. (BLZ 502 109 00) 600119052

[PATCH 2/3] rwlock: add support for HTM lock elision for x86

[Roman Dementiev]

This patch adds methods that use hardware memory transactions (HTM)
on fast-path for rwlock (a.k.a. lock elision). Here the methods
are implemented for x86 using Restricted Transactional Memory
instructions (Intel(r) Transactional Synchronization Extensions).
The implementation fall-backs to the normal rwlock if HTM is not
available or memory transactions fail. This is not a replacement
for all rwlock usages since not all critical sections protected
by locks are friendly to HTM.

Signed-off-by: Roman Dementiev <roman.dementiev@intel.com>
---
 lib/librte_eal/common/Makefile                     |   4 +-
 .../common/include/arch/ppc_64/rte_rwlock.h        |  38 ++++
 .../common/include/arch/x86/rte_rwlock.h           |  82 +++++++++
 lib/librte_eal/common/include/generic/rte_rwlock.h | 194 +++++++++++++++++++++
 lib/librte_eal/common/include/rte_rwlock.h         | 158 -----------------
 5 files changed, 316 insertions(+), 160 deletions(-)
 create mode 100644 lib/librte_eal/common/include/arch/ppc_64/rte_rwlock.h
 create mode 100644 lib/librte_eal/common/include/arch/x86/rte_rwlock.h
 create mode 100644 lib/librte_eal/common/include/generic/rte_rwlock.h
 delete mode 100644 lib/librte_eal/common/include/rte_rwlock.h

diff --git a/lib/librte_eal/common/Makefile b/lib/librte_eal/common/Makefile
index 3ea3bbf..38772d4 100644
--- a/lib/librte_eal/common/Makefile
+++ b/lib/librte_eal/common/Makefile
@@ -35,7 +35,7 @@ INC := rte_branch_prediction.h rte_common.h
 INC += rte_debug.h rte_eal.h rte_errno.h rte_launch.h rte_lcore.h
 INC += rte_log.h rte_memory.h rte_memzone.h rte_pci.h
 INC += rte_pci_dev_ids.h rte_per_lcore.h rte_random.h
-INC += rte_rwlock.h rte_tailq.h rte_interrupts.h rte_alarm.h
+INC += rte_tailq.h rte_interrupts.h rte_alarm.h
 INC += rte_string_fns.h rte_version.h
 INC += rte_eal_memconfig.h rte_malloc_heap.h
 INC += rte_hexdump.h rte_devargs.h rte_dev.h
@@ -46,7 +46,7 @@ INC += rte_warnings.h
 endif
 
 GENERIC_INC := rte_atomic.h rte_byteorder.h rte_cycles.h rte_prefetch.h
-GENERIC_INC += rte_spinlock.h rte_memcpy.h rte_cpuflags.h
+GENERIC_INC += rte_spinlock.h rte_memcpy.h rte_cpuflags.h rte_rwlock.h
 # defined in mk/arch/$(RTE_ARCH)/rte.vars.mk
 ARCH_DIR ?= $(RTE_ARCH)
 ARCH_INC := $(notdir $(wildcard $(RTE_SDK)/lib/librte_eal/common/include/arch/$(ARCH_DIR)/*.h))
diff --git a/lib/librte_eal/common/include/arch/ppc_64/rte_rwlock.h b/lib/librte_eal/common/include/arch/ppc_64/rte_rwlock.h
new file mode 100644
index 0000000..de8af19
--- /dev/null
+++ b/lib/librte_eal/common/include/arch/ppc_64/rte_rwlock.h
@@ -0,0 +1,38 @@
+#ifndef _RTE_RWLOCK_PPC_64_H_
+#define _RTE_RWLOCK_PPC_64_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "generic/rte_rwlock.h"
+
+static inline void
+rte_rwlock_read_lock_tm(rte_rwlock_t *rwl)
+{
+	rte_rwlock_read_lock(rwl);
+}
+
+static inline void
+rte_rwlock_read_unlock_tm(rte_rwlock_t *rwl)
+{
+	rte_rwlock_read_unlock(rwl);
+}
+
+static inline void
+rte_rwlock_write_lock_tm(rte_rwlock_t *rwl)
+{
+	rte_rwlock_write_lock(rwl);
+}
+
+static inline void
+rte_rwlock_write_unlock_tm(rte_rwlock_t *rwl)
+{
+	rte_rwlock_write_unlock(rwl);
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_RWLOCK_PPC_64_H_ */
diff --git a/lib/librte_eal/common/include/arch/x86/rte_rwlock.h b/lib/librte_eal/common/include/arch/x86/rte_rwlock.h
new file mode 100644
index 0000000..afd1c3c
--- /dev/null
+++ b/lib/librte_eal/common/include/arch/x86/rte_rwlock.h
@@ -0,0 +1,82 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2015 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _RTE_RWLOCK_X86_64_H_
+#define _RTE_RWLOCK_X86_64_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "generic/rte_rwlock.h"
+#include "rte_spinlock.h"
+
+static inline void
+rte_rwlock_read_lock_tm(rte_rwlock_t *rwl)
+{
+	if (likely(rte_try_tm(&rwl->cnt)))
+		return;
+	rte_rwlock_read_lock(rwl);
+}
+
+static inline void
+rte_rwlock_read_unlock_tm(rte_rwlock_t *rwl)
+{
+	if (unlikely(rwl->cnt))
+		rte_rwlock_read_unlock(rwl);
+	else
+		rte_xend();
+}
+
+static inline void
+rte_rwlock_write_lock_tm(rte_rwlock_t *rwl)
+{
+	if (likely(rte_try_tm(&rwl->cnt)))
+		return;
+	rte_rwlock_write_lock(rwl);
+}
+
+static inline void
+rte_rwlock_write_unlock_tm(rte_rwlock_t *rwl)
+{
+	if (unlikely(rwl->cnt))
+		rte_rwlock_write_unlock(rwl);
+	else
+		rte_xend();
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_RWLOCK_X86_64_H_ */
diff --git a/lib/librte_eal/common/include/generic/rte_rwlock.h b/lib/librte_eal/common/include/generic/rte_rwlock.h
new file mode 100644
index 0000000..85b0d3a
--- /dev/null
+++ b/lib/librte_eal/common/include/generic/rte_rwlock.h
@@ -0,0 +1,194 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _RTE_RWLOCK_H_
+#define _RTE_RWLOCK_H_
+
+/**
+ * @file
+ *
+ * RTE Read-Write Locks
+ *
+ * This file defines an API for read-write locks. The lock is used to
+ * protect data that allows multiple readers in parallel, but only
+ * one writer. All readers are blocked until the writer is finished
+ * writing.
+ *
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <rte_common.h>
+#include <rte_atomic.h>
+
+/**
+ * The rte_rwlock_t type.
+ *
+ * cnt is -1 when write lock is held, and > 0 when read locks are held.
+ */
+typedef struct {
+	volatile int32_t cnt; /**< -1 when W lock held, > 0 when R locks held. */
+} rte_rwlock_t;
+
+/**
+ * A static rwlock initializer.
+ */
+#define RTE_RWLOCK_INITIALIZER { 0 }
+
+/**
+ * Initialize the rwlock to an unlocked state.
+ *
+ * @param rwl
+ *   A pointer to the rwlock structure.
+ */
+static inline void
+rte_rwlock_init(rte_rwlock_t *rwl)
+{
+	rwl->cnt = 0;
+}
+
+/**
+ * Take a read lock. Loop until the lock is held.
+ *
+ * @param rwl
+ *   A pointer to a rwlock structure.
+ */
+static inline void
+rte_rwlock_read_lock(rte_rwlock_t *rwl)
+{
+	int32_t x;
+	int success = 0;
+
+	while (success == 0) {
+		x = rwl->cnt;
+		/* write lock is held */
+		if (x < 0) {
+			rte_pause();
+			continue;
+		}
+		success = rte_atomic32_cmpset((volatile uint32_t *)&rwl->cnt,
+					      x, x + 1);
+	}
+}
+
+/**
+ * Release a read lock.
+ *
+ * @param rwl
+ *   A pointer to the rwlock structure.
+ */
+static inline void
+rte_rwlock_read_unlock(rte_rwlock_t *rwl)
+{
+	rte_atomic32_dec((rte_atomic32_t *)(intptr_t)&rwl->cnt);
+}
+
+/**
+ * Take a write lock. Loop until the lock is held.
+ *
+ * @param rwl
+ *   A pointer to a rwlock structure.
+ */
+static inline void
+rte_rwlock_write_lock(rte_rwlock_t *rwl)
+{
+	int32_t x;
+	int success = 0;
+
+	while (success == 0) {
+		x = rwl->cnt;
+		/* a lock is held */
+		if (x != 0) {
+			rte_pause();
+			continue;
+		}
+		success = rte_atomic32_cmpset((volatile uint32_t *)&rwl->cnt,
+					      0, -1);
+	}
+}
+
+/**
+ * Release a write lock.
+ *
+ * @param rwl
+ *   A pointer to a rwlock structure.
+ */
+static inline void
+rte_rwlock_write_unlock(rte_rwlock_t *rwl)
+{
+	rte_atomic32_inc((rte_atomic32_t *)(intptr_t)&rwl->cnt);
+}
+
+/**
+ * Try to execute critical section in a hardware memory transaction, if it fails or not available take a read lock
+ *
+ * @param rwl
+ *   A pointer to a rwlock structure.
+ */
+static inline void
+rte_rwlock_read_lock_tm(rte_rwlock_t *rwl);
+
+/**
+ * Commit hardware memory transaction or release the read lock if the lock is used as a fall-back
+ *
+ * @param rwl
+ *   A pointer to the rwlock structure.
+ */
+static inline void
+rte_rwlock_read_unlock_tm(rte_rwlock_t *rwl);
+
+/**
+ * Try to execute critical section in a hardware memory transaction, if it fails or not available take a write lock
+ *
+ * @param rwl
+ *   A pointer to a rwlock structure.
+ */
+static inline void
+rte_rwlock_write_lock_tm(rte_rwlock_t *rwl);
+
+/**
+ * Commit hardware memory transaction or release the write lock if the lock is used as a fall-back
+ *
+ * @param rwl
+ *   A pointer to a rwlock structure.
+ */
+static inline void
+rte_rwlock_write_unlock_tm(rte_rwlock_t *rwl);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_RWLOCK_H_ */
diff --git a/lib/librte_eal/common/include/rte_rwlock.h b/lib/librte_eal/common/include/rte_rwlock.h
deleted file mode 100644
index 115731d..0000000
--- a/lib/librte_eal/common/include/rte_rwlock.h
+++ /dev/null
@@ -1,158 +0,0 @@
-/*-
- *   BSD LICENSE
- *
- *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
- *   All rights reserved.
- *
- *   Redistribution and use in source and binary forms, with or without
- *   modification, are permitted provided that the following conditions
- *   are met:
- *
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in
- *       the documentation and/or other materials provided with the
- *       distribution.
- *     * Neither the name of Intel Corporation nor the names of its
- *       contributors may be used to endorse or promote products derived
- *       from this software without specific prior written permission.
- *
- *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef _RTE_RWLOCK_H_
-#define _RTE_RWLOCK_H_
-
-/**
- * @file
- *
- * RTE Read-Write Locks
- *
- * This file defines an API for read-write locks. The lock is used to
- * protect data that allows multiple readers in parallel, but only
- * one writer. All readers are blocked until the writer is finished
- * writing.
- *
- */
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include <rte_common.h>
-#include <rte_atomic.h>
-
-/**
- * The rte_rwlock_t type.
- *
- * cnt is -1 when write lock is held, and > 0 when read locks are held.
- */
-typedef struct {
-	volatile int32_t cnt; /**< -1 when W lock held, > 0 when R locks held. */
-} rte_rwlock_t;
-
-/**
- * A static rwlock initializer.
- */
-#define RTE_RWLOCK_INITIALIZER { 0 }
-
-/**
- * Initialize the rwlock to an unlocked state.
- *
- * @param rwl
- *   A pointer to the rwlock structure.
- */
-static inline void
-rte_rwlock_init(rte_rwlock_t *rwl)
-{
-	rwl->cnt = 0;
-}
-
-/**
- * Take a read lock. Loop until the lock is held.
- *
- * @param rwl
- *   A pointer to a rwlock structure.
- */
-static inline void
-rte_rwlock_read_lock(rte_rwlock_t *rwl)
-{
-	int32_t x;
-	int success = 0;
-
-	while (success == 0) {
-		x = rwl->cnt;
-		/* write lock is held */
-		if (x < 0) {
-			rte_pause();
-			continue;
-		}
-		success = rte_atomic32_cmpset((volatile uint32_t *)&rwl->cnt,
-					      x, x + 1);
-	}
-}
-
-/**
- * Release a read lock.
- *
- * @param rwl
- *   A pointer to the rwlock structure.
- */
-static inline void
-rte_rwlock_read_unlock(rte_rwlock_t *rwl)
-{
-	rte_atomic32_dec((rte_atomic32_t *)(intptr_t)&rwl->cnt);
-}
-
-/**
- * Take a write lock. Loop until the lock is held.
- *
- * @param rwl
- *   A pointer to a rwlock structure.
- */
-static inline void
-rte_rwlock_write_lock(rte_rwlock_t *rwl)
-{
-	int32_t x;
-	int success = 0;
-
-	while (success == 0) {
-		x = rwl->cnt;
-		/* a lock is held */
-		if (x != 0) {
-			rte_pause();
-			continue;
-		}
-		success = rte_atomic32_cmpset((volatile uint32_t *)&rwl->cnt,
-					      0, -1);
-	}
-}
-
-/**
- * Release a write lock.
- *
- * @param rwl
- *   A pointer to a rwlock structure.
- */
-static inline void
-rte_rwlock_write_unlock(rte_rwlock_t *rwl)
-{
-	rte_atomic32_inc((rte_atomic32_t *)(intptr_t)&rwl->cnt);
-}
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _RTE_RWLOCK_H_ */
-- 
1.9.5.msysgit.0

Intel GmbH
Dornacher Strasse 1
85622 Feldkirchen/Muenchen, Deutschland
Sitz der Gesellschaft: Feldkirchen bei Muenchen
Geschaeftsfuehrer: Christian Lamprechter, Hannes Schwaderer, Douglas Lusk
Registergericht: Muenchen HRB 47456
Ust.-IdNr./VAT Registration No.: DE129385895
Citibank Frankfurt a.M. (BLZ 502 109 00) 600119052

[PATCH 3/3] test scaling of HTM lock elision protecting rte_hash

[Roman Dementiev]

This patch adds a new auto-test for testing the scaling
of concurrent inserts into rte_hash when protected by
the normal spinlock vs. the spinlock with HTM lock
elision. The test also benchmarks single-threaded
access without any locks.

Signed-off-by: Roman Dementiev <roman.dementiev@intel.com>
---
 app/test/Makefile            |   1 +
 app/test/test_hash_scaling.c | 223 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 224 insertions(+)
 create mode 100644 app/test/test_hash_scaling.c

diff --git a/app/test/Makefile b/app/test/Makefile
index 3c777bf..6ffe539 100644
--- a/app/test/Makefile
+++ b/app/test/Makefile
@@ -83,6 +83,7 @@ SRCS-y += test_memcpy_perf.c
 
 SRCS-$(CONFIG_RTE_LIBRTE_HASH) += test_hash.c
 SRCS-$(CONFIG_RTE_LIBRTE_HASH) += test_hash_perf.c
+SRCS-$(CONFIG_RTE_LIBRTE_HASH) += test_hash_scaling.c
 
 SRCS-$(CONFIG_RTE_LIBRTE_LPM) += test_lpm.c
 SRCS-$(CONFIG_RTE_LIBRTE_LPM) += test_lpm6.c
diff --git a/app/test/test_hash_scaling.c b/app/test/test_hash_scaling.c
new file mode 100644
index 0000000..682ae94
--- /dev/null
+++ b/app/test/test_hash_scaling.c
@@ -0,0 +1,223 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2015 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <rte_cycles.h>
+#include <rte_hash.h>
+#include <rte_hash_crc.h>
+#include <rte_spinlock.h>
+#include <rte_launch.h>
+
+#include "test.h"
+
+/*
+ * Check condition and return an error if true. Assumes that "handle" is the
+ * name of the hash structure pointer to be freed.
+ */
+#define RETURN_IF_ERROR(cond, str, ...) do {				\
+	if (cond) {							\
+		printf("ERROR line %d: " str "\n", __LINE__,		\
+							##__VA_ARGS__);	\
+		if (handle)						\
+			rte_hash_free(handle);				\
+		return -1;						\
+	}								\
+} while (0)
+
+enum locking_mode_t {
+	NORMAL_LOCK,
+	LOCK_ELISION,
+	NULL_LOCK
+};
+
+struct {
+	uint32_t num_iterations;
+	struct rte_hash *h;
+	rte_spinlock_t *lock;
+	int locking_mode;
+} tbl_scaling_test_params;
+
+static rte_atomic64_t gcycles;
+
+static int test_hash_scaling_worker(__attribute__((unused)) void *arg)
+{
+	uint64_t i, key;
+	uint32_t thr_id = rte_sys_gettid();
+	uint64_t begin, cycles = 0;
+
+	switch (tbl_scaling_test_params.locking_mode) {
+
+	case NORMAL_LOCK:
+
+		for (i = 0; i < tbl_scaling_test_params.num_iterations; i++) {
+			/*	different threads get different keys because
+				we use the thread-id in the key computation
+			 */
+			key = rte_hash_crc(&i, sizeof(i), thr_id);
+			begin = rte_rdtsc_precise();
+			rte_spinlock_lock(tbl_scaling_test_params.lock);
+			rte_hash_add_key(tbl_scaling_test_params.h, &key);
+			rte_spinlock_unlock(tbl_scaling_test_params.lock);
+			cycles += rte_rdtsc_precise() - begin;
+		}
+		break;
+
+	case LOCK_ELISION:
+
+		for (i = 0; i < tbl_scaling_test_params.num_iterations; i++) {
+			key = rte_hash_crc(&i, sizeof(i), thr_id);
+			begin = rte_rdtsc_precise();
+			rte_spinlock_lock_tm(tbl_scaling_test_params.lock);
+			rte_hash_add_key(tbl_scaling_test_params.h, &key);
+			rte_spinlock_unlock_tm(tbl_scaling_test_params.lock);
+			cycles += rte_rdtsc_precise() - begin;
+		}
+		break;
+
+	default:
+
+		for (i = 0; i < tbl_scaling_test_params.num_iterations; i++) {
+			key = rte_hash_crc(&i, sizeof(i), thr_id);
+			begin = rte_rdtsc_precise();
+			rte_hash_add_key(tbl_scaling_test_params.h, &key);
+			cycles += rte_rdtsc_precise() - begin;
+		}
+	}
+
+	rte_atomic64_add(&gcycles, cycles);
+
+	return 0;
+}
+
+/*
+ * Do scalability perf tests.
+ */
+static int
+test_hash_scaling(int locking_mode)
+{
+	static unsigned calledCount =    1;
+	uint32_t num_iterations = 1024*1024;
+	uint64_t i, key;
+	struct rte_hash_parameters hash_params = {
+		.entries = num_iterations*2,
+		.bucket_entries = 16,
+		.key_len = sizeof(key),
+		.hash_func = rte_hash_crc,
+		.hash_func_init_val = 0,
+		.socket_id = rte_socket_id(),
+	};
+	struct rte_hash *handle;
+	char name[RTE_HASH_NAMESIZE];
+	rte_spinlock_t lock;
+
+	rte_spinlock_init(&lock);
+
+	snprintf(name, 32, "test%u", calledCount++);
+	hash_params.name = name;
+
+	handle = rte_hash_create(&hash_params);
+	RETURN_IF_ERROR(handle == NULL, "hash creation failed");
+
+	tbl_scaling_test_params.num_iterations =
+		num_iterations/rte_lcore_count();
+	tbl_scaling_test_params.h = handle;
+	tbl_scaling_test_params.lock = &lock;
+	tbl_scaling_test_params.locking_mode = locking_mode;
+
+	rte_atomic64_init(&gcycles);
+	rte_atomic64_clear(&gcycles);
+
+	/* fill up to initial size */
+	for (i = 0; i < num_iterations; i++) {
+		key = rte_hash_crc(&i, sizeof(i), 0xabcdabcd);
+		rte_hash_add_key(tbl_scaling_test_params.h, &key);
+	}
+
+	rte_eal_mp_remote_launch(test_hash_scaling_worker, NULL, CALL_MASTER);
+	rte_eal_mp_wait_lcore();
+
+	unsigned long long int cycles_per_operation =
+		rte_atomic64_read(&gcycles)/
+		(tbl_scaling_test_params.num_iterations*rte_lcore_count());
+	const char *lock_name;
+
+	switch (locking_mode) {
+	case NORMAL_LOCK:
+		lock_name = "normal spinlock";
+		break;
+	case LOCK_ELISION:
+		lock_name = "lock elision";
+		break;
+	default:
+		lock_name = "null lock";
+	}
+	printf("--------------------------------------------------------\n");
+	printf("Cores: %d; %s mode ->  cycles per operation: %llu\n",
+		rte_lcore_count(), lock_name, cycles_per_operation);
+	printf("--------------------------------------------------------\n");
+	/* CSV output */
+	printf(">>>%d,%s,%llu\n", rte_lcore_count(), lock_name,
+		cycles_per_operation);
+
+	rte_hash_free(handle);
+	return 0;
+}
+
+static int
+test_hash_scaling_main(void)
+{
+	int r = 0;
+
+	if (rte_lcore_count() == 1)
+		r = test_hash_scaling(NULL_LOCK);
+
+	if (r == 0)
+		r = test_hash_scaling(NORMAL_LOCK);
+
+	if (!rte_tm_supported()) {
+		printf("Hardware transactional memory (lock elision) is NOT supported\n");
+		return r;
+	}
+	printf("Hardware transactional memory (lock elision) is supported\n");
+
+	if (r == 0)
+		r = test_hash_scaling(LOCK_ELISION);
+
+	return r;
+}
+
+
+static struct test_command hash_scaling_cmd = {
+	.command = "hash_scaling_autotest",
+	.callback = test_hash_scaling_main,
+};
+REGISTER_TEST_COMMAND(hash_scaling_cmd);
-- 
1.9.5.msysgit.0

Intel GmbH
Dornacher Strasse 1
85622 Feldkirchen/Muenchen, Deutschland
Sitz der Gesellschaft: Feldkirchen bei Muenchen
Geschaeftsfuehrer: Christian Lamprechter, Hannes Schwaderer, Douglas Lusk
Registergericht: Muenchen HRB 47456
Ust.-IdNr./VAT Registration No.: DE129385895
Citibank Frankfurt a.M. (BLZ 502 109 00) 600119052

Re: add support for HTM lock elision for x86

[Jay Rolette]

On Tue, Jun 2, 2015 at 8:11 AM, Roman Dementiev <roman.dementiev@intel.com>
wrote:

>
> This series of patches adds methods that use hardware memory transactions
> (HTM)
> on fast-path for DPDK locks (a.k.a. lock elision). Here the methods are
> implemented
> for x86 using Restricted Transactional Memory instructions (Intel(r)
> Transactional
> Synchronization Extensions). The implementation fall-backs to the normal
> DPDK lock
> if HTM is not available or memory transactions fail.
>

This is very interesting. Do you have any summary you can give us of what
the performance implications are from your test data?


> This is not a replacement for lock usages since not all critical sections
> protected
> by locks are friendly to HTM.
>

Any pointers to material that describe where HTM in its current incarnation
on x86 is approprate?


>
> Roman Dementiev (3):
>  spinlock: add support for HTM lock elision for x86
>  rwlock: add support for HTM lock elision for x86
>  test scaling of HTM lock elision protecting rte_hash
>
>  app/test/Makefile                                  |   1 +
>  app/test/test_hash_scaling.c                       | 223
> +++++++++++++++++++++
>  lib/librte_eal/common/Makefile                     |   4 +-
>  .../common/include/arch/ppc_64/rte_rwlock.h        |  38 ++++
>  .../common/include/arch/ppc_64/rte_spinlock.h      |  41 ++++
>  lib/librte_eal/common/include/arch/x86/rte_rtm.h   |  73 +++++++
>  .../common/include/arch/x86/rte_rwlock.h           |  82 ++++++++
>  .../common/include/arch/x86/rte_spinlock.h         | 107 ++++++++++
>  lib/librte_eal/common/include/generic/rte_rwlock.h | 194
> ++++++++++++++++++
>  .../common/include/generic/rte_spinlock.h          |  75 +++++++
>  lib/librte_eal/common/include/rte_rwlock.h         | 158 ---------------
>  11 files changed, 836 insertions(+), 160 deletions(-)
>

Thanks!
Jay

Re: add support for HTM lock elision for x86

[Dementiev, Roman]

Intel GmbH
Dornacher Strasse 1
85622 Feldkirchen/Muenchen, Deutschland
Sitz der Gesellschaft: Feldkirchen bei Muenchen
Geschaeftsfuehrer: Christian Lamprechter, Hannes Schwaderer, Douglas Lusk
Registergericht: Muenchen HRB 47456
Ust.-IdNr./VAT Registration No.: DE129385895
Citibank Frankfurt a.M. (BLZ 502 109 00) 600119052

Re: add support for HTM lock elision for x86

[Roman Dementiev]

Hello Jay,

Tuesday, June 2, 2015, 3:21:24 PM, you wrote:

> On Tue, Jun 2, 2015 at 8:11 AM, Roman Dementiev <roman.dementiev@intel.com>wrote:


>  This series of patches adds methods that use hardware memory transactions (HTM)
>  on fast-path for DPDK locks (a.k.a. lock elision). Here the methods are implemented
>  for x86 using Restricted Transactional Memory instructions (Intel(r) Transactional
>  Synchronization Extensions). The implementation fall-backs to the normal DPDK lock
>  if HTM is not available or memory transactions fail.


It provides a very good scaling given the protected data structure is friendly
to HTM. One example is rte_hash which is benchmarked in the unit test I have 
provided as the last patch.

There are some papers showing additional test results with similar lock 
elision implementations: www.intel.com/software/tsx


> This is very interesting. Do you have any summary you can give us
> of what the performance implications are from your test data?
>  
>  This is not a replacement for lock usages since not all critical sections protected
>  by locks are friendly to HTM.



> Any pointers to material that describe where HTM in its current incarnation on x86 is approprate?
>  
>


I meant to say: not a replacement to *ALL* lock usages. You can check the material here: www.intel.com/software/tsx
Please let me know if you need additional information.


>  Roman Dementiev (3):
>   spinlock: add support for HTM lock elision for x86
>   rwlock: add support for HTM lock elision for x86
>   test scaling of HTM lock elision protecting rte_hash
>  
>   app/test/Makefile                                  |   1 +
>   app/test/test_hash_scaling.c                       | 223 +++++++++++++++++++++
>   lib/librte_eal/common/Makefile                     |   4 +-
>   .../common/include/arch/ppc_64/rte_rwlock.h        |  38 ++++
>   .../common/include/arch/ppc_64/rte_spinlock.h      |  41 ++++
>   lib/librte_eal/common/include/arch/x86/rte_rtm.h   |  73 +++++++
>   .../common/include/arch/x86/rte_rwlock.h           |  82 ++++++++
>   .../common/include/arch/x86/rte_spinlock.h         | 107 ++++++++++
>   lib/librte_eal/common/include/generic/rte_rwlock.h | 194 ++++++++++++++++++
>   .../common/include/generic/rte_spinlock.h          |  75 +++++++
>   lib/librte_eal/common/include/rte_rwlock.h         | 158 ---------------
>   11 files changed, 836 insertions(+), 160 deletions(-)




> Thanks!
> Jay 



-- 
Best regards,
 Roman                            mailto:roman.dementiev@intel.com

Intel GmbH
Dornacher Strasse 1
85622 Feldkirchen/Muenchen, Deutschland
Sitz der Gesellschaft: Feldkirchen bei Muenchen
Geschaeftsfuehrer: Christian Lamprechter, Hannes Schwaderer, Douglas Lusk
Registergericht: Muenchen HRB 47456
Ust.-IdNr./VAT Registration No.: DE129385895
Citibank Frankfurt a.M. (BLZ 502 109 00) 600119052

Re: add support for HTM lock elision for x86

[Stephen Hemminger]

On Tue,  2 Jun 2015 15:11:30 +0200
Roman Dementiev <roman.dementiev@intel.com> wrote:

> 
> This series of patches adds methods that use hardware memory transactions (HTM)
> on fast-path for DPDK locks (a.k.a. lock elision). Here the methods are implemented 
> for x86 using Restricted Transactional Memory instructions (Intel(r) Transactional 
> Synchronization Extensions). The implementation fall-backs to the normal DPDK lock
> if HTM is not available or memory transactions fail.
> This is not a replacement for lock usages since not all critical sections protected
> by locks are friendly to HTM.
> 
> Roman Dementiev (3):
>  spinlock: add support for HTM lock elision for x86 
>  rwlock: add support for HTM lock elision for x86
>  test scaling of HTM lock elision protecting rte_hash
> 
>  app/test/Makefile                                  |   1 +
>  app/test/test_hash_scaling.c                       | 223 +++++++++++++++++++++
>  lib/librte_eal/common/Makefile                     |   4 +-
>  .../common/include/arch/ppc_64/rte_rwlock.h        |  38 ++++
>  .../common/include/arch/ppc_64/rte_spinlock.h      |  41 ++++
>  lib/librte_eal/common/include/arch/x86/rte_rtm.h   |  73 +++++++
>  .../common/include/arch/x86/rte_rwlock.h           |  82 ++++++++
>  .../common/include/arch/x86/rte_spinlock.h         | 107 ++++++++++
>  lib/librte_eal/common/include/generic/rte_rwlock.h | 194 ++++++++++++++++++
>  .../common/include/generic/rte_spinlock.h          |  75 +++++++
>  lib/librte_eal/common/include/rte_rwlock.h         | 158 ---------------
>  11 files changed, 836 insertions(+), 160 deletions(-)
> 
> Intel GmbH
> Dornacher Strasse 1
> 85622 Feldkirchen/Muenchen, Deutschland
> Sitz der Gesellschaft: Feldkirchen bei Muenchen
> Geschaeftsfuehrer: Christian Lamprechter, Hannes Schwaderer, Douglas Lusk
> Registergericht: Muenchen HRB 47456
> Ust.-IdNr./VAT Registration No.: DE129385895
> Citibank Frankfurt a.M. (BLZ 502 109 00) 600119052
> 

You probably want to put a caveat around this, it won't work for people
that expect to use spinlocks to protect I/O operations on hardware.
Since I/O operations aren't like memory.

Re: add support for HTM lock elision for x86

[Roman Dementiev]

Hello Stephen,

Wednesday, June 3, 2015, 8:40:14 PM, you wrote:

> On Tue,  2 Jun 2015 15:11:30 +0200
> Roman Dementiev <roman.dementiev@intel.com> wrote:

>> 
>> This series of patches adds methods that use hardware memory transactions (HTM)
>> on fast-path for DPDK locks (a.k.a. lock elision). Here the methods are implemented 
>> for x86 using Restricted Transactional Memory instructions (Intel(r) Transactional 
>> Synchronization Extensions). The implementation fall-backs to the normal DPDK lock
>> if HTM is not available or memory transactions fail.
>> This is not a replacement for all lock usages since not all critical sections protected
>> by locks are friendly to HTM.
>> 


> You probably want to put a caveat around this, it won't work for people
> that expect to use spinlocks to protect I/O operations on hardware.
> Since I/O operations aren't like memory.

yes, I/O  can not  be  rolled  back by the CPU if the transaction should fail. Thus
the  HTM  transaction  protecting I/O operations are always aborted by
CPU. In Intel TSX the I/O operations (MMIO, outp, etc) are TSX-unfriendly
causing immediate abort.

-- 
Best regards,
 Roman                            mailto:roman.dementiev@intel.com

Intel GmbH
Dornacher Strasse 1
85622 Feldkirchen/Muenchen, Deutschland
Sitz der Gesellschaft: Feldkirchen bei Muenchen
Geschaeftsfuehrer: Christian Lamprechter, Hannes Schwaderer, Douglas Lusk
Registergericht: Muenchen HRB 47456
Ust.-IdNr./VAT Registration No.: DE129385895
Citibank Frankfurt a.M. (BLZ 502 109 00) 600119052

[PATCH v2 0/3] add support for HTM lock elision for x86

[Roman Dementiev]

This series of patches adds methods that use hardware memory transactions (HTM)
on fast-path for DPDK locks (a.k.a. lock elision). Here the methods are
implemented for x86 using Restricted Transactional Memory instructions (Intel(r)
Transactional Synchronization Extensions). The implementation fall-backs to
the normal DPDK lock if HTM is not available or memory transactions fail. This
is not a replacement for ALL lock usages since not all critical sections
protected by locks are friendly to HTM. For example, an attempt to perform
a HW I/O operation inside a hardware memory transaction always aborts
the transaction since the CPU is not able to roll-back should the transaction
fail. Therefore, hardware transactional locks are not advised to be used around
rte_eth_rx_burst() and rte_eth_tx_burst() calls.

v2 changes
-added a documentation note about hardware limitations

Roman Dementiev (3):
  spinlock: add support for HTM lock elision for x86
  rwlock: add support for HTM lock elision for x86
  test scaling of HTM lock elision protecting rte_hash

 app/test/Makefile                                  |   1 +
 app/test/test_hash_scaling.c                       | 223 +++++++++++++++++++++
 lib/librte_eal/common/Makefile                     |   4 +-
 .../common/include/arch/ppc_64/rte_rwlock.h        |  38 ++++
 .../common/include/arch/ppc_64/rte_spinlock.h      |  41 ++++
 lib/librte_eal/common/include/arch/x86/rte_rtm.h   |  73 +++++++
 .../common/include/arch/x86/rte_rwlock.h           |  82 ++++++++
 .../common/include/arch/x86/rte_spinlock.h         | 107 ++++++++++
 lib/librte_eal/common/include/generic/rte_rwlock.h | 208 +++++++++++++++++++
 .../common/include/generic/rte_spinlock.h          |  99 +++++++++
 lib/librte_eal/common/include/rte_rwlock.h         | 158 ---------------
 11 files changed, 874 insertions(+), 160 deletions(-)
 create mode 100644 app/test/test_hash_scaling.c
 create mode 100644 lib/librte_eal/common/include/arch/ppc_64/rte_rwlock.h
 create mode 100644 lib/librte_eal/common/include/arch/x86/rte_rtm.h
 create mode 100644 lib/librte_eal/common/include/arch/x86/rte_rwlock.h
 create mode 100644 lib/librte_eal/common/include/generic/rte_rwlock.h
 delete mode 100644 lib/librte_eal/common/include/rte_rwlock.h

-- 
1.9.5.msysgit.0

Intel GmbH
Dornacher Strasse 1
85622 Feldkirchen/Muenchen, Deutschland
Sitz der Gesellschaft: Feldkirchen bei Muenchen
Geschaeftsfuehrer: Christian Lamprechter, Hannes Schwaderer, Douglas Lusk
Registergericht: Muenchen HRB 47456
Ust.-IdNr./VAT Registration No.: DE129385895
Citibank Frankfurt a.M. (BLZ 502 109 00) 600119052

[PATCH v2 1/3] spinlock: add support for HTM lock elision for x86

[Roman Dementiev]

This patch adds methods that use hardware memory transactions (HTM) on fast-path
for spinlocks (a.k.a. lock elision). Here the methods are implemented for x86
using Restricted Transactional Memory instructions (Intel(r) Transactional
Synchronization Extensions). The implementation fall-backs to the normal
spinlock if HTM is not available or memory transactions fail. This is not
a replacement for all spinlock usages since not all critical sections protected
by spinlocks are friendly to HTM. For example, an attempt to perform a HW I/O
operation inside a hardware memory transaction always aborts the transaction
since the CPU is not able to roll-back should the transaction fail.
Therefore, hardware transactional locks are not advised to be used around
rte_eth_rx_burst() and rte_eth_tx_burst() calls.

Signed-off-by: Roman Dementiev <roman.dementiev@intel.com>
---
 .../common/include/arch/ppc_64/rte_spinlock.h      |  41 ++++++++
 lib/librte_eal/common/include/arch/x86/rte_rtm.h   |  73 ++++++++++++++
 .../common/include/arch/x86/rte_spinlock.h         | 107 +++++++++++++++++++++
 .../common/include/generic/rte_spinlock.h          |  99 +++++++++++++++++++
 4 files changed, 320 insertions(+)
 create mode 100644 lib/librte_eal/common/include/arch/x86/rte_rtm.h

diff --git a/lib/librte_eal/common/include/arch/ppc_64/rte_spinlock.h b/lib/librte_eal/common/include/arch/ppc_64/rte_spinlock.h
index cf8b81a..3336435 100644
--- a/lib/librte_eal/common/include/arch/ppc_64/rte_spinlock.h
+++ b/lib/librte_eal/common/include/arch/ppc_64/rte_spinlock.h
@@ -66,6 +66,47 @@ rte_spinlock_trylock(rte_spinlock_t *sl)
 
 #endif
 
+static inline int rte_tm_supported(void)
+{
+	return 0;
+}
+
+static inline void
+rte_spinlock_lock_tm(rte_spinlock_t *sl)
+{
+	rte_spinlock_lock(sl); /* fall-back */
+}
+
+static inline int
+rte_spinlock_trylock_tm(rte_spinlock_t *sl)
+{
+	return rte_spinlock_trylock(sl);
+}
+
+static inline void
+rte_spinlock_unlock_tm(rte_spinlock_t *sl)
+{
+	rte_spinlock_unlock(sl);
+}
+
+static inline void
+rte_spinlock_recursive_lock_tm(rte_spinlock_recursive_t *slr)
+{
+	rte_spinlock_recursive_lock(slr); /* fall-back */
+}
+
+static inline void
+rte_spinlock_recursive_unlock_tm(rte_spinlock_recursive_t *slr)
+{
+	rte_spinlock_recursive_unlock(slr);
+}
+
+static inline int
+rte_spinlock_recursive_trylock_tm(rte_spinlock_recursive_t *slr)
+{
+	return rte_spinlock_recursive_trylock(slr);
+}
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/librte_eal/common/include/arch/x86/rte_rtm.h b/lib/librte_eal/common/include/arch/x86/rte_rtm.h
new file mode 100644
index 0000000..d935641
--- /dev/null
+++ b/lib/librte_eal/common/include/arch/x86/rte_rtm.h
@@ -0,0 +1,73 @@
+#ifndef _RTE_RTM_H_
+#define _RTE_RTM_H_ 1
+
+/*
+ * Copyright (c) 2012,2013 Intel Corporation
+ * Author: Andi Kleen
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that: (1) source code distributions
+ * retain the above copyright notice and this paragraph in its entirety, (2)
+ * distributions including binary code include the above copyright notice and
+ * this paragraph in its entirety in the documentation or other materials
+ * provided with the distribution
+ *
+ * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.
+ */
+
+/* Official RTM intrinsics interface matching gcc/icc, but works
+   on older gcc compatible compilers and binutils. */
+
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+#define RTE_XBEGIN_STARTED		(~0u)
+#define RTE_XABORT_EXPLICIT		(1 << 0)
+#define RTE_XABORT_RETRY		(1 << 1)
+#define RTE_XABORT_CONFLICT		(1 << 2)
+#define RTE_XABORT_CAPACITY		(1 << 3)
+#define RTE_XABORT_DEBUG		(1 << 4)
+#define RTE_XABORT_NESTED		(1 << 5)
+#define RTE_XABORT_CODE(x)		(((x) >> 24) & 0xff)
+
+static __attribute__((__always_inline__)) inline
+unsigned int rte_xbegin(void)
+{
+	unsigned int ret = RTE_XBEGIN_STARTED;
+
+	asm volatile(".byte 0xc7,0xf8 ; .long 0" : "+a" (ret) :: "memory");
+	return ret;
+}
+
+static __attribute__((__always_inline__)) inline
+void rte_xend(void)
+{
+	 asm volatile(".byte 0x0f,0x01,0xd5" ::: "memory");
+}
+
+static __attribute__((__always_inline__)) inline
+void rte_xabort(const unsigned int status)
+{
+	asm volatile(".byte 0xc6,0xf8,%P0" :: "i" (status) : "memory");
+}
+
+static __attribute__((__always_inline__)) inline
+int rte_xtest(void)
+{
+	unsigned char out;
+
+	asm volatile(".byte 0x0f,0x01,0xd6 ; setnz %0" :
+		"=r" (out) :: "memory");
+	return out;
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_RTM_H_ */
diff --git a/lib/librte_eal/common/include/arch/x86/rte_spinlock.h b/lib/librte_eal/common/include/arch/x86/rte_spinlock.h
index 54fba95..136f25a 100644
--- a/lib/librte_eal/common/include/arch/x86/rte_spinlock.h
+++ b/lib/librte_eal/common/include/arch/x86/rte_spinlock.h
@@ -39,6 +39,13 @@ extern "C" {
 #endif
 
 #include "generic/rte_spinlock.h"
+#include "rte_rtm.h"
+#include "rte_cpuflags.h"
+#include "rte_branch_prediction.h"
+#include <rte_common.h>
+
+#define RTE_RTM_MAX_RETRIES (10)
+#define RTE_XABORT_LOCK_BUSY (0xff)
 
 #ifndef RTE_FORCE_INTRINSICS
 static inline void
@@ -87,6 +94,106 @@ rte_spinlock_trylock (rte_spinlock_t *sl)
 }
 #endif
 
+static uint8_t rtm_supported; /* cache the flag to avoid the overhead
+				 of the rte_cpu_get_flag_enabled function */
+
+static inline void __attribute__((constructor))
+rte_rtm_init(void)
+{
+	rtm_supported = rte_cpu_get_flag_enabled(RTE_CPUFLAG_RTM);
+}
+
+static inline int rte_tm_supported(void)
+{
+	return rtm_supported;
+}
+
+static inline int
+rte_try_tm(volatile int *lock)
+{
+	if (!rtm_supported)
+		return 0;
+
+	int retries = RTE_RTM_MAX_RETRIES;
+
+	while (likely(retries--)) {
+
+		unsigned int status = rte_xbegin();
+
+		if (likely(RTE_XBEGIN_STARTED == status)) {
+			if (unlikely(*lock))
+				rte_xabort(RTE_XABORT_LOCK_BUSY);
+			else
+				return 1;
+		}
+		while (*lock)
+			rte_pause();
+
+		if ((status & RTE_XABORT_EXPLICIT) &&
+			(RTE_XABORT_CODE(status) == RTE_XABORT_LOCK_BUSY))
+			continue;
+
+		if ((status & RTE_XABORT_RETRY) == 0) /* do not retry */
+			break;
+	}
+	return 0;
+}
+
+static inline void
+rte_spinlock_lock_tm(rte_spinlock_t *sl)
+{
+	if (likely(rte_try_tm(&sl->locked)))
+		return;
+
+	rte_spinlock_lock(sl); /* fall-back */
+}
+
+static inline int
+rte_spinlock_trylock_tm(rte_spinlock_t *sl)
+{
+	if (likely(rte_try_tm(&sl->locked)))
+		return 1;
+
+	return rte_spinlock_trylock(sl);
+}
+
+static inline void
+rte_spinlock_unlock_tm(rte_spinlock_t *sl)
+{
+	if (unlikely(sl->locked))
+		rte_spinlock_unlock(sl);
+	else
+		rte_xend();
+}
+
+static inline void
+rte_spinlock_recursive_lock_tm(rte_spinlock_recursive_t *slr)
+{
+	if (likely(rte_try_tm(&slr->sl.locked)))
+		return;
+
+	rte_spinlock_recursive_lock(slr); /* fall-back */
+}
+
+static inline void
+rte_spinlock_recursive_unlock_tm(rte_spinlock_recursive_t *slr)
+{
+	if (unlikely(slr->sl.locked))
+		rte_spinlock_recursive_unlock(slr);
+	else
+		rte_xend();
+}
+
+static inline int
+rte_spinlock_recursive_trylock_tm(rte_spinlock_recursive_t *slr)
+{
+	if (likely(rte_try_tm(&slr->sl.locked)))
+		return 1;
+
+	return rte_spinlock_recursive_trylock(slr);
+}
+
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/librte_eal/common/include/generic/rte_spinlock.h b/lib/librte_eal/common/include/generic/rte_spinlock.h
index c7fb0df..4e0a3c3 100644
--- a/lib/librte_eal/common/include/generic/rte_spinlock.h
+++ b/lib/librte_eal/common/include/generic/rte_spinlock.h
@@ -145,6 +145,59 @@ static inline int rte_spinlock_is_locked (rte_spinlock_t *sl)
 }
 
 /**
+ * Test if hardware transactional memory (lock elision) is supported
+ *
+ * @return
+ *   1 if the hardware transactional memory is supported; 0 otherwise.
+ */
+static inline int rte_tm_supported(void);
+
+/**
+ * Try to execute critical section in a hardware memory transaction,
+ * if it fails or not available take the spinlock.
+ *
+ * NOTE: An attempt to perform a HW I/O operation inside a hardware memory
+ * transaction always aborts the transaction since the CPU is not able to
+ * roll-back should the transaction fail. Therefore, hardware transactional
+ * locks are not advised to be used around rte_eth_rx_burst() and
+ * rte_eth_tx_burst() calls.
+ *
+ * @param sl
+ *   A pointer to the spinlock.
+ */
+static inline void
+rte_spinlock_lock_tm(rte_spinlock_t *sl);
+
+/**
+ * Commit hardware memory transaction or release the spinlock if
+ * the spinlock is used as a fall-back
+ *
+ * @param sl
+ *   A pointer to the spinlock.
+ */
+static inline void
+rte_spinlock_unlock_tm(rte_spinlock_t *sl);
+
+/**
+ * Try to execute critical section in a hardware memory transaction,
+ * if it fails or not available try to take the lock.
+ *
+ * NOTE: An attempt to perform a HW I/O operation inside a hardware memory
+ * transaction always aborts the transaction since the CPU is not able to
+ * roll-back should the transaction fail. Therefore, hardware transactional
+ * locks are not advised to be used around rte_eth_rx_burst() and
+ * rte_eth_tx_burst() calls.
+ *
+ * @param sl
+ *   A pointer to the spinlock.
+ * @return
+ *   1 if the hardware memory transaction is successfully started
+ *   or lock is successfully taken; 0 otherwise.
+ */
+static inline int
+rte_spinlock_trylock_tm(rte_spinlock_t *sl);
+
+/**
  * The rte_spinlock_recursive_t type.
  */
 typedef struct {
@@ -223,4 +276,50 @@ static inline int rte_spinlock_recursive_trylock(rte_spinlock_recursive_t *slr)
 	return 1;
 }
 
+
+/**
+ * Try to execute critical section in a hardware memory transaction,
+ * if it fails or not available take the recursive spinlocks
+ *
+ * NOTE: An attempt to perform a HW I/O operation inside a hardware memory
+ * transaction always aborts the transaction since the CPU is not able to
+ * roll-back should the transaction fail. Therefore, hardware transactional
+ * locks are not advised to be used around rte_eth_rx_burst() and
+ * rte_eth_tx_burst() calls.
+ *
+ * @param slr
+ *   A pointer to the recursive spinlock.
+ */
+static inline void rte_spinlock_recursive_lock_tm(
+	rte_spinlock_recursive_t *slr);
+
+/**
+ * Commit hardware memory transaction or release the recursive spinlock
+ * if the recursive spinlock is used as a fall-back
+ *
+ * @param slr
+ *   A pointer to the recursive spinlock.
+ */
+static inline void rte_spinlock_recursive_unlock_tm(
+	rte_spinlock_recursive_t *slr);
+
+/**
+ * Try to execute critical section in a hardware memory transaction,
+ * if it fails or not available try to take the recursive lock
+ *
+ * NOTE: An attempt to perform a HW I/O operation inside a hardware memory
+ * transaction always aborts the transaction since the CPU is not able to
+ * roll-back should the transaction fail. Therefore, hardware transactional
+ * locks are not advised to be used around rte_eth_rx_burst() and
+ * rte_eth_tx_burst() calls.
+ *
+ * @param slr
+ *   A pointer to the recursive spinlock.
+ * @return
+ *   1 if the hardware memory transaction is successfully started
+ *   or lock is successfully taken; 0 otherwise.
+ */
+static inline int rte_spinlock_recursive_trylock_tm(
+	rte_spinlock_recursive_t *slr);
+
 #endif /* _RTE_SPINLOCK_H_ */
-- 
1.9.5.msysgit.0

Intel GmbH
Dornacher Strasse 1
85622 Feldkirchen/Muenchen, Deutschland
Sitz der Gesellschaft: Feldkirchen bei Muenchen
Geschaeftsfuehrer: Christian Lamprechter, Hannes Schwaderer, Douglas Lusk
Registergericht: Muenchen HRB 47456
Ust.-IdNr./VAT Registration No.: DE129385895
Citibank Frankfurt a.M. (BLZ 502 109 00) 600119052

Re: [PATCH v2 1/3] spinlock: add support for HTM lock elision for x86

[Thomas Monjalon]

2015-06-16 10:16, Roman Dementiev:
> --- a/lib/librte_eal/common/include/arch/x86/rte_spinlock.h
> +++ b/lib/librte_eal/common/include/arch/x86/rte_spinlock.h
> @@ -39,6 +39,13 @@ extern "C" {
>  #endif
>  
>  #include "generic/rte_spinlock.h"
> +#include "rte_rtm.h"
> +#include "rte_cpuflags.h"
> +#include "rte_branch_prediction.h"
> +#include <rte_common.h>

Why using angle brackets for rte_common.h?

Introducing rte_cpuflags.h in this header breaks the compilation of
the mlx4 pmd with CONFIG_RTE_LIBRTE_MLX4_DEBUG=y.
Indeed, it triggers the -pedantic flag which is not supported by rte_cpuflags.h.
Maybe it's time to fix this header?

Re: [PATCH v2 1/3] spinlock: add support for HTM lock elision for x86

[Bruce Richardson]

On Wed, Jun 17, 2015 at 11:29:49PM +0200, Thomas Monjalon wrote:
> 2015-06-16 10:16, Roman Dementiev:
> > --- a/lib/librte_eal/common/include/arch/x86/rte_spinlock.h
> > +++ b/lib/librte_eal/common/include/arch/x86/rte_spinlock.h
> > @@ -39,6 +39,13 @@ extern "C" {
> >  #endif
> >  
> >  #include "generic/rte_spinlock.h"
> > +#include "rte_rtm.h"
> > +#include "rte_cpuflags.h"
> > +#include "rte_branch_prediction.h"
> > +#include <rte_common.h>
> 
> Why using angle brackets for rte_common.h?
> 
> Introducing rte_cpuflags.h in this header breaks the compilation of
> the mlx4 pmd with CONFIG_RTE_LIBRTE_MLX4_DEBUG=y.
> Indeed, it triggers the -pedantic flag which is not supported by rte_cpuflags.h.
> Maybe it's time to fix this header?

Do all our headers need to support the pedantic C flag? I don't believe this
was a previous requirement for header files. The mlx4 driver appears to be the
only place in the dpdk.org codebase where the flag actually appears - and even
then the flag disabled in mlx.c where the dpdk headers are actually included.

  73 /* DPDK headers don't like -pedantic. */$
  74 #ifdef PEDANTIC$
  75 #pragma GCC diagnostic ignored "-pedantic"$
  76 #endif$
  77 #include <rte_config.h>$
	.....


I'm just not convinced that rte_cpuflags needs to be fixed at all here.

/Bruce

Re: [PATCH v2 1/3] spinlock: add support for HTM lock elision for x86

[Thomas Monjalon]

2015-06-18 11:00, Bruce Richardson:
> On Wed, Jun 17, 2015 at 11:29:49PM +0200, Thomas Monjalon wrote:
> > Introducing rte_cpuflags.h in this header breaks the compilation of
> > the mlx4 pmd with CONFIG_RTE_LIBRTE_MLX4_DEBUG=y.
> > Indeed, it triggers the -pedantic flag which is not supported by rte_cpuflags.h.
> > Maybe it's time to fix this header?
> 
> Do all our headers need to support the pedantic C flag? I don't believe this
> was a previous requirement for header files. The mlx4 driver appears to be the
> only place in the dpdk.org codebase where the flag actually appears - and even
> then the flag disabled in mlx.c where the dpdk headers are actually included.
> 
>   73 /* DPDK headers don't like -pedantic. */$
>   74 #ifdef PEDANTIC$
>   75 #pragma GCC diagnostic ignored "-pedantic"$
>   76 #endif$
>   77 #include <rte_config.h>$
> 	.....

You're right. It seems this disabling doesn't work.

> I'm just not convinced that rte_cpuflags needs to be fixed at all here.

Yes, it's probably simpler to remove the -pedantic flag.

Re: [PATCH v2 1/3] spinlock: add support for HTM lock elision for x86

[Adrien Mazarguil]

On Fri, Jun 19, 2015 at 03:35:38PM +0200, Thomas Monjalon wrote:
> 2015-06-18 11:00, Bruce Richardson:
> > On Wed, Jun 17, 2015 at 11:29:49PM +0200, Thomas Monjalon wrote:
> > > Introducing rte_cpuflags.h in this header breaks the compilation of
> > > the mlx4 pmd with CONFIG_RTE_LIBRTE_MLX4_DEBUG=y.
> > > Indeed, it triggers the -pedantic flag which is not supported by rte_cpuflags.h.
> > > Maybe it's time to fix this header?
> > 
> > Do all our headers need to support the pedantic C flag? I don't believe this
> > was a previous requirement for header files. The mlx4 driver appears to be the
> > only place in the dpdk.org codebase where the flag actually appears - and even
> > then the flag disabled in mlx.c where the dpdk headers are actually included.
> > 
> >   73 /* DPDK headers don't like -pedantic. */$
> >   74 #ifdef PEDANTIC$
> >   75 #pragma GCC diagnostic ignored "-pedantic"$
> >   76 #endif$
> >   77 #include <rte_config.h>$
> > 	.....
> 
> You're right. It seems this disabling doesn't work.

Well, it used to work, at least sufficiently until now.

The mlx4 driver started as an out of tree development long ago, this flag
is here from the beginning and was left around to maintain a clean code base
in the PMD itself. Unfortunately, it had to include a few headers that were
not quite ready to handle such constraints, hence the somewhat ugly #pragma
workarounds left until these headers could be fixed someday.

> > I'm just not convinced that rte_cpuflags needs to be fixed at all here.
> 
> Yes, it's probably simpler to remove the -pedantic flag.

I'm not going to argue against that, as a PMD's Makefile is obviously not
the right place to add a -pedantic parameter anyway.

However outside of PMD usage, I think public API headers (I'm not talking
about the entire DPDK code base, just headers) should handle all kind of
warnings a user application might throw at it for its own use (-pedantic and
other -Wstuff, I'd even say -std=c99 for strict ISO C compliance), as is the
case for the C library and most, if not all system-wide headers.

-- 
Adrien Mazarguil
6WIND

[PATCH] eal: fix cpu_feature_table[] compilation with -pedantic

[Adrien Mazarguil]

Since the commit below includes rte_cpuflags.h in rte_spinlock.h,
compilation of the mlx4 driver fails when CONFIG_RTE_LIBRTE_MLX4_DEBUG=y.

This mode adds -pedantic to the compiler's command line for mlx4, which
complains about the static definition of an empty cpu_feature_table[] in
common rte_cpuflags.h, then about its redefinition as a larger array in
arch-specific rte_cpuflags.h.

While DPDK does not officially support -pedantic internally, external
applications may enable it and include rte_spinlock.h from the public API.

Instead of removing -pedantic from mlx4, this commit fixes rte_cpuflags.h.

Fixes: ba7468997ea6 ("spinlock: add HTM lock elision for x86")

Signed-off-by: Adrien Mazarguil <adrien.mazarguil@6wind.com>
---
 lib/librte_eal/common/include/generic/rte_cpuflags.h | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/lib/librte_eal/common/include/generic/rte_cpuflags.h b/lib/librte_eal/common/include/generic/rte_cpuflags.h
index a04e021..61c4db1 100644
--- a/lib/librte_eal/common/include/generic/rte_cpuflags.h
+++ b/lib/librte_eal/common/include/generic/rte_cpuflags.h
@@ -74,8 +74,12 @@ struct feature_entry {
 
 /**
  * An array that holds feature entries
+ *
+ * Defined in arch-specific rte_cpuflags.h.
  */
+#ifdef __DOXYGEN__
 static const struct feature_entry cpu_feature_table[];
+#endif
 
 /**
  * Execute CPUID instruction and get contents of a specific register
-- 
2.1.0

Re: [PATCH] eal: fix cpu_feature_table[] compilation with -pedantic

[David Marchand]

On Mon, Jun 29, 2015 at 11:34 AM, Adrien Mazarguil <
adrien.mazarguil@6wind.com> wrote:

> Since the commit below includes rte_cpuflags.h in rte_spinlock.h,
> compilation of the mlx4 driver fails when CONFIG_RTE_LIBRTE_MLX4_DEBUG=y.
>
> This mode adds -pedantic to the compiler's command line for mlx4, which
> complains about the static definition of an empty cpu_feature_table[] in
> common rte_cpuflags.h, then about its redefinition as a larger array in
> arch-specific rte_cpuflags.h.
>
> While DPDK does not officially support -pedantic internally, external
> applications may enable it and include rte_spinlock.h from the public API.
>
> Instead of removing -pedantic from mlx4, this commit fixes rte_cpuflags.h.
>
> Fixes: ba7468997ea6 ("spinlock: add HTM lock elision for x86")
>
> Signed-off-by: Adrien Mazarguil <adrien.mazarguil@6wind.com>
>

Acked-by: David Marchand <david.marchand@6wind.com>

-- 
David Marchand

Re: [PATCH] eal: fix cpu_feature_table[] compilation with -pedantic

[Thomas Monjalon]

2015-06-29 14:10, David Marchand:
> On Mon, Jun 29, 2015 at 11:34 AM, Adrien Mazarguil <
> adrien.mazarguil@6wind.com> wrote:
> 
> > Since the commit below includes rte_cpuflags.h in rte_spinlock.h,
> > compilation of the mlx4 driver fails when CONFIG_RTE_LIBRTE_MLX4_DEBUG=y.
> >
> > This mode adds -pedantic to the compiler's command line for mlx4, which
> > complains about the static definition of an empty cpu_feature_table[] in
> > common rte_cpuflags.h, then about its redefinition as a larger array in
> > arch-specific rte_cpuflags.h.
> >
> > While DPDK does not officially support -pedantic internally, external
> > applications may enable it and include rte_spinlock.h from the public API.
> >
> > Instead of removing -pedantic from mlx4, this commit fixes rte_cpuflags.h.
> >
> > Fixes: ba7468997ea6 ("spinlock: add HTM lock elision for x86")
> >
> > Signed-off-by: Adrien Mazarguil <adrien.mazarguil@6wind.com>
> 
> Acked-by: David Marchand <david.marchand@6wind.com>

Applied, thanks

[PATCH v2 2/3] rwlock: add support for HTM lock elision for x86

[Roman Dementiev]

This patch adds methods that use hardware memory transactions (HTM) on
fast-path for rwlock (a.k.a. lock elision). Here the methods are implemented
for x86 using Restricted Transactional Memory instructions (Intel(r)
Transactional Synchronization Extensions). The implementation fall-backs to
the normal rwlock if HTM is not available or memory transactions fail. This is
not a replacement for all rwlock usages since not all critical sections
protected by locks are friendly to HTM. For example, an attempt to perform
a HW I/O operation inside a hardware memory transaction always aborts
the transaction since the CPU is not able to roll-back should the transaction
fail. Therefore, hardware transactional locks are not advised to be used around
rte_eth_rx_burst() and rte_eth_tx_burst() calls.

Signed-off-by: Roman Dementiev <roman.dementiev@intel.com>
---
 lib/librte_eal/common/Makefile                     |   4 +-
 .../common/include/arch/ppc_64/rte_rwlock.h        |  38 ++++
 .../common/include/arch/x86/rte_rwlock.h           |  82 ++++++++
 lib/librte_eal/common/include/generic/rte_rwlock.h | 208 +++++++++++++++++++++
 lib/librte_eal/common/include/rte_rwlock.h         | 158 ----------------
 5 files changed, 330 insertions(+), 160 deletions(-)
 create mode 100644 lib/librte_eal/common/include/arch/ppc_64/rte_rwlock.h
 create mode 100644 lib/librte_eal/common/include/arch/x86/rte_rwlock.h
 create mode 100644 lib/librte_eal/common/include/generic/rte_rwlock.h
 delete mode 100644 lib/librte_eal/common/include/rte_rwlock.h

diff --git a/lib/librte_eal/common/Makefile b/lib/librte_eal/common/Makefile
index 3ea3bbf..38772d4 100644
--- a/lib/librte_eal/common/Makefile
+++ b/lib/librte_eal/common/Makefile
@@ -35,7 +35,7 @@ INC := rte_branch_prediction.h rte_common.h
 INC += rte_debug.h rte_eal.h rte_errno.h rte_launch.h rte_lcore.h
 INC += rte_log.h rte_memory.h rte_memzone.h rte_pci.h
 INC += rte_pci_dev_ids.h rte_per_lcore.h rte_random.h
-INC += rte_rwlock.h rte_tailq.h rte_interrupts.h rte_alarm.h
+INC += rte_tailq.h rte_interrupts.h rte_alarm.h
 INC += rte_string_fns.h rte_version.h
 INC += rte_eal_memconfig.h rte_malloc_heap.h
 INC += rte_hexdump.h rte_devargs.h rte_dev.h
@@ -46,7 +46,7 @@ INC += rte_warnings.h
 endif
 
 GENERIC_INC := rte_atomic.h rte_byteorder.h rte_cycles.h rte_prefetch.h
-GENERIC_INC += rte_spinlock.h rte_memcpy.h rte_cpuflags.h
+GENERIC_INC += rte_spinlock.h rte_memcpy.h rte_cpuflags.h rte_rwlock.h
 # defined in mk/arch/$(RTE_ARCH)/rte.vars.mk
 ARCH_DIR ?= $(RTE_ARCH)
 ARCH_INC := $(notdir $(wildcard $(RTE_SDK)/lib/librte_eal/common/include/arch/$(ARCH_DIR)/*.h))
diff --git a/lib/librte_eal/common/include/arch/ppc_64/rte_rwlock.h b/lib/librte_eal/common/include/arch/ppc_64/rte_rwlock.h
new file mode 100644
index 0000000..de8af19
--- /dev/null
+++ b/lib/librte_eal/common/include/arch/ppc_64/rte_rwlock.h
@@ -0,0 +1,38 @@
+#ifndef _RTE_RWLOCK_PPC_64_H_
+#define _RTE_RWLOCK_PPC_64_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "generic/rte_rwlock.h"
+
+static inline void
+rte_rwlock_read_lock_tm(rte_rwlock_t *rwl)
+{
+	rte_rwlock_read_lock(rwl);
+}
+
+static inline void
+rte_rwlock_read_unlock_tm(rte_rwlock_t *rwl)
+{
+	rte_rwlock_read_unlock(rwl);
+}
+
+static inline void
+rte_rwlock_write_lock_tm(rte_rwlock_t *rwl)
+{
+	rte_rwlock_write_lock(rwl);
+}
+
+static inline void
+rte_rwlock_write_unlock_tm(rte_rwlock_t *rwl)
+{
+	rte_rwlock_write_unlock(rwl);
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_RWLOCK_PPC_64_H_ */
diff --git a/lib/librte_eal/common/include/arch/x86/rte_rwlock.h b/lib/librte_eal/common/include/arch/x86/rte_rwlock.h
new file mode 100644
index 0000000..afd1c3c
--- /dev/null
+++ b/lib/librte_eal/common/include/arch/x86/rte_rwlock.h
@@ -0,0 +1,82 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2015 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _RTE_RWLOCK_X86_64_H_
+#define _RTE_RWLOCK_X86_64_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "generic/rte_rwlock.h"
+#include "rte_spinlock.h"
+
+static inline void
+rte_rwlock_read_lock_tm(rte_rwlock_t *rwl)
+{
+	if (likely(rte_try_tm(&rwl->cnt)))
+		return;
+	rte_rwlock_read_lock(rwl);
+}
+
+static inline void
+rte_rwlock_read_unlock_tm(rte_rwlock_t *rwl)
+{
+	if (unlikely(rwl->cnt))
+		rte_rwlock_read_unlock(rwl);
+	else
+		rte_xend();
+}
+
+static inline void
+rte_rwlock_write_lock_tm(rte_rwlock_t *rwl)
+{
+	if (likely(rte_try_tm(&rwl->cnt)))
+		return;
+	rte_rwlock_write_lock(rwl);
+}
+
+static inline void
+rte_rwlock_write_unlock_tm(rte_rwlock_t *rwl)
+{
+	if (unlikely(rwl->cnt))
+		rte_rwlock_write_unlock(rwl);
+	else
+		rte_xend();
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_RWLOCK_X86_64_H_ */
diff --git a/lib/librte_eal/common/include/generic/rte_rwlock.h b/lib/librte_eal/common/include/generic/rte_rwlock.h
new file mode 100644
index 0000000..7a0fdc5
--- /dev/null
+++ b/lib/librte_eal/common/include/generic/rte_rwlock.h
@@ -0,0 +1,208 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _RTE_RWLOCK_H_
+#define _RTE_RWLOCK_H_
+
+/**
+ * @file
+ *
+ * RTE Read-Write Locks
+ *
+ * This file defines an API for read-write locks. The lock is used to
+ * protect data that allows multiple readers in parallel, but only
+ * one writer. All readers are blocked until the writer is finished
+ * writing.
+ *
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <rte_common.h>
+#include <rte_atomic.h>
+
+/**
+ * The rte_rwlock_t type.
+ *
+ * cnt is -1 when write lock is held, and > 0 when read locks are held.
+ */
+typedef struct {
+	volatile int32_t cnt; /**< -1 when W lock held, > 0 when R locks held. */
+} rte_rwlock_t;
+
+/**
+ * A static rwlock initializer.
+ */
+#define RTE_RWLOCK_INITIALIZER { 0 }
+
+/**
+ * Initialize the rwlock to an unlocked state.
+ *
+ * @param rwl
+ *   A pointer to the rwlock structure.
+ */
+static inline void
+rte_rwlock_init(rte_rwlock_t *rwl)
+{
+	rwl->cnt = 0;
+}
+
+/**
+ * Take a read lock. Loop until the lock is held.
+ *
+ * @param rwl
+ *   A pointer to a rwlock structure.
+ */
+static inline void
+rte_rwlock_read_lock(rte_rwlock_t *rwl)
+{
+	int32_t x;
+	int success = 0;
+
+	while (success == 0) {
+		x = rwl->cnt;
+		/* write lock is held */
+		if (x < 0) {
+			rte_pause();
+			continue;
+		}
+		success = rte_atomic32_cmpset((volatile uint32_t *)&rwl->cnt,
+					      x, x + 1);
+	}
+}
+
+/**
+ * Release a read lock.
+ *
+ * @param rwl
+ *   A pointer to the rwlock structure.
+ */
+static inline void
+rte_rwlock_read_unlock(rte_rwlock_t *rwl)
+{
+	rte_atomic32_dec((rte_atomic32_t *)(intptr_t)&rwl->cnt);
+}
+
+/**
+ * Take a write lock. Loop until the lock is held.
+ *
+ * @param rwl
+ *   A pointer to a rwlock structure.
+ */
+static inline void
+rte_rwlock_write_lock(rte_rwlock_t *rwl)
+{
+	int32_t x;
+	int success = 0;
+
+	while (success == 0) {
+		x = rwl->cnt;
+		/* a lock is held */
+		if (x != 0) {
+			rte_pause();
+			continue;
+		}
+		success = rte_atomic32_cmpset((volatile uint32_t *)&rwl->cnt,
+					      0, -1);
+	}
+}
+
+/**
+ * Release a write lock.
+ *
+ * @param rwl
+ *   A pointer to a rwlock structure.
+ */
+static inline void
+rte_rwlock_write_unlock(rte_rwlock_t *rwl)
+{
+	rte_atomic32_inc((rte_atomic32_t *)(intptr_t)&rwl->cnt);
+}
+
+/**
+ * Try to execute critical section in a hardware memory transaction, if it
+ * fails or not available take a read lock
+ *
+ * NOTE: An attempt to perform a HW I/O operation inside a hardware memory
+ * transaction always aborts the transaction since the CPU is not able to
+ * roll-back should the transaction fail. Therefore, hardware transactional
+ * locks are not advised to be used around rte_eth_rx_burst() and
+ * rte_eth_tx_burst() calls.
+ *
+ * @param rwl
+ *   A pointer to a rwlock structure.
+ */
+static inline void
+rte_rwlock_read_lock_tm(rte_rwlock_t *rwl);
+
+/**
+ * Commit hardware memory transaction or release the read lock if the lock is used as a fall-back
+ *
+ * @param rwl
+ *   A pointer to the rwlock structure.
+ */
+static inline void
+rte_rwlock_read_unlock_tm(rte_rwlock_t *rwl);
+
+/**
+ * Try to execute critical section in a hardware memory transaction, if it
+ * fails or not available take a write lock
+ *
+ * NOTE: An attempt to perform a HW I/O operation inside a hardware memory
+ * transaction always aborts the transaction since the CPU is not able to
+ * roll-back should the transaction fail. Therefore, hardware transactional
+ * locks are not advised to be used around rte_eth_rx_burst() and
+ * rte_eth_tx_burst() calls.
+ *
+ * @param rwl
+ *   A pointer to a rwlock structure.
+ */
+static inline void
+rte_rwlock_write_lock_tm(rte_rwlock_t *rwl);
+
+/**
+ * Commit hardware memory transaction or release the write lock if the lock is used as a fall-back
+ *
+ * @param rwl
+ *   A pointer to a rwlock structure.
+ */
+static inline void
+rte_rwlock_write_unlock_tm(rte_rwlock_t *rwl);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_RWLOCK_H_ */
diff --git a/lib/librte_eal/common/include/rte_rwlock.h b/lib/librte_eal/common/include/rte_rwlock.h
deleted file mode 100644
index 115731d..0000000
--- a/lib/librte_eal/common/include/rte_rwlock.h
+++ /dev/null
@@ -1,158 +0,0 @@
-/*-
- *   BSD LICENSE
- *
- *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
- *   All rights reserved.
- *
- *   Redistribution and use in source and binary forms, with or without
- *   modification, are permitted provided that the following conditions
- *   are met:
- *
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in
- *       the documentation and/or other materials provided with the
- *       distribution.
- *     * Neither the name of Intel Corporation nor the names of its
- *       contributors may be used to endorse or promote products derived
- *       from this software without specific prior written permission.
- *
- *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef _RTE_RWLOCK_H_
-#define _RTE_RWLOCK_H_
-
-/**
- * @file
- *
- * RTE Read-Write Locks
- *
- * This file defines an API for read-write locks. The lock is used to
- * protect data that allows multiple readers in parallel, but only
- * one writer. All readers are blocked until the writer is finished
- * writing.
- *
- */
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include <rte_common.h>
-#include <rte_atomic.h>
-
-/**
- * The rte_rwlock_t type.
- *
- * cnt is -1 when write lock is held, and > 0 when read locks are held.
- */
-typedef struct {
-	volatile int32_t cnt; /**< -1 when W lock held, > 0 when R locks held. */
-} rte_rwlock_t;
-
-/**
- * A static rwlock initializer.
- */
-#define RTE_RWLOCK_INITIALIZER { 0 }
-
-/**
- * Initialize the rwlock to an unlocked state.
- *
- * @param rwl
- *   A pointer to the rwlock structure.
- */
-static inline void
-rte_rwlock_init(rte_rwlock_t *rwl)
-{
-	rwl->cnt = 0;
-}
-
-/**
- * Take a read lock. Loop until the lock is held.
- *
- * @param rwl
- *   A pointer to a rwlock structure.
- */
-static inline void
-rte_rwlock_read_lock(rte_rwlock_t *rwl)
-{
-	int32_t x;
-	int success = 0;
-
-	while (success == 0) {
-		x = rwl->cnt;
-		/* write lock is held */
-		if (x < 0) {
-			rte_pause();
-			continue;
-		}
-		success = rte_atomic32_cmpset((volatile uint32_t *)&rwl->cnt,
-					      x, x + 1);
-	}
-}
-
-/**
- * Release a read lock.
- *
- * @param rwl
- *   A pointer to the rwlock structure.
- */
-static inline void
-rte_rwlock_read_unlock(rte_rwlock_t *rwl)
-{
-	rte_atomic32_dec((rte_atomic32_t *)(intptr_t)&rwl->cnt);
-}
-
-/**
- * Take a write lock. Loop until the lock is held.
- *
- * @param rwl
- *   A pointer to a rwlock structure.
- */
-static inline void
-rte_rwlock_write_lock(rte_rwlock_t *rwl)
-{
-	int32_t x;
-	int success = 0;
-
-	while (success == 0) {
-		x = rwl->cnt;
-		/* a lock is held */
-		if (x != 0) {
-			rte_pause();
-			continue;
-		}
-		success = rte_atomic32_cmpset((volatile uint32_t *)&rwl->cnt,
-					      0, -1);
-	}
-}
-
-/**
- * Release a write lock.
- *
- * @param rwl
- *   A pointer to a rwlock structure.
- */
-static inline void
-rte_rwlock_write_unlock(rte_rwlock_t *rwl)
-{
-	rte_atomic32_inc((rte_atomic32_t *)(intptr_t)&rwl->cnt);
-}
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _RTE_RWLOCK_H_ */
-- 
1.9.5.msysgit.0

Intel GmbH
Dornacher Strasse 1
85622 Feldkirchen/Muenchen, Deutschland
Sitz der Gesellschaft: Feldkirchen bei Muenchen
Geschaeftsfuehrer: Christian Lamprechter, Hannes Schwaderer, Douglas Lusk
Registergericht: Muenchen HRB 47456
Ust.-IdNr./VAT Registration No.: DE129385895
Citibank Frankfurt a.M. (BLZ 502 109 00) 600119052

[PATCH v2 3/3] test scaling of HTM lock elision protecting rte_hash

[Roman Dementiev]

This patch adds a new auto-test for testing the scaling
of concurrent inserts into rte_hash when protected by
the normal spinlock vs. the spinlock with HTM lock
elision. The test also benchmarks single-threaded
access without any locks.

Signed-off-by: Roman Dementiev <roman.dementiev@intel.com>
---
 app/test/Makefile            |   1 +
 app/test/test_hash_scaling.c | 223 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 224 insertions(+)
 create mode 100644 app/test/test_hash_scaling.c

diff --git a/app/test/Makefile b/app/test/Makefile
index 3c777bf..6ffe539 100644
--- a/app/test/Makefile
+++ b/app/test/Makefile
@@ -83,6 +83,7 @@ SRCS-y += test_memcpy_perf.c
 
 SRCS-$(CONFIG_RTE_LIBRTE_HASH) += test_hash.c
 SRCS-$(CONFIG_RTE_LIBRTE_HASH) += test_hash_perf.c
+SRCS-$(CONFIG_RTE_LIBRTE_HASH) += test_hash_scaling.c
 
 SRCS-$(CONFIG_RTE_LIBRTE_LPM) += test_lpm.c
 SRCS-$(CONFIG_RTE_LIBRTE_LPM) += test_lpm6.c
diff --git a/app/test/test_hash_scaling.c b/app/test/test_hash_scaling.c
new file mode 100644
index 0000000..682ae94
--- /dev/null
+++ b/app/test/test_hash_scaling.c
@@ -0,0 +1,223 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2015 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <rte_cycles.h>
+#include <rte_hash.h>
+#include <rte_hash_crc.h>
+#include <rte_spinlock.h>
+#include <rte_launch.h>
+
+#include "test.h"
+
+/*
+ * Check condition and return an error if true. Assumes that "handle" is the
+ * name of the hash structure pointer to be freed.
+ */
+#define RETURN_IF_ERROR(cond, str, ...) do {				\
+	if (cond) {							\
+		printf("ERROR line %d: " str "\n", __LINE__,		\
+							##__VA_ARGS__);	\
+		if (handle)						\
+			rte_hash_free(handle);				\
+		return -1;						\
+	}								\
+} while (0)
+
+enum locking_mode_t {
+	NORMAL_LOCK,
+	LOCK_ELISION,
+	NULL_LOCK
+};
+
+struct {
+	uint32_t num_iterations;
+	struct rte_hash *h;
+	rte_spinlock_t *lock;
+	int locking_mode;
+} tbl_scaling_test_params;
+
+static rte_atomic64_t gcycles;
+
+static int test_hash_scaling_worker(__attribute__((unused)) void *arg)
+{
+	uint64_t i, key;
+	uint32_t thr_id = rte_sys_gettid();
+	uint64_t begin, cycles = 0;
+
+	switch (tbl_scaling_test_params.locking_mode) {
+
+	case NORMAL_LOCK:
+
+		for (i = 0; i < tbl_scaling_test_params.num_iterations; i++) {
+			/*	different threads get different keys because
+				we use the thread-id in the key computation
+			 */
+			key = rte_hash_crc(&i, sizeof(i), thr_id);
+			begin = rte_rdtsc_precise();
+			rte_spinlock_lock(tbl_scaling_test_params.lock);
+			rte_hash_add_key(tbl_scaling_test_params.h, &key);
+			rte_spinlock_unlock(tbl_scaling_test_params.lock);
+			cycles += rte_rdtsc_precise() - begin;
+		}
+		break;
+
+	case LOCK_ELISION:
+
+		for (i = 0; i < tbl_scaling_test_params.num_iterations; i++) {
+			key = rte_hash_crc(&i, sizeof(i), thr_id);
+			begin = rte_rdtsc_precise();
+			rte_spinlock_lock_tm(tbl_scaling_test_params.lock);
+			rte_hash_add_key(tbl_scaling_test_params.h, &key);
+			rte_spinlock_unlock_tm(tbl_scaling_test_params.lock);
+			cycles += rte_rdtsc_precise() - begin;
+		}
+		break;
+
+	default:
+
+		for (i = 0; i < tbl_scaling_test_params.num_iterations; i++) {
+			key = rte_hash_crc(&i, sizeof(i), thr_id);
+			begin = rte_rdtsc_precise();
+			rte_hash_add_key(tbl_scaling_test_params.h, &key);
+			cycles += rte_rdtsc_precise() - begin;
+		}
+	}
+
+	rte_atomic64_add(&gcycles, cycles);
+
+	return 0;
+}
+
+/*
+ * Do scalability perf tests.
+ */
+static int
+test_hash_scaling(int locking_mode)
+{
+	static unsigned calledCount =    1;
+	uint32_t num_iterations = 1024*1024;
+	uint64_t i, key;
+	struct rte_hash_parameters hash_params = {
+		.entries = num_iterations*2,
+		.bucket_entries = 16,
+		.key_len = sizeof(key),
+		.hash_func = rte_hash_crc,
+		.hash_func_init_val = 0,
+		.socket_id = rte_socket_id(),
+	};
+	struct rte_hash *handle;
+	char name[RTE_HASH_NAMESIZE];
+	rte_spinlock_t lock;
+
+	rte_spinlock_init(&lock);
+
+	snprintf(name, 32, "test%u", calledCount++);
+	hash_params.name = name;
+
+	handle = rte_hash_create(&hash_params);
+	RETURN_IF_ERROR(handle == NULL, "hash creation failed");
+
+	tbl_scaling_test_params.num_iterations =
+		num_iterations/rte_lcore_count();
+	tbl_scaling_test_params.h = handle;
+	tbl_scaling_test_params.lock = &lock;
+	tbl_scaling_test_params.locking_mode = locking_mode;
+
+	rte_atomic64_init(&gcycles);
+	rte_atomic64_clear(&gcycles);
+
+	/* fill up to initial size */
+	for (i = 0; i < num_iterations; i++) {
+		key = rte_hash_crc(&i, sizeof(i), 0xabcdabcd);
+		rte_hash_add_key(tbl_scaling_test_params.h, &key);
+	}
+
+	rte_eal_mp_remote_launch(test_hash_scaling_worker, NULL, CALL_MASTER);
+	rte_eal_mp_wait_lcore();
+
+	unsigned long long int cycles_per_operation =
+		rte_atomic64_read(&gcycles)/
+		(tbl_scaling_test_params.num_iterations*rte_lcore_count());
+	const char *lock_name;
+
+	switch (locking_mode) {
+	case NORMAL_LOCK:
+		lock_name = "normal spinlock";
+		break;
+	case LOCK_ELISION:
+		lock_name = "lock elision";
+		break;
+	default:
+		lock_name = "null lock";
+	}
+	printf("--------------------------------------------------------\n");
+	printf("Cores: %d; %s mode ->  cycles per operation: %llu\n",
+		rte_lcore_count(), lock_name, cycles_per_operation);
+	printf("--------------------------------------------------------\n");
+	/* CSV output */
+	printf(">>>%d,%s,%llu\n", rte_lcore_count(), lock_name,
+		cycles_per_operation);
+
+	rte_hash_free(handle);
+	return 0;
+}
+
+static int
+test_hash_scaling_main(void)
+{
+	int r = 0;
+
+	if (rte_lcore_count() == 1)
+		r = test_hash_scaling(NULL_LOCK);
+
+	if (r == 0)
+		r = test_hash_scaling(NORMAL_LOCK);
+
+	if (!rte_tm_supported()) {
+		printf("Hardware transactional memory (lock elision) is NOT supported\n");
+		return r;
+	}
+	printf("Hardware transactional memory (lock elision) is supported\n");
+
+	if (r == 0)
+		r = test_hash_scaling(LOCK_ELISION);
+
+	return r;
+}
+
+
+static struct test_command hash_scaling_cmd = {
+	.command = "hash_scaling_autotest",
+	.callback = test_hash_scaling_main,
+};
+REGISTER_TEST_COMMAND(hash_scaling_cmd);
-- 
1.9.5.msysgit.0

Intel GmbH
Dornacher Strasse 1
85622 Feldkirchen/Muenchen, Deutschland
Sitz der Gesellschaft: Feldkirchen bei Muenchen
Geschaeftsfuehrer: Christian Lamprechter, Hannes Schwaderer, Douglas Lusk
Registergericht: Muenchen HRB 47456
Ust.-IdNr./VAT Registration No.: DE129385895
Citibank Frankfurt a.M. (BLZ 502 109 00) 600119052

Re: [PATCH v2 0/3] add support for HTM lock elision for x86

[Bruce Richardson]

On Tue, Jun 16, 2015 at 10:16:43AM -0700, Roman Dementiev wrote:
> This series of patches adds methods that use hardware memory transactions (HTM)
> on fast-path for DPDK locks (a.k.a. lock elision). Here the methods are
> implemented for x86 using Restricted Transactional Memory instructions (Intel(r)
> Transactional Synchronization Extensions). The implementation fall-backs to
> the normal DPDK lock if HTM is not available or memory transactions fail. This
> is not a replacement for ALL lock usages since not all critical sections
> protected by locks are friendly to HTM. For example, an attempt to perform
> a HW I/O operation inside a hardware memory transaction always aborts
> the transaction since the CPU is not able to roll-back should the transaction
> fail. Therefore, hardware transactional locks are not advised to be used around
> rte_eth_rx_burst() and rte_eth_tx_burst() calls.
> 
> v2 changes
> -added a documentation note about hardware limitations
> 
> Roman Dementiev (3):
>   spinlock: add support for HTM lock elision for x86
>   rwlock: add support for HTM lock elision for x86
>   test scaling of HTM lock elision protecting rte_hash
> 
A change with a conflict in the test makefile was merged last night. However,
the patches themselves otherwise seem ok. 

Thomas, is a V3 needed for this small conflict, or can you handle it on applying
the patch?

Re: [PATCH v2 0/3] add support for HTM lock elision for x86

[Thomas Monjalon]

2015-06-17 14:05, Bruce Richardson:
> On Tue, Jun 16, 2015 at 10:16:43AM -0700, Roman Dementiev wrote:
> > This series of patches adds methods that use hardware memory transactions (HTM)
> > on fast-path for DPDK locks (a.k.a. lock elision). Here the methods are
> > implemented for x86 using Restricted Transactional Memory instructions (Intel(r)
> > Transactional Synchronization Extensions). The implementation fall-backs to
> > the normal DPDK lock if HTM is not available or memory transactions fail. This
> > is not a replacement for ALL lock usages since not all critical sections
> > protected by locks are friendly to HTM. For example, an attempt to perform
> > a HW I/O operation inside a hardware memory transaction always aborts
> > the transaction since the CPU is not able to roll-back should the transaction
> > fail. Therefore, hardware transactional locks are not advised to be used around
> > rte_eth_rx_burst() and rte_eth_tx_burst() calls.
> > 
> > v2 changes
> > -added a documentation note about hardware limitations
> > 
> > Roman Dementiev (3):
> >   spinlock: add support for HTM lock elision for x86
> >   rwlock: add support for HTM lock elision for x86
> >   test scaling of HTM lock elision protecting rte_hash
> > 
> A change with a conflict in the test makefile was merged last night. However,
> the patches themselves otherwise seem ok. 

Does it mean you ack these patches and they can be blindly applied
without double checking?

> Thomas, is a V3 needed for this small conflict, or can you handle it on applying
> the patch?

Don't worry about conflicts.

Re: [PATCH v2 0/3] add support for HTM lock elision for x86

[Bruce Richardson]

On Wed, Jun 17, 2015 at 03:14:49PM +0200, Thomas Monjalon wrote:
> 2015-06-17 14:05, Bruce Richardson:
> > On Tue, Jun 16, 2015 at 10:16:43AM -0700, Roman Dementiev wrote:
> > > This series of patches adds methods that use hardware memory transactions (HTM)
> > > on fast-path for DPDK locks (a.k.a. lock elision). Here the methods are
> > > implemented for x86 using Restricted Transactional Memory instructions (Intel(r)
> > > Transactional Synchronization Extensions). The implementation fall-backs to
> > > the normal DPDK lock if HTM is not available or memory transactions fail. This
> > > is not a replacement for ALL lock usages since not all critical sections
> > > protected by locks are friendly to HTM. For example, an attempt to perform
> > > a HW I/O operation inside a hardware memory transaction always aborts
> > > the transaction since the CPU is not able to roll-back should the transaction
> > > fail. Therefore, hardware transactional locks are not advised to be used around
> > > rte_eth_rx_burst() and rte_eth_tx_burst() calls.
> > > 
> > > v2 changes
> > > -added a documentation note about hardware limitations
> > > 
> > > Roman Dementiev (3):
> > >   spinlock: add support for HTM lock elision for x86
> > >   rwlock: add support for HTM lock elision for x86
> > >   test scaling of HTM lock elision protecting rte_hash
> > > 
> > A change with a conflict in the test makefile was merged last night. However,
> > the patches themselves otherwise seem ok. 
> 
> Does it mean you ack these patches and they can be blindly applied
> without double checking?
> 

Series Acked-by: Bruce Richardson <bruce.richardson@intel.com>

[PATCH v3 0/3] add support for HTM lock elision for x86

[Roman Dementiev]

This series of patches adds methods that use hardware memory transactions (HTM)
on fast-path for DPDK locks (a.k.a. lock elision). Here the methods are
implemented for x86 using Restricted Transactional Memory instructions (Intel(r)
Transactional Synchronization Extensions). The implementation fall-backs to
the normal DPDK lock if HTM is not available or memory transactions fail. This
is not a replacement for ALL lock usages since not all critical sections
protected by locks are friendly to HTM. For example, an attempt to perform
a HW I/O operation inside a hardware memory transaction always aborts
the transaction since the CPU is not able to roll-back should the transaction
fail. Therefore, hardware transactional locks are not advised to be used around
rte_eth_rx_burst() and rte_eth_tx_burst() calls.

Acked-by: Bruce Richardson <bruce.richardson@intel.com>

v3 changes
-resolved a conflict in app/test/Makefile
-don't use angle brackets for rte_common.h include

v2 changes
-added a documentation note about hardware limitations


Roman Dementiev (3):
  spinlock: add support for HTM lock elision for x86
  rwlock: add support for HTM lock elision for x86
  test scaling of HTM lock elision protecting rte_hash

 app/test/Makefile                                  |   1 +
 app/test/test_hash_scaling.c                       | 223 +++++++++++++++++++++
 lib/librte_eal/common/Makefile                     |   4 +-
 .../common/include/arch/ppc_64/rte_rwlock.h        |  38 ++++
 .../common/include/arch/ppc_64/rte_spinlock.h      |  41 ++++
 lib/librte_eal/common/include/arch/x86/rte_rtm.h   |  73 +++++++
 .../common/include/arch/x86/rte_rwlock.h           |  82 ++++++++
 .../common/include/arch/x86/rte_spinlock.h         | 107 ++++++++++
 lib/librte_eal/common/include/generic/rte_rwlock.h | 208 +++++++++++++++++++
 .../common/include/generic/rte_spinlock.h          |  99 +++++++++
 lib/librte_eal/common/include/rte_rwlock.h         | 158 ---------------
 11 files changed, 874 insertions(+), 160 deletions(-)
 create mode 100644 app/test/test_hash_scaling.c
 create mode 100644 lib/librte_eal/common/include/arch/ppc_64/rte_rwlock.h
 create mode 100644 lib/librte_eal/common/include/arch/x86/rte_rtm.h
 create mode 100644 lib/librte_eal/common/include/arch/x86/rte_rwlock.h
 create mode 100644 lib/librte_eal/common/include/generic/rte_rwlock.h
 delete mode 100644 lib/librte_eal/common/include/rte_rwlock.h

-- 
1.9.5.msysgit.0

Intel GmbH
Dornacher Strasse 1
85622 Feldkirchen/Muenchen, Deutschland
Sitz der Gesellschaft: Feldkirchen bei Muenchen
Geschaeftsfuehrer: Christian Lamprechter, Hannes Schwaderer, Douglas Lusk
Registergericht: Muenchen HRB 47456
Ust.-IdNr./VAT Registration No.: DE129385895
Citibank Frankfurt a.M. (BLZ 502 109 00) 600119052

[PATCH v3 1/3] spinlock: add support for HTM lock elision for x86

[Roman Dementiev]

This patch adds methods that use hardware memory transactions (HTM) on fast-path
for spinlocks (a.k.a. lock elision). Here the methods are implemented for x86
using Restricted Transactional Memory instructions (Intel(r) Transactional
Synchronization Extensions). The implementation fall-backs to the normal
spinlock if HTM is not available or memory transactions fail. This is not
a replacement for all spinlock usages since not all critical sections protected
by spinlocks are friendly to HTM. For example, an attempt to perform a HW I/O
operation inside a hardware memory transaction always aborts the transaction
since the CPU is not able to roll-back should the transaction fail.
Therefore, hardware transactional locks are not advised to be used around
rte_eth_rx_burst() and rte_eth_tx_burst() calls.

Signed-off-by: Roman Dementiev <roman.dementiev@intel.com>
Acked-by: Bruce Richardson <bruce.richardson@intel.com>
---
 .../common/include/arch/ppc_64/rte_spinlock.h      |  41 ++++++++
 lib/librte_eal/common/include/arch/x86/rte_rtm.h   |  73 ++++++++++++++
 .../common/include/arch/x86/rte_spinlock.h         | 107 +++++++++++++++++++++
 .../common/include/generic/rte_spinlock.h          |  99 +++++++++++++++++++
 4 files changed, 320 insertions(+)
 create mode 100644 lib/librte_eal/common/include/arch/x86/rte_rtm.h

diff --git a/lib/librte_eal/common/include/arch/ppc_64/rte_spinlock.h b/lib/librte_eal/common/include/arch/ppc_64/rte_spinlock.h
index cf8b81a..3336435 100644
--- a/lib/librte_eal/common/include/arch/ppc_64/rte_spinlock.h
+++ b/lib/librte_eal/common/include/arch/ppc_64/rte_spinlock.h
@@ -66,6 +66,47 @@ rte_spinlock_trylock(rte_spinlock_t *sl)
 
 #endif
 
+static inline int rte_tm_supported(void)
+{
+	return 0;
+}
+
+static inline void
+rte_spinlock_lock_tm(rte_spinlock_t *sl)
+{
+	rte_spinlock_lock(sl); /* fall-back */
+}
+
+static inline int
+rte_spinlock_trylock_tm(rte_spinlock_t *sl)
+{
+	return rte_spinlock_trylock(sl);
+}
+
+static inline void
+rte_spinlock_unlock_tm(rte_spinlock_t *sl)
+{
+	rte_spinlock_unlock(sl);
+}
+
+static inline void
+rte_spinlock_recursive_lock_tm(rte_spinlock_recursive_t *slr)
+{
+	rte_spinlock_recursive_lock(slr); /* fall-back */
+}
+
+static inline void
+rte_spinlock_recursive_unlock_tm(rte_spinlock_recursive_t *slr)
+{
+	rte_spinlock_recursive_unlock(slr);
+}
+
+static inline int
+rte_spinlock_recursive_trylock_tm(rte_spinlock_recursive_t *slr)
+{
+	return rte_spinlock_recursive_trylock(slr);
+}
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/librte_eal/common/include/arch/x86/rte_rtm.h b/lib/librte_eal/common/include/arch/x86/rte_rtm.h
new file mode 100644
index 0000000..d935641
--- /dev/null
+++ b/lib/librte_eal/common/include/arch/x86/rte_rtm.h
@@ -0,0 +1,73 @@
+#ifndef _RTE_RTM_H_
+#define _RTE_RTM_H_ 1
+
+/*
+ * Copyright (c) 2012,2013 Intel Corporation
+ * Author: Andi Kleen
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that: (1) source code distributions
+ * retain the above copyright notice and this paragraph in its entirety, (2)
+ * distributions including binary code include the above copyright notice and
+ * this paragraph in its entirety in the documentation or other materials
+ * provided with the distribution
+ *
+ * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.
+ */
+
+/* Official RTM intrinsics interface matching gcc/icc, but works
+   on older gcc compatible compilers and binutils. */
+
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+#define RTE_XBEGIN_STARTED		(~0u)
+#define RTE_XABORT_EXPLICIT		(1 << 0)
+#define RTE_XABORT_RETRY		(1 << 1)
+#define RTE_XABORT_CONFLICT		(1 << 2)
+#define RTE_XABORT_CAPACITY		(1 << 3)
+#define RTE_XABORT_DEBUG		(1 << 4)
+#define RTE_XABORT_NESTED		(1 << 5)
+#define RTE_XABORT_CODE(x)		(((x) >> 24) & 0xff)
+
+static __attribute__((__always_inline__)) inline
+unsigned int rte_xbegin(void)
+{
+	unsigned int ret = RTE_XBEGIN_STARTED;
+
+	asm volatile(".byte 0xc7,0xf8 ; .long 0" : "+a" (ret) :: "memory");
+	return ret;
+}
+
+static __attribute__((__always_inline__)) inline
+void rte_xend(void)
+{
+	 asm volatile(".byte 0x0f,0x01,0xd5" ::: "memory");
+}
+
+static __attribute__((__always_inline__)) inline
+void rte_xabort(const unsigned int status)
+{
+	asm volatile(".byte 0xc6,0xf8,%P0" :: "i" (status) : "memory");
+}
+
+static __attribute__((__always_inline__)) inline
+int rte_xtest(void)
+{
+	unsigned char out;
+
+	asm volatile(".byte 0x0f,0x01,0xd6 ; setnz %0" :
+		"=r" (out) :: "memory");
+	return out;
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_RTM_H_ */
diff --git a/lib/librte_eal/common/include/arch/x86/rte_spinlock.h b/lib/librte_eal/common/include/arch/x86/rte_spinlock.h
index 54fba95..20ef0a7 100644
--- a/lib/librte_eal/common/include/arch/x86/rte_spinlock.h
+++ b/lib/librte_eal/common/include/arch/x86/rte_spinlock.h
@@ -39,6 +39,13 @@ extern "C" {
 #endif
 
 #include "generic/rte_spinlock.h"
+#include "rte_rtm.h"
+#include "rte_cpuflags.h"
+#include "rte_branch_prediction.h"
+#include "rte_common.h"
+
+#define RTE_RTM_MAX_RETRIES (10)
+#define RTE_XABORT_LOCK_BUSY (0xff)
 
 #ifndef RTE_FORCE_INTRINSICS
 static inline void
@@ -87,6 +94,106 @@ rte_spinlock_trylock (rte_spinlock_t *sl)
 }
 #endif
 
+static uint8_t rtm_supported; /* cache the flag to avoid the overhead
+				 of the rte_cpu_get_flag_enabled function */
+
+static inline void __attribute__((constructor))
+rte_rtm_init(void)
+{
+	rtm_supported = rte_cpu_get_flag_enabled(RTE_CPUFLAG_RTM);
+}
+
+static inline int rte_tm_supported(void)
+{
+	return rtm_supported;
+}
+
+static inline int
+rte_try_tm(volatile int *lock)
+{
+	if (!rtm_supported)
+		return 0;
+
+	int retries = RTE_RTM_MAX_RETRIES;
+
+	while (likely(retries--)) {
+
+		unsigned int status = rte_xbegin();
+
+		if (likely(RTE_XBEGIN_STARTED == status)) {
+			if (unlikely(*lock))
+				rte_xabort(RTE_XABORT_LOCK_BUSY);
+			else
+				return 1;
+		}
+		while (*lock)
+			rte_pause();
+
+		if ((status & RTE_XABORT_EXPLICIT) &&
+			(RTE_XABORT_CODE(status) == RTE_XABORT_LOCK_BUSY))
+			continue;
+
+		if ((status & RTE_XABORT_RETRY) == 0) /* do not retry */
+			break;
+	}
+	return 0;
+}
+
+static inline void
+rte_spinlock_lock_tm(rte_spinlock_t *sl)
+{
+	if (likely(rte_try_tm(&sl->locked)))
+		return;
+
+	rte_spinlock_lock(sl); /* fall-back */
+}
+
+static inline int
+rte_spinlock_trylock_tm(rte_spinlock_t *sl)
+{
+	if (likely(rte_try_tm(&sl->locked)))
+		return 1;
+
+	return rte_spinlock_trylock(sl);
+}
+
+static inline void
+rte_spinlock_unlock_tm(rte_spinlock_t *sl)
+{
+	if (unlikely(sl->locked))
+		rte_spinlock_unlock(sl);
+	else
+		rte_xend();
+}
+
+static inline void
+rte_spinlock_recursive_lock_tm(rte_spinlock_recursive_t *slr)
+{
+	if (likely(rte_try_tm(&slr->sl.locked)))
+		return;
+
+	rte_spinlock_recursive_lock(slr); /* fall-back */
+}
+
+static inline void
+rte_spinlock_recursive_unlock_tm(rte_spinlock_recursive_t *slr)
+{
+	if (unlikely(slr->sl.locked))
+		rte_spinlock_recursive_unlock(slr);
+	else
+		rte_xend();
+}
+
+static inline int
+rte_spinlock_recursive_trylock_tm(rte_spinlock_recursive_t *slr)
+{
+	if (likely(rte_try_tm(&slr->sl.locked)))
+		return 1;
+
+	return rte_spinlock_recursive_trylock(slr);
+}
+
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/librte_eal/common/include/generic/rte_spinlock.h b/lib/librte_eal/common/include/generic/rte_spinlock.h
index c7fb0df..4e0a3c3 100644
--- a/lib/librte_eal/common/include/generic/rte_spinlock.h
+++ b/lib/librte_eal/common/include/generic/rte_spinlock.h
@@ -145,6 +145,59 @@ static inline int rte_spinlock_is_locked (rte_spinlock_t *sl)
 }
 
 /**
+ * Test if hardware transactional memory (lock elision) is supported
+ *
+ * @return
+ *   1 if the hardware transactional memory is supported; 0 otherwise.
+ */
+static inline int rte_tm_supported(void);
+
+/**
+ * Try to execute critical section in a hardware memory transaction,
+ * if it fails or not available take the spinlock.
+ *
+ * NOTE: An attempt to perform a HW I/O operation inside a hardware memory
+ * transaction always aborts the transaction since the CPU is not able to
+ * roll-back should the transaction fail. Therefore, hardware transactional
+ * locks are not advised to be used around rte_eth_rx_burst() and
+ * rte_eth_tx_burst() calls.
+ *
+ * @param sl
+ *   A pointer to the spinlock.
+ */
+static inline void
+rte_spinlock_lock_tm(rte_spinlock_t *sl);
+
+/**
+ * Commit hardware memory transaction or release the spinlock if
+ * the spinlock is used as a fall-back
+ *
+ * @param sl
+ *   A pointer to the spinlock.
+ */
+static inline void
+rte_spinlock_unlock_tm(rte_spinlock_t *sl);
+
+/**
+ * Try to execute critical section in a hardware memory transaction,
+ * if it fails or not available try to take the lock.
+ *
+ * NOTE: An attempt to perform a HW I/O operation inside a hardware memory
+ * transaction always aborts the transaction since the CPU is not able to
+ * roll-back should the transaction fail. Therefore, hardware transactional
+ * locks are not advised to be used around rte_eth_rx_burst() and
+ * rte_eth_tx_burst() calls.
+ *
+ * @param sl
+ *   A pointer to the spinlock.
+ * @return
+ *   1 if the hardware memory transaction is successfully started
+ *   or lock is successfully taken; 0 otherwise.
+ */
+static inline int
+rte_spinlock_trylock_tm(rte_spinlock_t *sl);
+
+/**
  * The rte_spinlock_recursive_t type.
  */
 typedef struct {
@@ -223,4 +276,50 @@ static inline int rte_spinlock_recursive_trylock(rte_spinlock_recursive_t *slr)
 	return 1;
 }
 
+
+/**
+ * Try to execute critical section in a hardware memory transaction,
+ * if it fails or not available take the recursive spinlocks
+ *
+ * NOTE: An attempt to perform a HW I/O operation inside a hardware memory
+ * transaction always aborts the transaction since the CPU is not able to
+ * roll-back should the transaction fail. Therefore, hardware transactional
+ * locks are not advised to be used around rte_eth_rx_burst() and
+ * rte_eth_tx_burst() calls.
+ *
+ * @param slr
+ *   A pointer to the recursive spinlock.
+ */
+static inline void rte_spinlock_recursive_lock_tm(
+	rte_spinlock_recursive_t *slr);
+
+/**
+ * Commit hardware memory transaction or release the recursive spinlock
+ * if the recursive spinlock is used as a fall-back
+ *
+ * @param slr
+ *   A pointer to the recursive spinlock.
+ */
+static inline void rte_spinlock_recursive_unlock_tm(
+	rte_spinlock_recursive_t *slr);
+
+/**
+ * Try to execute critical section in a hardware memory transaction,
+ * if it fails or not available try to take the recursive lock
+ *
+ * NOTE: An attempt to perform a HW I/O operation inside a hardware memory
+ * transaction always aborts the transaction since the CPU is not able to
+ * roll-back should the transaction fail. Therefore, hardware transactional
+ * locks are not advised to be used around rte_eth_rx_burst() and
+ * rte_eth_tx_burst() calls.
+ *
+ * @param slr
+ *   A pointer to the recursive spinlock.
+ * @return
+ *   1 if the hardware memory transaction is successfully started
+ *   or lock is successfully taken; 0 otherwise.
+ */
+static inline int rte_spinlock_recursive_trylock_tm(
+	rte_spinlock_recursive_t *slr);
+
 #endif /* _RTE_SPINLOCK_H_ */
-- 
1.9.5.msysgit.0

Intel GmbH
Dornacher Strasse 1
85622 Feldkirchen/Muenchen, Deutschland
Sitz der Gesellschaft: Feldkirchen bei Muenchen
Geschaeftsfuehrer: Christian Lamprechter, Hannes Schwaderer, Douglas Lusk
Registergericht: Muenchen HRB 47456
Ust.-IdNr./VAT Registration No.: DE129385895
Citibank Frankfurt a.M. (BLZ 502 109 00) 600119052

[PATCH v3 2/3] rwlock: add support for HTM lock elision for x86

[Roman Dementiev]

This patch adds methods that use hardware memory transactions (HTM) on
fast-path for rwlock (a.k.a. lock elision). Here the methods are implemented
for x86 using Restricted Transactional Memory instructions (Intel(r)
Transactional Synchronization Extensions). The implementation fall-backs to
the normal rwlock if HTM is not available or memory transactions fail. This is
not a replacement for all rwlock usages since not all critical sections
protected by locks are friendly to HTM. For example, an attempt to perform
a HW I/O operation inside a hardware memory transaction always aborts
the transaction since the CPU is not able to roll-back should the transaction
fail. Therefore, hardware transactional locks are not advised to be used around
rte_eth_rx_burst() and rte_eth_tx_burst() calls.

Signed-off-by: Roman Dementiev <roman.dementiev@intel.com>
Acked-by: Bruce Richardson <bruce.richardson@intel.com>
---
 lib/librte_eal/common/Makefile                     |   4 +-
 .../common/include/arch/ppc_64/rte_rwlock.h        |  38 ++++
 .../common/include/arch/x86/rte_rwlock.h           |  82 ++++++++
 lib/librte_eal/common/include/generic/rte_rwlock.h | 208 +++++++++++++++++++++
 lib/librte_eal/common/include/rte_rwlock.h         | 158 ----------------
 5 files changed, 330 insertions(+), 160 deletions(-)
 create mode 100644 lib/librte_eal/common/include/arch/ppc_64/rte_rwlock.h
 create mode 100644 lib/librte_eal/common/include/arch/x86/rte_rwlock.h
 create mode 100644 lib/librte_eal/common/include/generic/rte_rwlock.h
 delete mode 100644 lib/librte_eal/common/include/rte_rwlock.h

diff --git a/lib/librte_eal/common/Makefile b/lib/librte_eal/common/Makefile
index 3ea3bbf..38772d4 100644
--- a/lib/librte_eal/common/Makefile
+++ b/lib/librte_eal/common/Makefile
@@ -35,7 +35,7 @@ INC := rte_branch_prediction.h rte_common.h
 INC += rte_debug.h rte_eal.h rte_errno.h rte_launch.h rte_lcore.h
 INC += rte_log.h rte_memory.h rte_memzone.h rte_pci.h
 INC += rte_pci_dev_ids.h rte_per_lcore.h rte_random.h
-INC += rte_rwlock.h rte_tailq.h rte_interrupts.h rte_alarm.h
+INC += rte_tailq.h rte_interrupts.h rte_alarm.h
 INC += rte_string_fns.h rte_version.h
 INC += rte_eal_memconfig.h rte_malloc_heap.h
 INC += rte_hexdump.h rte_devargs.h rte_dev.h
@@ -46,7 +46,7 @@ INC += rte_warnings.h
 endif
 
 GENERIC_INC := rte_atomic.h rte_byteorder.h rte_cycles.h rte_prefetch.h
-GENERIC_INC += rte_spinlock.h rte_memcpy.h rte_cpuflags.h
+GENERIC_INC += rte_spinlock.h rte_memcpy.h rte_cpuflags.h rte_rwlock.h
 # defined in mk/arch/$(RTE_ARCH)/rte.vars.mk
 ARCH_DIR ?= $(RTE_ARCH)
 ARCH_INC := $(notdir $(wildcard $(RTE_SDK)/lib/librte_eal/common/include/arch/$(ARCH_DIR)/*.h))
diff --git a/lib/librte_eal/common/include/arch/ppc_64/rte_rwlock.h b/lib/librte_eal/common/include/arch/ppc_64/rte_rwlock.h
new file mode 100644
index 0000000..de8af19
--- /dev/null
+++ b/lib/librte_eal/common/include/arch/ppc_64/rte_rwlock.h
@@ -0,0 +1,38 @@
+#ifndef _RTE_RWLOCK_PPC_64_H_
+#define _RTE_RWLOCK_PPC_64_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "generic/rte_rwlock.h"
+
+static inline void
+rte_rwlock_read_lock_tm(rte_rwlock_t *rwl)
+{
+	rte_rwlock_read_lock(rwl);
+}
+
+static inline void
+rte_rwlock_read_unlock_tm(rte_rwlock_t *rwl)
+{
+	rte_rwlock_read_unlock(rwl);
+}
+
+static inline void
+rte_rwlock_write_lock_tm(rte_rwlock_t *rwl)
+{
+	rte_rwlock_write_lock(rwl);
+}
+
+static inline void
+rte_rwlock_write_unlock_tm(rte_rwlock_t *rwl)
+{
+	rte_rwlock_write_unlock(rwl);
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_RWLOCK_PPC_64_H_ */
diff --git a/lib/librte_eal/common/include/arch/x86/rte_rwlock.h b/lib/librte_eal/common/include/arch/x86/rte_rwlock.h
new file mode 100644
index 0000000..afd1c3c
--- /dev/null
+++ b/lib/librte_eal/common/include/arch/x86/rte_rwlock.h
@@ -0,0 +1,82 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2015 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _RTE_RWLOCK_X86_64_H_
+#define _RTE_RWLOCK_X86_64_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "generic/rte_rwlock.h"
+#include "rte_spinlock.h"
+
+static inline void
+rte_rwlock_read_lock_tm(rte_rwlock_t *rwl)
+{
+	if (likely(rte_try_tm(&rwl->cnt)))
+		return;
+	rte_rwlock_read_lock(rwl);
+}
+
+static inline void
+rte_rwlock_read_unlock_tm(rte_rwlock_t *rwl)
+{
+	if (unlikely(rwl->cnt))
+		rte_rwlock_read_unlock(rwl);
+	else
+		rte_xend();
+}
+
+static inline void
+rte_rwlock_write_lock_tm(rte_rwlock_t *rwl)
+{
+	if (likely(rte_try_tm(&rwl->cnt)))
+		return;
+	rte_rwlock_write_lock(rwl);
+}
+
+static inline void
+rte_rwlock_write_unlock_tm(rte_rwlock_t *rwl)
+{
+	if (unlikely(rwl->cnt))
+		rte_rwlock_write_unlock(rwl);
+	else
+		rte_xend();
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_RWLOCK_X86_64_H_ */
diff --git a/lib/librte_eal/common/include/generic/rte_rwlock.h b/lib/librte_eal/common/include/generic/rte_rwlock.h
new file mode 100644
index 0000000..7a0fdc5
--- /dev/null
+++ b/lib/librte_eal/common/include/generic/rte_rwlock.h
@@ -0,0 +1,208 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _RTE_RWLOCK_H_
+#define _RTE_RWLOCK_H_
+
+/**
+ * @file
+ *
+ * RTE Read-Write Locks
+ *
+ * This file defines an API for read-write locks. The lock is used to
+ * protect data that allows multiple readers in parallel, but only
+ * one writer. All readers are blocked until the writer is finished
+ * writing.
+ *
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <rte_common.h>
+#include <rte_atomic.h>
+
+/**
+ * The rte_rwlock_t type.
+ *
+ * cnt is -1 when write lock is held, and > 0 when read locks are held.
+ */
+typedef struct {
+	volatile int32_t cnt; /**< -1 when W lock held, > 0 when R locks held. */
+} rte_rwlock_t;
+
+/**
+ * A static rwlock initializer.
+ */
+#define RTE_RWLOCK_INITIALIZER { 0 }
+
+/**
+ * Initialize the rwlock to an unlocked state.
+ *
+ * @param rwl
+ *   A pointer to the rwlock structure.
+ */
+static inline void
+rte_rwlock_init(rte_rwlock_t *rwl)
+{
+	rwl->cnt = 0;
+}
+
+/**
+ * Take a read lock. Loop until the lock is held.
+ *
+ * @param rwl
+ *   A pointer to a rwlock structure.
+ */
+static inline void
+rte_rwlock_read_lock(rte_rwlock_t *rwl)
+{
+	int32_t x;
+	int success = 0;
+
+	while (success == 0) {
+		x = rwl->cnt;
+		/* write lock is held */
+		if (x < 0) {
+			rte_pause();
+			continue;
+		}
+		success = rte_atomic32_cmpset((volatile uint32_t *)&rwl->cnt,
+					      x, x + 1);
+	}
+}
+
+/**
+ * Release a read lock.
+ *
+ * @param rwl
+ *   A pointer to the rwlock structure.
+ */
+static inline void
+rte_rwlock_read_unlock(rte_rwlock_t *rwl)
+{
+	rte_atomic32_dec((rte_atomic32_t *)(intptr_t)&rwl->cnt);
+}
+
+/**
+ * Take a write lock. Loop until the lock is held.
+ *
+ * @param rwl
+ *   A pointer to a rwlock structure.
+ */
+static inline void
+rte_rwlock_write_lock(rte_rwlock_t *rwl)
+{
+	int32_t x;
+	int success = 0;
+
+	while (success == 0) {
+		x = rwl->cnt;
+		/* a lock is held */
+		if (x != 0) {
+			rte_pause();
+			continue;
+		}
+		success = rte_atomic32_cmpset((volatile uint32_t *)&rwl->cnt,
+					      0, -1);
+	}
+}
+
+/**
+ * Release a write lock.
+ *
+ * @param rwl
+ *   A pointer to a rwlock structure.
+ */
+static inline void
+rte_rwlock_write_unlock(rte_rwlock_t *rwl)
+{
+	rte_atomic32_inc((rte_atomic32_t *)(intptr_t)&rwl->cnt);
+}
+
+/**
+ * Try to execute critical section in a hardware memory transaction, if it
+ * fails or not available take a read lock
+ *
+ * NOTE: An attempt to perform a HW I/O operation inside a hardware memory
+ * transaction always aborts the transaction since the CPU is not able to
+ * roll-back should the transaction fail. Therefore, hardware transactional
+ * locks are not advised to be used around rte_eth_rx_burst() and
+ * rte_eth_tx_burst() calls.
+ *
+ * @param rwl
+ *   A pointer to a rwlock structure.
+ */
+static inline void
+rte_rwlock_read_lock_tm(rte_rwlock_t *rwl);
+
+/**
+ * Commit hardware memory transaction or release the read lock if the lock is used as a fall-back
+ *
+ * @param rwl
+ *   A pointer to the rwlock structure.
+ */
+static inline void
+rte_rwlock_read_unlock_tm(rte_rwlock_t *rwl);
+
+/**
+ * Try to execute critical section in a hardware memory transaction, if it
+ * fails or not available take a write lock
+ *
+ * NOTE: An attempt to perform a HW I/O operation inside a hardware memory
+ * transaction always aborts the transaction since the CPU is not able to
+ * roll-back should the transaction fail. Therefore, hardware transactional
+ * locks are not advised to be used around rte_eth_rx_burst() and
+ * rte_eth_tx_burst() calls.
+ *
+ * @param rwl
+ *   A pointer to a rwlock structure.
+ */
+static inline void
+rte_rwlock_write_lock_tm(rte_rwlock_t *rwl);
+
+/**
+ * Commit hardware memory transaction or release the write lock if the lock is used as a fall-back
+ *
+ * @param rwl
+ *   A pointer to a rwlock structure.
+ */
+static inline void
+rte_rwlock_write_unlock_tm(rte_rwlock_t *rwl);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_RWLOCK_H_ */
diff --git a/lib/librte_eal/common/include/rte_rwlock.h b/lib/librte_eal/common/include/rte_rwlock.h
deleted file mode 100644
index 115731d..0000000
--- a/lib/librte_eal/common/include/rte_rwlock.h
+++ /dev/null
@@ -1,158 +0,0 @@
-/*-
- *   BSD LICENSE
- *
- *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
- *   All rights reserved.
- *
- *   Redistribution and use in source and binary forms, with or without
- *   modification, are permitted provided that the following conditions
- *   are met:
- *
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in
- *       the documentation and/or other materials provided with the
- *       distribution.
- *     * Neither the name of Intel Corporation nor the names of its
- *       contributors may be used to endorse or promote products derived
- *       from this software without specific prior written permission.
- *
- *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef _RTE_RWLOCK_H_
-#define _RTE_RWLOCK_H_
-
-/**
- * @file
- *
- * RTE Read-Write Locks
- *
- * This file defines an API for read-write locks. The lock is used to
- * protect data that allows multiple readers in parallel, but only
- * one writer. All readers are blocked until the writer is finished
- * writing.
- *
- */
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include <rte_common.h>
-#include <rte_atomic.h>
-
-/**
- * The rte_rwlock_t type.
- *
- * cnt is -1 when write lock is held, and > 0 when read locks are held.
- */
-typedef struct {
-	volatile int32_t cnt; /**< -1 when W lock held, > 0 when R locks held. */
-} rte_rwlock_t;
-
-/**
- * A static rwlock initializer.
- */
-#define RTE_RWLOCK_INITIALIZER { 0 }
-
-/**
- * Initialize the rwlock to an unlocked state.
- *
- * @param rwl
- *   A pointer to the rwlock structure.
- */
-static inline void
-rte_rwlock_init(rte_rwlock_t *rwl)
-{
-	rwl->cnt = 0;
-}
-
-/**
- * Take a read lock. Loop until the lock is held.
- *
- * @param rwl
- *   A pointer to a rwlock structure.
- */
-static inline void
-rte_rwlock_read_lock(rte_rwlock_t *rwl)
-{
-	int32_t x;
-	int success = 0;
-
-	while (success == 0) {
-		x = rwl->cnt;
-		/* write lock is held */
-		if (x < 0) {
-			rte_pause();
-			continue;
-		}
-		success = rte_atomic32_cmpset((volatile uint32_t *)&rwl->cnt,
-					      x, x + 1);
-	}
-}
-
-/**
- * Release a read lock.
- *
- * @param rwl
- *   A pointer to the rwlock structure.
- */
-static inline void
-rte_rwlock_read_unlock(rte_rwlock_t *rwl)
-{
-	rte_atomic32_dec((rte_atomic32_t *)(intptr_t)&rwl->cnt);
-}
-
-/**
- * Take a write lock. Loop until the lock is held.
- *
- * @param rwl
- *   A pointer to a rwlock structure.
- */
-static inline void
-rte_rwlock_write_lock(rte_rwlock_t *rwl)
-{
-	int32_t x;
-	int success = 0;
-
-	while (success == 0) {
-		x = rwl->cnt;
-		/* a lock is held */
-		if (x != 0) {
-			rte_pause();
-			continue;
-		}
-		success = rte_atomic32_cmpset((volatile uint32_t *)&rwl->cnt,
-					      0, -1);
-	}
-}
-
-/**
- * Release a write lock.
- *
- * @param rwl
- *   A pointer to a rwlock structure.
- */
-static inline void
-rte_rwlock_write_unlock(rte_rwlock_t *rwl)
-{
-	rte_atomic32_inc((rte_atomic32_t *)(intptr_t)&rwl->cnt);
-}
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _RTE_RWLOCK_H_ */
-- 
1.9.5.msysgit.0

Intel GmbH
Dornacher Strasse 1
85622 Feldkirchen/Muenchen, Deutschland
Sitz der Gesellschaft: Feldkirchen bei Muenchen
Geschaeftsfuehrer: Christian Lamprechter, Hannes Schwaderer, Douglas Lusk
Registergericht: Muenchen HRB 47456
Ust.-IdNr./VAT Registration No.: DE129385895
Citibank Frankfurt a.M. (BLZ 502 109 00) 600119052

[PATCH v3 3/3] test scaling of HTM lock elision protecting rte_hash

[Roman Dementiev]

This patch adds a new auto-test for testing the scaling
of concurrent inserts into rte_hash when protected by
the normal spinlock vs. the spinlock with HTM lock
elision. The test also benchmarks single-threaded
access without any locks.

Signed-off-by: Roman Dementiev <roman.dementiev@intel.com>
Acked-by: Bruce Richardson <bruce.richardson@intel.com>
---
 app/test/Makefile            |   1 +
 app/test/test_hash_scaling.c | 223 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 224 insertions(+)
 create mode 100644 app/test/test_hash_scaling.c

diff --git a/app/test/Makefile b/app/test/Makefile
index 5cf8296..2e2758c 100644
--- a/app/test/Makefile
+++ b/app/test/Makefile
@@ -84,6 +84,7 @@ SRCS-y += test_memcpy_perf.c
 SRCS-$(CONFIG_RTE_LIBRTE_HASH) += test_hash.c
 SRCS-$(CONFIG_RTE_LIBRTE_HASH) += test_hash_perf.c
 SRCS-$(CONFIG_RTE_LIBRTE_HASH) += test_hash_functions.c
+SRCS-$(CONFIG_RTE_LIBRTE_HASH) += test_hash_scaling.c
 
 SRCS-$(CONFIG_RTE_LIBRTE_LPM) += test_lpm.c
 SRCS-$(CONFIG_RTE_LIBRTE_LPM) += test_lpm6.c
diff --git a/app/test/test_hash_scaling.c b/app/test/test_hash_scaling.c
new file mode 100644
index 0000000..682ae94
--- /dev/null
+++ b/app/test/test_hash_scaling.c
@@ -0,0 +1,223 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2015 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <rte_cycles.h>
+#include <rte_hash.h>
+#include <rte_hash_crc.h>
+#include <rte_spinlock.h>
+#include <rte_launch.h>
+
+#include "test.h"
+
+/*
+ * Check condition and return an error if true. Assumes that "handle" is the
+ * name of the hash structure pointer to be freed.
+ */
+#define RETURN_IF_ERROR(cond, str, ...) do {				\
+	if (cond) {							\
+		printf("ERROR line %d: " str "\n", __LINE__,		\
+							##__VA_ARGS__);	\
+		if (handle)						\
+			rte_hash_free(handle);				\
+		return -1;						\
+	}								\
+} while (0)
+
+enum locking_mode_t {
+	NORMAL_LOCK,
+	LOCK_ELISION,
+	NULL_LOCK
+};
+
+struct {
+	uint32_t num_iterations;
+	struct rte_hash *h;
+	rte_spinlock_t *lock;
+	int locking_mode;
+} tbl_scaling_test_params;
+
+static rte_atomic64_t gcycles;
+
+static int test_hash_scaling_worker(__attribute__((unused)) void *arg)
+{
+	uint64_t i, key;
+	uint32_t thr_id = rte_sys_gettid();
+	uint64_t begin, cycles = 0;
+
+	switch (tbl_scaling_test_params.locking_mode) {
+
+	case NORMAL_LOCK:
+
+		for (i = 0; i < tbl_scaling_test_params.num_iterations; i++) {
+			/*	different threads get different keys because
+				we use the thread-id in the key computation
+			 */
+			key = rte_hash_crc(&i, sizeof(i), thr_id);
+			begin = rte_rdtsc_precise();
+			rte_spinlock_lock(tbl_scaling_test_params.lock);
+			rte_hash_add_key(tbl_scaling_test_params.h, &key);
+			rte_spinlock_unlock(tbl_scaling_test_params.lock);
+			cycles += rte_rdtsc_precise() - begin;
+		}
+		break;
+
+	case LOCK_ELISION:
+
+		for (i = 0; i < tbl_scaling_test_params.num_iterations; i++) {
+			key = rte_hash_crc(&i, sizeof(i), thr_id);
+			begin = rte_rdtsc_precise();
+			rte_spinlock_lock_tm(tbl_scaling_test_params.lock);
+			rte_hash_add_key(tbl_scaling_test_params.h, &key);
+			rte_spinlock_unlock_tm(tbl_scaling_test_params.lock);
+			cycles += rte_rdtsc_precise() - begin;
+		}
+		break;
+
+	default:
+
+		for (i = 0; i < tbl_scaling_test_params.num_iterations; i++) {
+			key = rte_hash_crc(&i, sizeof(i), thr_id);
+			begin = rte_rdtsc_precise();
+			rte_hash_add_key(tbl_scaling_test_params.h, &key);
+			cycles += rte_rdtsc_precise() - begin;
+		}
+	}
+
+	rte_atomic64_add(&gcycles, cycles);
+
+	return 0;
+}
+
+/*
+ * Do scalability perf tests.
+ */
+static int
+test_hash_scaling(int locking_mode)
+{
+	static unsigned calledCount =    1;
+	uint32_t num_iterations = 1024*1024;
+	uint64_t i, key;
+	struct rte_hash_parameters hash_params = {
+		.entries = num_iterations*2,
+		.bucket_entries = 16,
+		.key_len = sizeof(key),
+		.hash_func = rte_hash_crc,
+		.hash_func_init_val = 0,
+		.socket_id = rte_socket_id(),
+	};
+	struct rte_hash *handle;
+	char name[RTE_HASH_NAMESIZE];
+	rte_spinlock_t lock;
+
+	rte_spinlock_init(&lock);
+
+	snprintf(name, 32, "test%u", calledCount++);
+	hash_params.name = name;
+
+	handle = rte_hash_create(&hash_params);
+	RETURN_IF_ERROR(handle == NULL, "hash creation failed");
+
+	tbl_scaling_test_params.num_iterations =
+		num_iterations/rte_lcore_count();
+	tbl_scaling_test_params.h = handle;
+	tbl_scaling_test_params.lock = &lock;
+	tbl_scaling_test_params.locking_mode = locking_mode;
+
+	rte_atomic64_init(&gcycles);
+	rte_atomic64_clear(&gcycles);
+
+	/* fill up to initial size */
+	for (i = 0; i < num_iterations; i++) {
+		key = rte_hash_crc(&i, sizeof(i), 0xabcdabcd);
+		rte_hash_add_key(tbl_scaling_test_params.h, &key);
+	}
+
+	rte_eal_mp_remote_launch(test_hash_scaling_worker, NULL, CALL_MASTER);
+	rte_eal_mp_wait_lcore();
+
+	unsigned long long int cycles_per_operation =
+		rte_atomic64_read(&gcycles)/
+		(tbl_scaling_test_params.num_iterations*rte_lcore_count());
+	const char *lock_name;
+
+	switch (locking_mode) {
+	case NORMAL_LOCK:
+		lock_name = "normal spinlock";
+		break;
+	case LOCK_ELISION:
+		lock_name = "lock elision";
+		break;
+	default:
+		lock_name = "null lock";
+	}
+	printf("--------------------------------------------------------\n");
+	printf("Cores: %d; %s mode ->  cycles per operation: %llu\n",
+		rte_lcore_count(), lock_name, cycles_per_operation);
+	printf("--------------------------------------------------------\n");
+	/* CSV output */
+	printf(">>>%d,%s,%llu\n", rte_lcore_count(), lock_name,
+		cycles_per_operation);
+
+	rte_hash_free(handle);
+	return 0;
+}
+
+static int
+test_hash_scaling_main(void)
+{
+	int r = 0;
+
+	if (rte_lcore_count() == 1)
+		r = test_hash_scaling(NULL_LOCK);
+
+	if (r == 0)
+		r = test_hash_scaling(NORMAL_LOCK);
+
+	if (!rte_tm_supported()) {
+		printf("Hardware transactional memory (lock elision) is NOT supported\n");
+		return r;
+	}
+	printf("Hardware transactional memory (lock elision) is supported\n");
+
+	if (r == 0)
+		r = test_hash_scaling(LOCK_ELISION);
+
+	return r;
+}
+
+
+static struct test_command hash_scaling_cmd = {
+	.command = "hash_scaling_autotest",
+	.callback = test_hash_scaling_main,
+};
+REGISTER_TEST_COMMAND(hash_scaling_cmd);
-- 
1.9.5.msysgit.0

Intel GmbH
Dornacher Strasse 1
85622 Feldkirchen/Muenchen, Deutschland
Sitz der Gesellschaft: Feldkirchen bei Muenchen
Geschaeftsfuehrer: Christian Lamprechter, Hannes Schwaderer, Douglas Lusk
Registergericht: Muenchen HRB 47456
Ust.-IdNr./VAT Registration No.: DE129385895
Citibank Frankfurt a.M. (BLZ 502 109 00) 600119052

Re: [PATCH v3 0/3] add support for HTM lock elision for x86

[Thomas Monjalon]

2015-06-19 13:08, Roman Dementiev:
> This series of patches adds methods that use hardware memory transactions (HTM)
> on fast-path for DPDK locks (a.k.a. lock elision). Here the methods are
> implemented for x86 using Restricted Transactional Memory instructions (Intel(r)
> Transactional Synchronization Extensions). The implementation fall-backs to
> the normal DPDK lock if HTM is not available or memory transactions fail. This
> is not a replacement for ALL lock usages since not all critical sections
> protected by locks are friendly to HTM. For example, an attempt to perform
> a HW I/O operation inside a hardware memory transaction always aborts
> the transaction since the CPU is not able to roll-back should the transaction
> fail. Therefore, hardware transactional locks are not advised to be used around
> rte_eth_rx_burst() and rte_eth_tx_burst() calls.
> 
> Acked-by: Bruce Richardson <bruce.richardson@intel.com>
> 
> v3 changes
> -resolved a conflict in app/test/Makefile
> -don't use angle brackets for rte_common.h include
> 
> v2 changes
> -added a documentation note about hardware limitations
> 
> 
> Roman Dementiev (3):
>   spinlock: add support for HTM lock elision for x86
>   rwlock: add support for HTM lock elision for x86
>   test scaling of HTM lock elision protecting rte_hash

Applied, thanks