From 652e87b7e7d84a8331e9a8eb6664d42134c4bf5d Mon Sep 17 00:00:00 2001 From: Andrey Semashev Date: Wed, 7 May 2014 22:21:04 +0400 Subject: [PATCH] Added Linux ARM backend. --- .../boost/atomic/detail/ops_gcc_atomic.hpp | 1 - include/boost/atomic/detail/ops_gcc_sync.hpp | 58 ++++- include/boost/atomic/detail/ops_linux_arm.hpp | 232 ++++++++++++++++++ 3 files changed, 289 insertions(+), 2 deletions(-) create mode 100644 include/boost/atomic/detail/ops_linux_arm.hpp diff --git a/include/boost/atomic/detail/ops_gcc_atomic.hpp b/include/boost/atomic/detail/ops_gcc_atomic.hpp index 5f81f19..8260780 100644 --- a/include/boost/atomic/detail/ops_gcc_atomic.hpp +++ b/include/boost/atomic/detail/ops_gcc_atomic.hpp @@ -359,7 +359,6 @@ BOOST_FORCEINLINE void signal_fence(memory_order order) BOOST_NOEXCEPT } } // namespace detail - } // namespace atomics } // namespace boost diff --git a/include/boost/atomic/detail/ops_gcc_sync.hpp b/include/boost/atomic/detail/ops_gcc_sync.hpp index 79a7311..1e969cc 100644 --- a/include/boost/atomic/detail/ops_gcc_sync.hpp +++ b/include/boost/atomic/detail/ops_gcc_sync.hpp @@ -16,6 +16,7 @@ #ifndef BOOST_ATOMIC_DETAIL_OPS_GCC_SYNC_HPP_INCLUDED_ #define BOOST_ATOMIC_DETAIL_OPS_GCC_SYNC_HPP_INCLUDED_ +#include // UINT64_C #include #include #include @@ -183,6 +184,20 @@ struct operations< 1u > : #endif > { +#if !defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_1) + static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT + { + // We must resort to a CAS loop to handle overflows + storage_type res = storage; + while (!compare_exchange_strong(storage, res, (res + v) & 0x000000ff, order, memory_order_relaxed)) {} + return res; + } + + static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT + { + return fetch_add(storage, -v, order); + } +#endif }; #endif @@ -201,6 +216,20 @@ struct operations< 2u > : #endif > { +#if !defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_2) + static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT + { + // We must resort to a CAS loop to handle overflows + storage_type res = storage; + while (!compare_exchange_strong(storage, res, (res + v) & 0x0000ffff, order, memory_order_relaxed)) {} + return res; + } + + static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT + { + return fetch_add(storage, -v, order); + } +#endif }; #endif @@ -217,6 +246,20 @@ struct operations< 4u > : #endif > { +#if !defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_4) + static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT + { + // We must resort to a CAS loop to handle overflows + storage_type res = storage; + while (!compare_exchange_strong(storage, res, (res + v) & 0xffffffff, order, memory_order_relaxed)) {} + return res; + } + + static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT + { + return fetch_add(storage, -v, order); + } +#endif }; #endif @@ -231,6 +274,20 @@ struct operations< 8u > : #endif > { +#if !defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_8) && defined(BOOST_HAS_INT128) + static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT + { + // We must resort to a CAS loop to handle overflows + storage_type res = storage; + while (!compare_exchange_strong(storage, res, (res + v) & UINT64_C(0xffffffffffffffff), order, memory_order_relaxed)) {} + return res; + } + + static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT + { + return fetch_add(storage, -v, order); + } +#endif }; #endif @@ -277,7 +334,6 @@ BOOST_FORCEINLINE void signal_fence(memory_order order) BOOST_NOEXCEPT } } // namespace detail - } // namespace atomics } // namespace boost diff --git a/include/boost/atomic/detail/ops_linux_arm.hpp b/include/boost/atomic/detail/ops_linux_arm.hpp new file mode 100644 index 0000000..3ce4c1e --- /dev/null +++ b/include/boost/atomic/detail/ops_linux_arm.hpp @@ -0,0 +1,232 @@ +/* + * Distributed under the Boost Software License, Version 1.0. + * (See accompanying file LICENSE_1_0.txt or copy at + * http://www.boost.org/LICENSE_1_0.txt) + * + * Copyright (c) 2009, 2011 Helge Bahmann + * Copyright (c) 2009 Phil Endecott + * Copyright (c) 2013 Tim Blechmann + * Linux-specific code by Phil Endecott + * Copyright (c) 2014 Andrey Semashev + */ +/*! + * \file atomic/detail/ops_linux_arm.hpp + * + * This header contains implementation of the \c operations template. + */ + +#ifndef BOOST_ATOMIC_DETAIL_OPS_GCC_LINUX_ARM_HPP_INCLUDED_ +#define BOOST_ATOMIC_DETAIL_OPS_GCC_LINUX_ARM_HPP_INCLUDED_ + +#include +#include +#include +#include +#include +#include + +#ifdef BOOST_HAS_PRAGMA_ONCE +#pragma once +#endif + +namespace boost { +namespace atomics { +namespace detail { + +// Different ARM processors have different atomic instructions. In particular, +// architecture versions before v6 (which are still in widespread use, e.g. the +// Intel/Marvell XScale chips like the one in the NSLU2) have only atomic swap. +// On Linux the kernel provides some support that lets us abstract away from +// these differences: it provides emulated CAS and barrier functions at special +// addresses that are guaranteed not to be interrupted by the kernel. Using +// this facility is slightly slower than inline assembler would be, but much +// faster than a system call. +// +// While this emulated CAS is "strong" in the sense that it does not fail +// "spuriously" (i.e.: it never fails to perform the exchange when the value +// found equals the value expected), it does not return the found value on +// failure. To satisfy the atomic API, compare_exchange_{weak|strong} must +// return the found value on failure, and we have to manually load this value +// after the emulated CAS reports failure. This in turn introduces a race +// between the CAS failing (due to the "wrong" value being found) and subsequently +// loading (which might turn up the "right" value). From an application's +// point of view this looks like "spurious failure", and therefore the +// emulated CAS is only good enough to provide compare_exchange_weak +// semantics. + +struct linux_arm_cas +{ + typedef storage32_t storage_type; + + static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT + { + fence_before_store(order); + storage = v; + fence_after_store(order); + } + + static BOOST_FORCEINLINE storage_type load(storage_type const volatile& storage, memory_order order) BOOST_NOEXCEPT + { + storage_type v = storage; + fence_after_load(order); + return v; + } + + static BOOST_FORCEINLINE bool compare_exchange_strong( + storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT + { + while (true) + { + storage_type tmp = expected; + if (compare_exchange_weak(storage, tmp, desired, success_order, failure_order)) + return true; + if (tmp != expected) + { + expected = tmp; + return false; + } + } + } + + static BOOST_FORCEINLINE bool compare_exchange_weak( + storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order, memory_order) BOOST_NOEXCEPT + { + typedef storage_type (*kernel_cmpxchg32_t)(storage_type oldval, storage_type newval, volatile storage_type* ptr); + + if (((kernel_cmpxchg32_t)0xffff0fc0)(expected, desired, &storage) == 0) + { + return true; + } + else + { + expected = storage; + return false; + } + } + + static BOOST_FORCEINLINE bool is_lock_free(storage_type const volatile&) BOOST_NOEXCEPT + { + return true; + } + + static BOOST_FORCEINLINE void hardware_full_fence() BOOST_NOEXCEPT + { + typedef void (*kernel_dmb_t)(void); + ((kernel_dmb_t)0xffff0fa0)(); + } + +private: + static BOOST_FORCEINLINE void fence_before_store(memory_order order) BOOST_NOEXCEPT + { + switch (order) + { + case memory_order_release: + case memory_order_acq_rel: + case memory_order_seq_cst: + hardware_full_fence(); + break; + case memory_order_consume: + default:; + } + } + + static BOOST_FORCEINLINE void fence_after_store(memory_order order) BOOST_NOEXCEPT + { + if (order == memory_order_seq_cst) + hardware_full_fence(); + } + + static BOOST_FORCEINLINE void fence_after_load(memory_order order) BOOST_NOEXCEPT + { + switch (order) + { + case memory_order_acquire: + case memory_order_acq_rel: + case memory_order_seq_cst: + hardware_full_fence(); + break; + default:; + } + } +}; + +template< > +struct operations< 1u > : + public cas_based_operations< linux_arm_cas > +{ + static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT + { + // We must resort to a CAS loop to handle overflows + storage_type res = storage; + while (!compare_exchange_weak(storage, res, (res + v) & 0x000000ff, order, memory_order_relaxed)) {} + return res; + } + + static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT + { + return fetch_add(storage, -v, order); + } +}; + +template< > +struct operations< 2u > : + public cas_based_operations< linux_arm_cas > +{ + static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT + { + // We must resort to a CAS loop to handle overflows + storage_type res = storage; + while (!compare_exchange_weak(storage, res, (res + v) & 0x0000ffff, order, memory_order_relaxed)) {} + return res; + } + + static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT + { + return fetch_add(storage, -v, order); + } +}; + +template< > +struct operations< 4u > : + public cas_based_operations< linux_arm_cas > +{ +}; + +BOOST_FORCEINLINE void thread_fence(memory_order order) BOOST_NOEXCEPT +{ + switch (order) + { + case memory_order_relaxed: + break; + case memory_order_release: + case memory_order_consume: + case memory_order_acquire: + case memory_order_acq_rel: + case memory_order_seq_cst: + linux_arm_cas::hardware_full_fence(); + break; + } +} + +BOOST_FORCEINLINE void signal_fence(memory_order order) BOOST_NOEXCEPT +{ + switch (order) + { + case memory_order_relaxed: + case memory_order_consume: + break; + case memory_order_acquire: + case memory_order_release: + case memory_order_acq_rel: + case memory_order_seq_cst: + __asm__ __volatile__ ("" ::: "memory"); + break; + default:; + } +} + +} // namespace detail +} // namespace atomics +} // namespace boost + +#endif // BOOST_ATOMIC_DETAIL_OPS_GCC_LINUX_ARM_HPP_INCLUDED_