1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
|
/* Copyright (c) 2017-2018, Linaro Limited
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
*/
#ifndef ODP_FRAGREASS_PP_ATOMICS_ARM_H_
#define ODP_FRAGREASS_PP_ATOMICS_ARM_H_
#include <odp_api.h>
#if __SIZEOF_POINTER__ == 8 && defined(__aarch64__)
__extension__ typedef __int128 _int128_t;
static inline _int128_t lld(_int128_t *var, int mo)
{
_int128_t old;
uint64_t lo, hi;
if (mo == __ATOMIC_ACQUIRE)
__asm__ volatile("ldaxp %0, %1, [%2]" : "=&r" (lo), "=&r" (hi)
: "r" (var) : "memory");
else /* mo == __ATOMIC_RELAXED */
__asm__ volatile("ldxp %0, %1, [%2]" : "=&r" (lo), "=&r" (hi)
: "r" (var) : );
old = hi;
old <<= 64;
old |= lo;
return old;
}
static inline uint32_t scd(_int128_t *var, _int128_t neu, int mo)
{
uint32_t ret;
uint64_t lo = neu, hi = neu >> 64;
if (mo == __ATOMIC_RELEASE)
__asm__ volatile("stlxp %w0, %1, %2, [%3]" : "=&r" (ret)
: "r" (lo), "r" (hi), "r" (var) : "memory");
else /* mo == __ATOMIC_RELAXED */
__asm__ volatile("stxp %w0, %1, %2, [%3]" : "=&r" (ret)
: "r" (lo), "r" (hi), "r" (var) : "memory");
return ret;
}
static inline bool atomic_strong_cas_dblptr(_int128_t *var, _int128_t *exp,
_int128_t neu, int mo_success,
int mo_failure ODP_UNUSED)
{
register _int128_t old;
register _int128_t expected = *exp;
int ll_mo, sc_mo;
ll_mo = (mo_success != __ATOMIC_RELAXED &&
mo_success != __ATOMIC_RELEASE) ? __ATOMIC_ACQUIRE
: __ATOMIC_RELAXED;
sc_mo = (mo_success == __ATOMIC_RELEASE ||
mo_success == __ATOMIC_ACQ_REL ||
mo_success == __ATOMIC_SEQ_CST) ? __ATOMIC_RELEASE
: __ATOMIC_RELAXED;
/*
* To prevent spurious failures and ensure atomicity, we must write some
* value back -- whether it's the value we wanted to write, or the value
* that is currently there. Repeat until we perform a successful write.
*/
do {
old = lld(var, ll_mo);
} while (scd(var, old == expected ? neu : old, sc_mo));
*exp = old;
return (old == expected);
}
#elif __SIZEOF_POINTER__ == 4 && defined(__ARM_ARCH) && __ARM_ARCH == 7
static inline uint64_t lld(uint64_t *var, int mo)
{
uint64_t old;
__asm__ volatile("ldrexd %0, %H0, [%1]" : "=&r" (old) : "r" (var) : );
if (mo == __ATOMIC_ACQUIRE)
__asm__ volatile("dmb ish" ::: "memory");
return old;
}
static inline uint32_t scd(uint64_t *var, uint64_t neu, int mo)
{
uint32_t ret;
if (mo == __ATOMIC_RELEASE)
__asm__ volatile("dmb ish" ::: "memory");
__asm__ volatile("strexd %0, %1, %H1, [%2]" : "=&r" (ret)
: "r" (neu), "r" (var) : );
return ret;
}
static inline bool atomic_strong_cas_dblptr(uint64_t *var, uint64_t *exp,
uint64_t neu, int mo_success,
int mo_failure ODP_UNUSED)
{
register uint64_t old;
register uint64_t expected = *exp;
int ll_mo, sc_mo;
ll_mo = (mo_success != __ATOMIC_RELAXED &&
mo_success != __ATOMIC_RELEASE) ? __ATOMIC_ACQUIRE
: __ATOMIC_RELAXED;
sc_mo = (mo_success == __ATOMIC_RELEASE ||
mo_success == __ATOMIC_ACQ_REL ||
mo_success == __ATOMIC_SEQ_CST) ? __ATOMIC_RELEASE
: __ATOMIC_RELAXED;
/*
* To prevent spurious failures and ensure atomicity, we must write some
* value back -- whether it's the value we wanted to write, or the value
* that is currently there. Repeat until we perform a successful write.
*/
do {
old = lld(var, ll_mo);
} while (scd(var, old == expected ? neu : old, sc_mo));
*exp = old;
return (old == expected);
}
#else
#include "odp_ipfragreass_atomics.h"
#endif
#endif
|
