Kokkos Core Kernels Package  Version of the Day
Kokkos_Atomics_Desul_Wrapper.hpp
1 //@HEADER
2 // ************************************************************************
3 //
4 // Kokkos v. 4.0
5 // Copyright (2022) National Technology & Engineering
6 // Solutions of Sandia, LLC (NTESS).
7 //
8 // Under the terms of Contract DE-NA0003525 with NTESS,
9 // the U.S. Government retains certain rights in this software.
10 //
11 // Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
12 // See https://kokkos.org/LICENSE for license information.
13 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
14 //
15 //@HEADER
16 
17 #ifndef KOKKOS_IMPL_PUBLIC_INCLUDE
18 #include <Kokkos_Macros.hpp>
19 static_assert(false,
20  "Including non-public Kokkos header files is not allowed.");
21 #endif
22 #ifndef KOKKOS_DESUL_ATOMICS_WRAPPER_HPP_
23 #define KOKKOS_DESUL_ATOMICS_WRAPPER_HPP_
24 #include <Kokkos_Macros.hpp>
25 
26 #ifdef KOKKOS_ENABLE_IMPL_DESUL_ATOMICS
27 #include <Kokkos_Atomics_Desul_Config.hpp>
28 #include <desul/atomics.hpp>
29 
30 #include <impl/Kokkos_Atomic_Memory_Order.hpp>
31 #include <impl/Kokkos_Volatile_Load.hpp>
32 
33 // clang-format off
34 namespace Kokkos {
35 
36 // FIXME: These functions don't have any use/test in unit tests ...
37 // ==========================================================
38 inline const char* atomic_query_version() { return "KOKKOS_DESUL_ATOMICS"; }
39 
40 #if defined(KOKKOS_COMPILER_GNU) && !defined(__PGIC__) && \
41  !defined(__CUDA_ARCH__)
42 
43 #define KOKKOS_NONTEMPORAL_PREFETCH_LOAD(addr) __builtin_prefetch(addr, 0, 0)
44 #define KOKKOS_NONTEMPORAL_PREFETCH_STORE(addr) __builtin_prefetch(addr, 1, 0)
45 
46 #else
47 
48 #define KOKKOS_NONTEMPORAL_PREFETCH_LOAD(addr) ((void)0)
49 #define KOKKOS_NONTEMPORAL_PREFETCH_STORE(addr) ((void)0)
50 
51 #endif
52 // ============================================================
53 
54 #ifdef KOKKOS_INTERNAL_NOT_PARALLEL
55 #define KOKKOS_DESUL_MEM_SCOPE desul::MemoryScopeCaller()
56 #else
57 #define KOKKOS_DESUL_MEM_SCOPE desul::MemoryScopeDevice()
58 #endif
59 
60 template<class T> KOKKOS_INLINE_FUNCTION
61 T atomic_load(T* const dest) { return desul::atomic_load(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
62 
63 template<class T> KOKKOS_INLINE_FUNCTION
64 void atomic_store(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_store(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
65 
66 template<class T> KOKKOS_INLINE_FUNCTION
67 void atomic_assign(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { atomic_store(dest,val); }
68 
69 KOKKOS_INLINE_FUNCTION
70 void memory_fence() {
71  desul::atomic_thread_fence(desul::MemoryOrderSeqCst(), KOKKOS_DESUL_MEM_SCOPE);
72 }
73 
74 KOKKOS_INLINE_FUNCTION
75 void load_fence() { return desul::atomic_thread_fence(desul::MemoryOrderAcquire(), KOKKOS_DESUL_MEM_SCOPE); }
76 
77 KOKKOS_INLINE_FUNCTION
78 void store_fence() { return desul::atomic_thread_fence(desul::MemoryOrderRelease(), KOKKOS_DESUL_MEM_SCOPE); }
79 
80 // atomic_fetch_op
81 template<class T> KOKKOS_INLINE_FUNCTION
82 T atomic_fetch_add (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_add (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
83 
84 template<class T> KOKKOS_INLINE_FUNCTION
85 T atomic_fetch_sub (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_sub (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
86 
87 template<class T> KOKKOS_INLINE_FUNCTION
88 T atomic_fetch_max (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_max (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
89 
90 template<class T> KOKKOS_INLINE_FUNCTION
91 T atomic_fetch_min (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_min (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
92 
93 template<class T> KOKKOS_INLINE_FUNCTION
94 T atomic_fetch_mul (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_mul (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
95 
96 template<class T> KOKKOS_INLINE_FUNCTION
97 T atomic_fetch_div (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_div (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
98 
99 template<class T> KOKKOS_INLINE_FUNCTION
100 T atomic_fetch_mod (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_mod (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
101 
102 template<class T> KOKKOS_INLINE_FUNCTION
103 T atomic_fetch_and (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_and (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
104 
105 template<class T> KOKKOS_INLINE_FUNCTION
106 T atomic_fetch_or (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_or (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
107 
108 template<class T> KOKKOS_INLINE_FUNCTION
109 T atomic_fetch_xor (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_xor (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
110 
111 template<class T> KOKKOS_INLINE_FUNCTION
112 T atomic_fetch_nand(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_nand(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
113 
114 template<class T> KOKKOS_INLINE_FUNCTION
115 T atomic_fetch_lshift(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_lshift(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
116 
117 template<class T> KOKKOS_INLINE_FUNCTION
118 T atomic_fetch_rshift(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_rshift(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
119 
120 template<class T> KOKKOS_INLINE_FUNCTION
121 T atomic_fetch_inc(T* const dest) { return desul::atomic_fetch_inc(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
122 
123 template<class T> KOKKOS_INLINE_FUNCTION
124 T atomic_fetch_dec(T* const dest) { return desul::atomic_fetch_dec(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
125 
126 
127 // atomic_op_fetch
128 template<class T> KOKKOS_INLINE_FUNCTION
129 T atomic_add_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_add_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
130 
131 template<class T> KOKKOS_INLINE_FUNCTION
132 T atomic_sub_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_sub_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
133 
134 template<class T> KOKKOS_INLINE_FUNCTION
135 T atomic_max_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_max_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
136 
137 template<class T> KOKKOS_INLINE_FUNCTION
138 T atomic_min_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_min_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
139 
140 template<class T> KOKKOS_INLINE_FUNCTION
141 T atomic_mul_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_mul_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
142 
143 template<class T> KOKKOS_INLINE_FUNCTION
144 T atomic_div_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_div_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
145 
146 template<class T> KOKKOS_INLINE_FUNCTION
147 T atomic_mod_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_mod_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
148 
149 template<class T> KOKKOS_INLINE_FUNCTION
150 T atomic_and_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_and_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
151 
152 template<class T> KOKKOS_INLINE_FUNCTION
153 T atomic_or_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_or_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
154 
155 template<class T> KOKKOS_INLINE_FUNCTION
156 T atomic_xor_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_xor_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
157 
158 template<class T> KOKKOS_INLINE_FUNCTION
159 T atomic_nand_fetch(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_nand_fetch(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
160 
161 template<class T> KOKKOS_INLINE_FUNCTION
162 T atomic_lshift_fetch(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_lshift_fetch(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
163 
164 template<class T> KOKKOS_INLINE_FUNCTION
165 T atomic_rshift_fetch(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_rshift_fetch(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
166 
167 template<class T> KOKKOS_INLINE_FUNCTION
168 T atomic_inc_fetch(T* const dest) { return desul::atomic_inc_fetch(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
169 
170 template<class T> KOKKOS_INLINE_FUNCTION
171 T atomic_dec_fetch(T* const dest) { return desul::atomic_dec_fetch(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
172 
173 
174 // atomic_op
175 template<class T> KOKKOS_INLINE_FUNCTION
176 void atomic_add(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_add (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
177 
178 template<class T> KOKKOS_INLINE_FUNCTION
179 void atomic_sub(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_sub (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
180 
181 template<class T> KOKKOS_INLINE_FUNCTION
182 void atomic_mul(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_mul (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
183 
184 template<class T> KOKKOS_INLINE_FUNCTION
185 void atomic_div(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_div (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
186 
187 template<class T> KOKKOS_INLINE_FUNCTION
188 void atomic_min(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_min (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
189 
190 template<class T> KOKKOS_INLINE_FUNCTION
191 void atomic_max(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_max (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
192 
193 // FIXME: Desul doesn't have atomic_and yet so call fetch_and
194 template<class T> KOKKOS_INLINE_FUNCTION
195 void atomic_and(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { (void) desul::atomic_fetch_and (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
196 
197 // FIXME: Desul doesn't have atomic_or yet so call fetch_or
198 template<class T> KOKKOS_INLINE_FUNCTION
199 void atomic_or(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { (void) desul::atomic_fetch_or (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
200 
201 template<class T> KOKKOS_INLINE_FUNCTION
202 void atomic_inc(T* const dest) { return desul::atomic_inc(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
203 
204 template<class T> KOKKOS_INLINE_FUNCTION
205 void atomic_dec(T* const dest) { return desul::atomic_dec(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
206 
207 template<class T> KOKKOS_INLINE_FUNCTION
208 void atomic_increment(T* const dest) { return desul::atomic_inc(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
209 
210 template<class T> KOKKOS_INLINE_FUNCTION
211 void atomic_decrement(T* const dest) { return desul::atomic_dec(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
212 
213 // Exchange
214 
215 template<class T> KOKKOS_INLINE_FUNCTION
216 T atomic_exchange(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_exchange(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
217 
218 template<class T> KOKKOS_INLINE_FUNCTION
219 bool atomic_compare_exchange_strong(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> expected, desul::Impl::dont_deduce_this_parameter_t<const T> desired) {
220  T expected_ref = expected;
221  return desul::atomic_compare_exchange_strong(dest, expected_ref, desired,
222  desul::MemoryOrderRelaxed(), desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE);
223 }
224 
225 template<class T> KOKKOS_INLINE_FUNCTION
226 T atomic_compare_exchange(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> compare, desul::Impl::dont_deduce_this_parameter_t<const T> desired) {
227  return desul::atomic_compare_exchange(dest, compare, desired,
228  desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE);
229 }
230 
231 namespace Impl {
232 
233  template<class MemoryOrder>
234  struct KokkosToDesulMemoryOrder;
235 
236  template<>
237  struct KokkosToDesulMemoryOrder<memory_order_seq_cst_t> {
238  using type = desul::MemoryOrderSeqCst;
239  };
240  template<>
241  struct KokkosToDesulMemoryOrder<memory_order_acquire_t> {
242  using type = desul::MemoryOrderAcquire;
243  };
244  template<>
245  struct KokkosToDesulMemoryOrder<memory_order_release_t> {
246  using type = desul::MemoryOrderRelease;
247  };
248  template<>
249  struct KokkosToDesulMemoryOrder<memory_order_acq_rel_t> {
250  using type = desul::MemoryOrderAcqRel;
251  };
252  template<>
253  struct KokkosToDesulMemoryOrder<memory_order_relaxed_t> {
254  using type = desul::MemoryOrderRelaxed;
255  };
256  template<class T, class MemOrderSuccess, class MemOrderFailure> KOKKOS_INLINE_FUNCTION
257  bool atomic_compare_exchange_strong(T* const dest, T& expected, const T desired, MemOrderSuccess, MemOrderFailure) {
258  return desul::atomic_compare_exchange_strong(dest, expected, desired,
259  typename KokkosToDesulMemoryOrder<MemOrderSuccess>::type(),
260  typename KokkosToDesulMemoryOrder<MemOrderFailure>::type(),
261  KOKKOS_DESUL_MEM_SCOPE);
262 
263  }
264  template<class T, class MemoryOrder>
265  KOKKOS_INLINE_FUNCTION
266  T atomic_load(const T* const src, MemoryOrder) {
267  return desul::atomic_load(src, typename KokkosToDesulMemoryOrder<MemoryOrder>::type(), KOKKOS_DESUL_MEM_SCOPE);
268  }
269  template<class T, class MemoryOrder>
270  KOKKOS_INLINE_FUNCTION
271  void atomic_store(T* const src, const T val, MemoryOrder) {
272  return desul::atomic_store(src, val, typename KokkosToDesulMemoryOrder<MemoryOrder>::type(), KOKKOS_DESUL_MEM_SCOPE);
273  }
274 }
275 
276 }
277 
278 #undef KOKKOS_DESUL_MEM_SCOPE
279 
280 // clang-format on
281 #endif // KOKKOS_ENABLE_IMPL_DESUL_ATOMICS
282 #endif
Kokkos
Definition: dummy.cpp:17
Generated by   doxygen 1.8.14