30 #ifndef _GLIBCXX_ATOMIC_WAIT_H
31 #define _GLIBCXX_ATOMIC_WAIT_H 1
33 #pragma GCC system_header
36 #if defined _GLIBCXX_HAS_GTHREADS || defined _GLIBCXX_HAVE_LINUX_FUTEX
38 #include <bits/gthr.h>
41 #ifdef _GLIBCXX_HAVE_LINUX_FUTEX
51 #define __cpp_lib_atomic_wait 201907L
53 namespace std _GLIBCXX_VISIBILITY(default)
55 _GLIBCXX_BEGIN_NAMESPACE_VERSION
58 #ifdef _GLIBCXX_HAVE_LINUX_FUTEX
59 using __platform_wait_t = int;
60 static constexpr
size_t __platform_wait_alignment = 4;
62 using __platform_wait_t = uint64_t;
63 static constexpr
size_t __platform_wait_alignment
64 = __alignof__(__platform_wait_t);
68 template<
typename _Tp>
69 inline constexpr
bool __platform_wait_uses_type
70 #ifdef _GLIBCXX_HAVE_PLATFORM_WAIT
72 && ((
sizeof(_Tp) ==
sizeof(__detail::__platform_wait_t))
73 && (
alignof(_Tp*) >= __platform_wait_alignment));
80 #ifdef _GLIBCXX_HAVE_LINUX_FUTEX
81 #define _GLIBCXX_HAVE_PLATFORM_WAIT 1
82 enum class __futex_wait_flags : int
84 #ifdef _GLIBCXX_HAVE_LINUX_FUTEX_PRIVATE
93 __wait_private = __wait | __private_flag,
94 __wake_private = __wake | __private_flag,
95 __wait_bitset_private = __wait_bitset | __private_flag,
96 __wake_bitset_private = __wake_bitset | __private_flag,
97 __bitset_match_any = -1
100 template<
typename _Tp>
102 __platform_wait(
const _Tp* __addr, __platform_wait_t __val) noexcept
104 auto __e = syscall (SYS_futex, static_cast<const void*>(__addr),
105 static_cast<int>(__futex_wait_flags::__wait_private),
107 if (!__e || errno == EAGAIN)
110 __throw_system_error(errno);
113 template<
typename _Tp>
115 __platform_notify(
const _Tp* __addr,
bool __all) noexcept
117 syscall (SYS_futex, static_cast<const void*>(__addr),
118 static_cast<int>(__futex_wait_flags::__wake_private),
119 __all ? INT_MAX : 1);
129 __thread_yield() noexcept
131 #if defined _GLIBCXX_HAS_GTHREADS && defined _GLIBCXX_USE_SCHED_YIELD
137 __thread_relax() noexcept
139 #if defined __i386__ || defined __x86_64__
140 __builtin_ia32_pause();
146 constexpr
auto __atomic_spin_count_1 = 12;
147 constexpr
auto __atomic_spin_count_2 = 4;
149 struct __default_spin_policy
152 operator()() const noexcept
156 template<
typename _Pred,
157 typename _Spin = __default_spin_policy>
159 __atomic_spin(_Pred& __pred, _Spin __spin = _Spin{ }) noexcept
161 for (
auto __i = 0; __i < __atomic_spin_count_1; ++__i)
165 __detail::__thread_relax();
168 for (
auto __i = 0; __i < __atomic_spin_count_2; ++__i)
172 __detail::__thread_yield();
185 template<
typename _Tp>
186 bool __atomic_compare(
const _Tp& __a,
const _Tp& __b)
189 return __builtin_memcmp(&__a, &__b,
sizeof(_Tp)) == 0;
192 struct __waiter_pool_base
194 #ifdef __cpp_lib_hardware_interference_size
195 static constexpr
auto _S_align = hardware_destructive_interference_size;
197 static constexpr
auto _S_align = 64;
200 alignas(_S_align) __platform_wait_t _M_wait = 0;
202 #ifndef _GLIBCXX_HAVE_PLATFORM_WAIT
206 alignas(_S_align) __platform_wait_t _M_ver = 0;
208 #ifndef _GLIBCXX_HAVE_PLATFORM_WAIT
211 __waiter_pool_base() =
default;
214 _M_enter_wait() noexcept
215 { __atomic_fetch_add(&_M_wait, 1, __ATOMIC_ACQ_REL); }
218 _M_leave_wait() noexcept
219 { __atomic_fetch_sub(&_M_wait, 1, __ATOMIC_ACQ_REL); }
222 _M_waiting() const noexcept
224 __platform_wait_t __res;
225 __atomic_load(&_M_wait, &__res, __ATOMIC_ACQUIRE);
230 _M_notify(
const __platform_wait_t* __addr,
bool __all,
bool __bare) noexcept
232 if (!(__bare || _M_waiting()))
235 #ifdef _GLIBCXX_HAVE_PLATFORM_WAIT
236 __platform_notify(__addr, __all);
245 static __waiter_pool_base&
246 _S_for(
const void* __addr) noexcept
248 constexpr uintptr_t __ct = 16;
249 static __waiter_pool_base __w[__ct];
250 auto __key = (uintptr_t(__addr) >> 2) % __ct;
255 struct __waiter_pool : __waiter_pool_base
258 _M_do_wait(
const __platform_wait_t* __addr, __platform_wait_t __old) noexcept
260 #ifdef _GLIBCXX_HAVE_PLATFORM_WAIT
261 __platform_wait(__addr, __old);
263 __platform_wait_t __val;
264 __atomic_load(__addr, &__val, __ATOMIC_RELAXED);
267 lock_guard<mutex> __l(_M_mtx);
270 #endif // __GLIBCXX_HAVE_PLATFORM_WAIT
274 template<
typename _Tp>
277 using __waiter_type = _Tp;
280 __platform_wait_t* _M_addr;
282 template<
typename _Up>
283 static __platform_wait_t*
284 _S_wait_addr(
const _Up* __a, __platform_wait_t* __b)
286 if constexpr (__platform_wait_uses_type<_Up>)
287 return reinterpret_cast<__platform_wait_t*>(const_cast<_Up*>(__a));
292 static __waiter_type&
293 _S_for(const
void* __addr) noexcept
295 static_assert(
sizeof(__waiter_type) ==
sizeof(__waiter_pool_base));
296 auto& res = __waiter_pool_base::_S_for(__addr);
297 return reinterpret_cast<__waiter_type&
>(res);
300 template<
typename _Up>
301 explicit __waiter_base(
const _Up* __addr) noexcept
302 : _M_w(_S_for(__addr))
303 , _M_addr(_S_wait_addr(__addr, &_M_w._M_ver))
308 {
return _M_addr == &_M_w._M_ver; }
311 _M_notify(
bool __all,
bool __bare =
false)
315 __atomic_fetch_add(_M_addr, 1, __ATOMIC_ACQ_REL);
318 _M_w._M_notify(_M_addr, __all, __bare);
321 template<
typename _Up,
typename _ValFn,
322 typename _Spin = __default_spin_policy>
324 _S_do_spin_v(__platform_wait_t* __addr,
325 const _Up& __old, _ValFn __vfn,
326 __platform_wait_t& __val,
327 _Spin __spin = _Spin{ })
329 auto const __pred = [=]
330 {
return !__detail::__atomic_compare(__old, __vfn()); };
332 if constexpr (__platform_wait_uses_type<_Up>)
338 __atomic_load(__addr, &__val, __ATOMIC_RELAXED);
340 return __atomic_spin(__pred, __spin);
343 template<
typename _Up,
typename _ValFn,
344 typename _Spin = __default_spin_policy>
346 _M_do_spin_v(
const _Up& __old, _ValFn __vfn,
347 __platform_wait_t& __val,
348 _Spin __spin = _Spin{ })
349 {
return _S_do_spin_v(_M_addr, __old, __vfn, __val, __spin); }
351 template<
typename _Pred,
352 typename _Spin = __default_spin_policy>
354 _S_do_spin(
const __platform_wait_t* __addr,
356 __platform_wait_t& __val,
357 _Spin __spin = _Spin{ })
359 __atomic_load(__addr, &__val, __ATOMIC_RELAXED);
360 return __atomic_spin(__pred, __spin);
363 template<
typename _Pred,
364 typename _Spin = __default_spin_policy>
366 _M_do_spin(_Pred __pred, __platform_wait_t& __val,
367 _Spin __spin = _Spin{ })
368 {
return _S_do_spin(_M_addr, __pred, __val, __spin); }
371 template<
typename _EntersWait>
372 struct __waiter : __waiter_base<__waiter_pool>
374 using __base_type = __waiter_base<__waiter_pool>;
376 template<
typename _Tp>
377 explicit __waiter(
const _Tp* __addr) noexcept
378 : __base_type(__addr)
380 if constexpr (_EntersWait::value)
381 _M_w._M_enter_wait();
386 if constexpr (_EntersWait::value)
387 _M_w._M_leave_wait();
390 template<typename _Tp, typename _ValFn>
392 _M_do_wait_v(_Tp __old, _ValFn __vfn)
394 __platform_wait_t __val;
395 if (__base_type::_M_do_spin_v(__old, __vfn, __val))
400 __base_type::_M_w._M_do_wait(__base_type::_M_addr, __val);
402 while (__detail::__atomic_compare(__old, __vfn()));
405 template<
typename _Pred>
407 _M_do_wait(_Pred __pred) noexcept
411 __platform_wait_t __val;
412 if (__base_type::_M_do_spin(__pred, __val))
414 __base_type::_M_w._M_do_wait(__base_type::_M_addr, __val);
420 using __enters_wait = __waiter<std::true_type>;
421 using __bare_wait = __waiter<std::false_type>;
424 template<
typename _Tp,
typename _ValFn>
426 __atomic_wait_address_v(
const _Tp* __addr, _Tp __old,
427 _ValFn __vfn) noexcept
429 __detail::__enters_wait __w(__addr);
430 __w._M_do_wait_v(__old, __vfn);
433 template<
typename _Tp,
typename _Pred>
435 __atomic_wait_address(
const _Tp* __addr, _Pred __pred) noexcept
437 __detail::__enters_wait __w(__addr);
438 __w._M_do_wait(__pred);
442 template<
typename _Pred>
444 __atomic_wait_address_bare(
const __detail::__platform_wait_t* __addr,
445 _Pred __pred) noexcept
447 #ifdef _GLIBCXX_HAVE_PLATFORM_WAIT
450 __detail::__platform_wait_t __val;
451 if (__detail::__bare_wait::_S_do_spin(__addr, __pred, __val))
453 __detail::__platform_wait(__addr, __val);
456 #else // !_GLIBCXX_HAVE_PLATFORM_WAIT
457 __detail::__bare_wait __w(__addr);
458 __w._M_do_wait(__pred);
462 template<
typename _Tp>
464 __atomic_notify_address(
const _Tp* __addr,
bool __all) noexcept
466 __detail::__bare_wait __w(__addr);
467 __w._M_notify(__all);
472 __atomic_notify_address_bare(
const __detail::__platform_wait_t* __addr,
475 #ifdef _GLIBCXX_HAVE_PLATFORM_WAIT
476 __detail::__platform_notify(__addr, __all);
478 __detail::__bare_wait __w(__addr);
479 __w._M_notify(__all,
true);
482 _GLIBCXX_END_NAMESPACE_VERSION
484 #endif // GTHREADS || LINUX_FUTEX
485 #endif // _GLIBCXX_ATOMIC_WAIT_H