//===----------------------------------------------------------------------===//
//
// Part of libcu++, the C++ Standard Library for your entire system,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//

#if defined(__CUDA_MINIMUM_ARCH__)                                   \
  && ((!defined(_CCCL_COMPILER_MSVC) && __CUDA_MINIMUM_ARCH__ < 600) \
      || (defined(_CCCL_COMPILER_MSVC) && __CUDA_MINIMUM_ARCH__ < 700))
#  error "CUDA atomics are only supported for sm_60 and up on *nix and sm_70 and up on Windows."
#endif

inline _CCCL_HOST_DEVICE int __stronger_order_cuda(int __a, int __b)
{
  int const __max = __a > __b ? __a : __b;
  if (__max != __ATOMIC_RELEASE)
  {
    return __max;
  }
  static int const __xform[] = {__ATOMIC_RELEASE, __ATOMIC_ACQ_REL, __ATOMIC_ACQ_REL, __ATOMIC_RELEASE};
  return __xform[__a < __b ? __a : __b];
}

// pre-define lock free query for heterogeneous compatibility
#ifndef _LIBCUDACXX_ATOMIC_IS_LOCK_FREE
#  define _LIBCUDACXX_ATOMIC_IS_LOCK_FREE(__x) (__x <= 8)
#endif

// Wrap host atomic implementations into a sub-namespace
namespace __host
{
#if defined(_CCCL_COMPILER_MSVC)
#  include <cuda/std/detail/libcxx/include/support/atomic/atomic_msvc.h>
#elif defined(_LIBCUDACXX_HAS_GCC_ATOMIC_IMP)
#  include <cuda/std/detail/libcxx/include/support/atomic/atomic_gcc.h>
#elif defined(_LIBCUDACXX_HAS_C11_ATOMIC_IMP)
// TODO
//  #  include <cuda/std/detail/libcxx/include/support/atomic/atomic_c11.h>
#elif defined(_CCCL_COMPILER_NVRTC)
#  include <cuda/std/detail/libcxx/include/support/atomic/atomic_nvrtc.h>
#endif
} // namespace __host

using __host::__cxx_atomic_underlying_t;

#include <cuda/std/detail/libcxx/include/support/atomic/atomic_cuda_derived.h>
#include <cuda/std/detail/libcxx/include/support/atomic/atomic_cuda_generated.h>

_CCCL_HOST_DEVICE inline void __cxx_atomic_thread_fence(memory_order __order)
{
  NV_DISPATCH_TARGET(
    NV_IS_DEVICE,
    (__atomic_thread_fence_cuda(static_cast<__memory_order_underlying_t>(__order), __thread_scope_system_tag());),
    NV_IS_HOST,
    (__host::__cxx_atomic_thread_fence(__order);))
}

_CCCL_HOST_DEVICE inline void __cxx_atomic_signal_fence(memory_order __order)
{
  NV_DISPATCH_TARGET(NV_IS_DEVICE,
                     (__atomic_signal_fence_cuda(static_cast<__memory_order_underlying_t>(__order));),
                     NV_IS_HOST,
                     (__host::__cxx_atomic_signal_fence(__order);))
}

template <typename _Tp, int _Sco, bool _Ref = false>
struct __cxx_atomic_base_heterogeneous_impl
{
  __cxx_atomic_base_heterogeneous_impl() noexcept = default;

  _CCCL_HOST_DEVICE constexpr explicit __cxx_atomic_base_heterogeneous_impl(_Tp __value)
      : __a_value(__value)
  {}

  using __underlying_t       = _Tp;
  static constexpr int __sco = _Sco;

  __host::__cxx_atomic_base_impl<_Tp, _Sco> __a_value;
};

template <typename _Tp, int _Sco>
struct __cxx_atomic_base_heterogeneous_impl<_Tp, _Sco, true>
{
  __cxx_atomic_base_heterogeneous_impl() noexcept = default;

  static_assert(sizeof(_Tp) >= 4, "atomic_ref does not support 1 or 2 byte types");
  static_assert(sizeof(_Tp) <= 8, "atomic_ref does not support types larger than 8 bytes");

  _CCCL_HOST_DEVICE constexpr explicit __cxx_atomic_base_heterogeneous_impl(_Tp& __value)
      : __a_value(__value)
  {}

  using __underlying_t       = _Tp;
  static constexpr int __sco = _Sco;

  __host::__cxx_atomic_ref_base_impl<_Tp, _Sco> __a_value;
};

template <typename _Tp, int _Sco, bool _Ref>
_CCCL_HOST_DEVICE constexpr _Tp*
__cxx_get_underlying_device_atomic(__cxx_atomic_base_heterogeneous_impl<_Tp, _Sco, _Ref>* __a) noexcept
{
  return __cxx_get_underlying_atomic(&__a->__a_value);
}

template <typename _Tp, int _Sco, bool _Ref>
_CCCL_HOST_DEVICE constexpr volatile _Tp*
__cxx_get_underlying_device_atomic(__cxx_atomic_base_heterogeneous_impl<_Tp, _Sco, _Ref> volatile* __a) noexcept
{
  return __cxx_get_underlying_atomic(&__a->__a_value);
}

template <typename _Tp, int _Sco, bool _Ref>
_CCCL_HOST_DEVICE constexpr const _Tp*
__cxx_get_underlying_device_atomic(__cxx_atomic_base_heterogeneous_impl<_Tp, _Sco, _Ref> const* __a) noexcept
{
  return __cxx_get_underlying_atomic(&__a->__a_value);
}

template <typename _Tp, int _Sco, bool _Ref>
_CCCL_HOST_DEVICE constexpr const volatile _Tp*
__cxx_get_underlying_device_atomic(__cxx_atomic_base_heterogeneous_impl<_Tp, _Sco, _Ref> const volatile* __a) noexcept
{
  return __cxx_get_underlying_atomic(&__a->__a_value);
}

template <typename _Tp>
using __cxx_atomic_small_to_32 = __conditional_t<is_signed<_Tp>::value, int32_t, uint32_t>;

// Arithmetic conversions to/from proxy types
template <class _Tp, __enable_if_t<is_arithmetic<_Tp>::value, int> = 0>
constexpr _CCCL_HOST_DEVICE inline __cxx_atomic_small_to_32<_Tp> __cxx_small_to_32(_Tp __val)
{
  return static_cast<__cxx_atomic_small_to_32<_Tp>>(__val);
}

template <class _Tp, __enable_if_t<is_arithmetic<_Tp>::value, int> = 0>
constexpr _CCCL_HOST_DEVICE inline _Tp __cxx_small_from_32(__cxx_atomic_small_to_32<_Tp> __val)
{
  return static_cast<_Tp>(__val);
}

// Non-arithmetic conversion to/from proxy types
template <class _Tp, __enable_if_t<!is_arithmetic<_Tp>::value, int> = 0>
_CCCL_HOST_DEVICE inline __cxx_atomic_small_to_32<_Tp> __cxx_small_to_32(_Tp __val)
{
  __cxx_atomic_small_to_32<_Tp> __temp{};
  memcpy(&__temp, &__val, sizeof(_Tp));
  return __temp;
}

template <class _Tp, __enable_if_t<!is_arithmetic<_Tp>::value, int> = 0>
_CCCL_HOST_DEVICE inline _Tp __cxx_small_from_32(__cxx_atomic_small_to_32<_Tp> __val)
{
  _Tp __temp{};
  memcpy(&__temp, &__val, sizeof(_Tp));
  return __temp;
}

template <typename _Tp, int _Sco>
struct __cxx_atomic_base_small_impl
{
  __cxx_atomic_base_small_impl() noexcept = default;
  _CCCL_HOST_DEVICE constexpr explicit __cxx_atomic_base_small_impl(_Tp __value)
      : __a_value(__cxx_small_to_32(__value))
  {}

  using __underlying_t       = _Tp;
  static constexpr int __sco = _Sco;

  __cxx_atomic_base_heterogeneous_impl<__cxx_atomic_small_to_32<_Tp>, _Sco, false> __a_value;
};

template <typename _Tp, int _Sco>
using __cxx_atomic_base_impl =
  __conditional_t<sizeof(_Tp) < 4,
                  __cxx_atomic_base_small_impl<_Tp, _Sco>,
                  __cxx_atomic_base_heterogeneous_impl<_Tp, _Sco>>;

template <typename _Tp, int _Sco>
using __cxx_atomic_ref_base_impl = __cxx_atomic_base_heterogeneous_impl<_Tp, _Sco, true>;

template <typename _Tp, int _Sco, bool _Ref>
_CCCL_HOST_DEVICE void __cxx_atomic_init(__cxx_atomic_base_heterogeneous_impl<_Tp, _Sco, _Ref>* __a, _Tp __val)
{
  alignas(_Tp) auto __tmp = __val;
  __cxx_atomic_assign_volatile(*__cxx_get_underlying_device_atomic(__a), __tmp);
}

template <typename _Tp, int _Sco, bool _Ref>
_CCCL_HOST_DEVICE void __cxx_atomic_init(__cxx_atomic_base_heterogeneous_impl<_Tp, _Sco, _Ref> volatile* __a, _Tp __val)
{
  alignas(_Tp) auto __tmp = __val;
  __cxx_atomic_assign_volatile(*__cxx_get_underlying_device_atomic(__a), __tmp);
}

template <typename _Tp, int _Sco, bool _Ref>
_CCCL_HOST_DEVICE void
__cxx_atomic_store(__cxx_atomic_base_heterogeneous_impl<_Tp, _Sco, _Ref>* __a, _Tp __val, memory_order __order)
{
  alignas(_Tp) auto __tmp = __val;
  NV_DISPATCH_TARGET(
    NV_IS_DEVICE,
    (__atomic_store_n_cuda(__cxx_get_underlying_device_atomic(__a),
                           __tmp,
                           static_cast<__memory_order_underlying_t>(__order),
                           __scope_tag<_Sco>());),
    NV_IS_HOST,
    (__host::__cxx_atomic_store(&__a->__a_value, __tmp, __order);))
}

template <typename _Tp, int _Sco, bool _Ref>
_CCCL_HOST_DEVICE void
__cxx_atomic_store(__cxx_atomic_base_heterogeneous_impl<_Tp, _Sco, _Ref> volatile* __a, _Tp __val, memory_order __order)
{
  alignas(_Tp) auto __tmp = __val;
  NV_DISPATCH_TARGET(
    NV_IS_DEVICE,
    (__atomic_store_n_cuda(__cxx_get_underlying_device_atomic(__a),
                           __tmp,
                           static_cast<__memory_order_underlying_t>(__order),
                           __scope_tag<_Sco>());),
    NV_IS_HOST,
    (__host::__cxx_atomic_store(&__a->__a_value, __tmp, __order);))
}

template <typename _Tp, int _Sco, bool _Ref>
_CCCL_HOST_DEVICE _Tp
__cxx_atomic_load(__cxx_atomic_base_heterogeneous_impl<_Tp, _Sco, _Ref> const* __a, memory_order __order)
{
  NV_DISPATCH_TARGET(
    NV_IS_DEVICE,
    (return __atomic_load_n_cuda(__cxx_get_underlying_device_atomic(__a),
                                 static_cast<__memory_order_underlying_t>(__order),
                                 __scope_tag<_Sco>());),
    NV_IS_HOST,
    (return __host::__cxx_atomic_load(&__a->__a_value, __order);))
}

template <typename _Tp, int _Sco, bool _Ref>
_CCCL_HOST_DEVICE _Tp
__cxx_atomic_load(__cxx_atomic_base_heterogeneous_impl<_Tp, _Sco, _Ref> const volatile* __a, memory_order __order)
{
  NV_DISPATCH_TARGET(
    NV_IS_DEVICE,
    (return __atomic_load_n_cuda(__cxx_get_underlying_device_atomic(__a),
                                 static_cast<__memory_order_underlying_t>(__order),
                                 __scope_tag<_Sco>());),
    NV_IS_HOST,
    (return __host::__cxx_atomic_load(&__a->__a_value, __order);))
}

template <typename _Tp, int _Sco, bool _Ref>
_CCCL_HOST_DEVICE _Tp
__cxx_atomic_exchange(__cxx_atomic_base_heterogeneous_impl<_Tp, _Sco, _Ref>* __a, _Tp __val, memory_order __order)
{
  alignas(_Tp) auto __tmp = __val;
  NV_DISPATCH_TARGET(
    NV_IS_DEVICE,
    (return __atomic_exchange_n_cuda(__cxx_get_underlying_device_atomic(__a),
                                     __tmp,
                                     static_cast<__memory_order_underlying_t>(__order),
                                     __scope_tag<_Sco>());),
    NV_IS_HOST,
    (return __host::__cxx_atomic_exchange(&__a->__a_value, __tmp, __order);))
}

template <typename _Tp, int _Sco, bool _Ref>
_CCCL_HOST_DEVICE _Tp __cxx_atomic_exchange(
  __cxx_atomic_base_heterogeneous_impl<_Tp, _Sco, _Ref> volatile* __a, _Tp __val, memory_order __order)
{
  alignas(_Tp) auto __tmp = __val;
  NV_DISPATCH_TARGET(
    NV_IS_DEVICE,
    (return __atomic_exchange_n_cuda(__cxx_get_underlying_device_atomic(__a),
                                     __tmp,
                                     static_cast<__memory_order_underlying_t>(__order),
                                     __scope_tag<_Sco>());),
    NV_IS_HOST,
    (return __host::__cxx_atomic_exchange(&__a->__a_value, __tmp, __order);))
}

template <typename _Tp, int _Sco, bool _Ref>
_CCCL_HOST_DEVICE bool __cxx_atomic_compare_exchange_strong(
  __cxx_atomic_base_heterogeneous_impl<_Tp, _Sco, _Ref>* __a,
  _Tp* __expected,
  _Tp __val,
  memory_order __success,
  memory_order __failure)
{
  alignas(_Tp) auto __tmp = *__expected;
  bool __result           = false;
  NV_DISPATCH_TARGET(
    NV_IS_DEVICE,
    (alignas(_Tp) auto __tmp_v = __val;
     __result                  = __atomic_compare_exchange_cuda(
       __cxx_get_underlying_device_atomic(__a),
       &__tmp,
       &__tmp_v,
       false,
       static_cast<__memory_order_underlying_t>(__success),
       static_cast<__memory_order_underlying_t>(__failure),
       __scope_tag<_Sco>());),
    NV_IS_HOST,
    (__result = __host::__cxx_atomic_compare_exchange_strong(&__a->__a_value, &__tmp, __val, __success, __failure);))
  *__expected = __tmp;
  return __result;
}

template <typename _Tp, int _Sco, bool _Ref>
_CCCL_HOST_DEVICE bool __cxx_atomic_compare_exchange_strong(
  __cxx_atomic_base_heterogeneous_impl<_Tp, _Sco, _Ref> volatile* __a,
  _Tp* __expected,
  _Tp __val,
  memory_order __success,
  memory_order __failure)
{
  alignas(_Tp) auto __tmp = *__expected;
  bool __result           = false;
  NV_DISPATCH_TARGET(
    NV_IS_DEVICE,
    (alignas(_Tp) auto __tmp_v = __val;
     __result                  = __atomic_compare_exchange_cuda(
       __cxx_get_underlying_device_atomic(__a),
       &__tmp,
       &__tmp_v,
       false,
       static_cast<__memory_order_underlying_t>(__success),
       static_cast<__memory_order_underlying_t>(__failure),
       __scope_tag<_Sco>());),
    NV_IS_HOST,
    (__result = __host::__cxx_atomic_compare_exchange_strong(&__a->__a_value, &__tmp, __val, __success, __failure);))
  *__expected = __tmp;
  return __result;
}

template <typename _Tp, int _Sco, bool _Ref>
_CCCL_HOST_DEVICE bool __cxx_atomic_compare_exchange_weak(
  __cxx_atomic_base_heterogeneous_impl<_Tp, _Sco, _Ref>* __a,
  _Tp* __expected,
  _Tp __val,
  memory_order __success,
  memory_order __failure)
{
  alignas(_Tp) auto __tmp = *__expected;
  bool __result           = false;
  NV_DISPATCH_TARGET(
    NV_IS_DEVICE,
    (alignas(_Tp) auto __tmp_v = __val;
     __result                  = __atomic_compare_exchange_cuda(
       __cxx_get_underlying_device_atomic(__a),
       &__tmp,
       &__tmp_v,
       true,
       static_cast<__memory_order_underlying_t>(__success),
       static_cast<__memory_order_underlying_t>(__failure),
       __scope_tag<_Sco>());),
    NV_IS_HOST,
    (__result = __host::__cxx_atomic_compare_exchange_weak(&__a->__a_value, &__tmp, __val, __success, __failure);))
  *__expected = __tmp;
  return __result;
}

template <typename _Tp, int _Sco, bool _Ref>
_CCCL_HOST_DEVICE bool __cxx_atomic_compare_exchange_weak(
  __cxx_atomic_base_heterogeneous_impl<_Tp, _Sco, _Ref> volatile* __a,
  _Tp* __expected,
  _Tp __val,
  memory_order __success,
  memory_order __failure)
{
  alignas(_Tp) auto __tmp = *__expected;
  bool __result           = false;
  NV_DISPATCH_TARGET(
    NV_IS_DEVICE,
    (alignas(_Tp) auto __tmp_v = __val;
     __result                  = __atomic_compare_exchange_cuda(
       __cxx_get_underlying_device_atomic(__a),
       &__tmp,
       &__tmp_v,
       true,
       static_cast<__memory_order_underlying_t>(__success),
       static_cast<__memory_order_underlying_t>(__failure),
       __scope_tag<_Sco>());),
    NV_IS_HOST,
    (__result = __host::__cxx_atomic_compare_exchange_weak(&__a->__a_value, &__tmp, __val, __success, __failure);))
  *__expected = __tmp;
  return __result;
}

template <typename _Tp, int _Sco, bool _Ref>
_CCCL_HOST_DEVICE _Tp
__cxx_atomic_fetch_add(__cxx_atomic_base_heterogeneous_impl<_Tp, _Sco, _Ref>* __a, _Tp __delta, memory_order __order)
{
  NV_DISPATCH_TARGET(
    NV_IS_DEVICE,
    (return __atomic_fetch_add_cuda(__cxx_get_underlying_device_atomic(__a),
                                    __delta,
                                    static_cast<__memory_order_underlying_t>(__order),
                                    __scope_tag<_Sco>());),
    NV_IS_HOST,
    (return __host::__cxx_atomic_fetch_add(&__a->__a_value, __delta, __order);))
}

template <typename _Tp, int _Sco, bool _Ref>
_CCCL_HOST_DEVICE _Tp __cxx_atomic_fetch_add(
  __cxx_atomic_base_heterogeneous_impl<_Tp, _Sco, _Ref> volatile* __a, _Tp __delta, memory_order __order)
{
  NV_DISPATCH_TARGET(
    NV_IS_DEVICE,
    (return __atomic_fetch_add_cuda(__cxx_get_underlying_device_atomic(__a),
                                    __delta,
                                    static_cast<__memory_order_underlying_t>(__order),
                                    __scope_tag<_Sco>());),
    NV_IS_HOST,
    (return __host::__cxx_atomic_fetch_add(&__a->__a_value, __delta, __order);))
}

template <typename _Tp, int _Sco, bool _Ref>
_CCCL_HOST_DEVICE _Tp* __cxx_atomic_fetch_add(
  __cxx_atomic_base_heterogeneous_impl<_Tp*, _Sco, _Ref>* __a, ptrdiff_t __delta, memory_order __order)
{
  NV_DISPATCH_TARGET(
    NV_IS_DEVICE,
    (return __atomic_fetch_add_cuda(__cxx_get_underlying_device_atomic(__a),
                                    __delta,
                                    static_cast<__memory_order_underlying_t>(__order),
                                    __scope_tag<_Sco>());),
    NV_IS_HOST,
    (return __host::__cxx_atomic_fetch_add(&__a->__a_value, __delta, __order);))
}

template <typename _Tp, int _Sco, bool _Ref>
_CCCL_HOST_DEVICE _Tp* __cxx_atomic_fetch_add(
  __cxx_atomic_base_heterogeneous_impl<_Tp*, _Sco, _Ref> volatile* __a, ptrdiff_t __delta, memory_order __order)
{
  NV_DISPATCH_TARGET(
    NV_IS_DEVICE,
    (return __atomic_fetch_add_cuda(__cxx_get_underlying_device_atomic(__a),
                                    __delta,
                                    static_cast<__memory_order_underlying_t>(__order),
                                    __scope_tag<_Sco>());),
    NV_IS_HOST,
    (return __host::__cxx_atomic_fetch_add(&__a->__a_value, __delta, __order);))
}

template <typename _Tp, int _Sco, bool _Ref>
_CCCL_HOST_DEVICE _Tp
__cxx_atomic_fetch_sub(__cxx_atomic_base_heterogeneous_impl<_Tp, _Sco, _Ref>* __a, _Tp __delta, memory_order __order)
{
  NV_DISPATCH_TARGET(
    NV_IS_DEVICE,
    (return __atomic_fetch_sub_cuda(__cxx_get_underlying_device_atomic(__a),
                                    __delta,
                                    static_cast<__memory_order_underlying_t>(__order),
                                    __scope_tag<_Sco>());),
    NV_IS_HOST,
    (return __host::__cxx_atomic_fetch_sub(&__a->__a_value, __delta, __order);))
}

template <typename _Tp, int _Sco, bool _Ref>
_CCCL_HOST_DEVICE _Tp __cxx_atomic_fetch_sub(
  __cxx_atomic_base_heterogeneous_impl<_Tp, _Sco, _Ref> volatile* __a, _Tp __delta, memory_order __order)
{
  NV_DISPATCH_TARGET(
    NV_IS_DEVICE,
    (return __atomic_fetch_sub_cuda(__cxx_get_underlying_device_atomic(__a),
                                    __delta,
                                    static_cast<__memory_order_underlying_t>(__order),
                                    __scope_tag<_Sco>());),
    NV_IS_HOST,
    (return __host::__cxx_atomic_fetch_sub(&__a->__a_value, __delta, __order);))
}

template <typename _Tp, int _Sco, bool _Ref>
_CCCL_HOST_DEVICE _Tp* __cxx_atomic_fetch_sub(
  __cxx_atomic_base_heterogeneous_impl<_Tp*, _Sco, _Ref>* __a, ptrdiff_t __delta, memory_order __order)
{
  NV_DISPATCH_TARGET(
    NV_IS_DEVICE,
    (return __atomic_fetch_sub_cuda(__cxx_get_underlying_device_atomic(__a),
                                    __delta,
                                    static_cast<__memory_order_underlying_t>(__order),
                                    __scope_tag<_Sco>());),
    NV_IS_HOST,
    (return __host::__cxx_atomic_fetch_sub(&__a->__a_value, __delta, __order);))
}

template <typename _Tp, int _Sco, bool _Ref>
_CCCL_HOST_DEVICE _Tp* __cxx_atomic_fetch_sub(
  __cxx_atomic_base_heterogeneous_impl<_Tp*, _Sco, _Ref> volatile* __a, ptrdiff_t __delta, memory_order __order)
{
  NV_DISPATCH_TARGET(
    NV_IS_DEVICE,
    (return __atomic_fetch_sub_cuda(__cxx_get_underlying_device_atomic(__a),
                                    __delta,
                                    static_cast<__memory_order_underlying_t>(__order),
                                    __scope_tag<_Sco>());),
    NV_IS_HOST,
    (return __host::__cxx_atomic_fetch_sub(&__a->__a_value, __delta, __order);))
}

template <typename _Tp, int _Sco, bool _Ref>
_CCCL_HOST_DEVICE _Tp
__cxx_atomic_fetch_and(__cxx_atomic_base_heterogeneous_impl<_Tp, _Sco, _Ref>* __a, _Tp __pattern, memory_order __order)
{
  NV_DISPATCH_TARGET(
    NV_IS_DEVICE,
    (return __atomic_fetch_and_cuda(__cxx_get_underlying_device_atomic(__a),
                                    __pattern,
                                    static_cast<__memory_order_underlying_t>(__order),
                                    __scope_tag<_Sco>());),
    NV_IS_HOST,
    (return __host::__cxx_atomic_fetch_and(&__a->__a_value, __pattern, __order);))
}

template <typename _Tp, int _Sco, bool _Ref>
_CCCL_HOST_DEVICE _Tp __cxx_atomic_fetch_and(
  __cxx_atomic_base_heterogeneous_impl<_Tp, _Sco, _Ref> volatile* __a, _Tp __pattern, memory_order __order)
{
  NV_DISPATCH_TARGET(
    NV_IS_DEVICE,
    (return __atomic_fetch_and_cuda(__cxx_get_underlying_device_atomic(__a),
                                    __pattern,
                                    static_cast<__memory_order_underlying_t>(__order),
                                    __scope_tag<_Sco>());),
    NV_IS_HOST,
    (return __host::__cxx_atomic_fetch_and(&__a->__a_value, __pattern, __order);))
}

template <typename _Tp, int _Sco, bool _Ref>
_CCCL_HOST_DEVICE _Tp
__cxx_atomic_fetch_or(__cxx_atomic_base_heterogeneous_impl<_Tp, _Sco, _Ref>* __a, _Tp __pattern, memory_order __order)
{
  NV_DISPATCH_TARGET(
    NV_IS_DEVICE,
    (return __atomic_fetch_or_cuda(__cxx_get_underlying_device_atomic(__a),
                                   __pattern,
                                   static_cast<__memory_order_underlying_t>(__order),
                                   __scope_tag<_Sco>());),
    NV_IS_HOST,
    (return __host::__cxx_atomic_fetch_or(&__a->__a_value, __pattern, __order);))
}

template <typename _Tp, int _Sco, bool _Ref>
_CCCL_HOST_DEVICE _Tp __cxx_atomic_fetch_or(
  __cxx_atomic_base_heterogeneous_impl<_Tp, _Sco, _Ref> volatile* __a, _Tp __pattern, memory_order __order)
{
  NV_DISPATCH_TARGET(
    NV_IS_DEVICE,
    (return __atomic_fetch_or_cuda(__cxx_get_underlying_device_atomic(__a),
                                   __pattern,
                                   static_cast<__memory_order_underlying_t>(__order),
                                   __scope_tag<_Sco>());),
    NV_IS_HOST,
    (return __host::__cxx_atomic_fetch_or(&__a->__a_value, __pattern, __order);))
}

template <typename _Tp, int _Sco, bool _Ref>
_CCCL_HOST_DEVICE _Tp
__cxx_atomic_fetch_xor(__cxx_atomic_base_heterogeneous_impl<_Tp, _Sco, _Ref>* __a, _Tp __pattern, memory_order __order)
{
  NV_DISPATCH_TARGET(
    NV_IS_DEVICE,
    (return __atomic_fetch_xor_cuda(__cxx_get_underlying_device_atomic(__a),
                                    __pattern,
                                    static_cast<__memory_order_underlying_t>(__order),
                                    __scope_tag<_Sco>());),
    NV_IS_HOST,
    (return __host::__cxx_atomic_fetch_xor(&__a->__a_value, __pattern, __order);))
}

template <typename _Tp, int _Sco, bool _Ref>
_CCCL_HOST_DEVICE _Tp __cxx_atomic_fetch_xor(
  __cxx_atomic_base_heterogeneous_impl<_Tp, _Sco, _Ref> volatile* __a, _Tp __pattern, memory_order __order)
{
  NV_DISPATCH_TARGET(
    NV_IS_DEVICE,
    (return __atomic_fetch_xor_cuda(__cxx_get_underlying_device_atomic(__a),
                                    __pattern,
                                    static_cast<__memory_order_underlying_t>(__order),
                                    __scope_tag<_Sco>());),
    NV_IS_HOST,
    (return __host::__cxx_atomic_fetch_xor(&__a->__a_value, __pattern, __order);))
}

template <typename _Tp, typename _Delta, int _Sco, bool _Ref>
_CCCL_HOST_DEVICE _Tp
__cxx_atomic_fetch_max(__cxx_atomic_base_heterogeneous_impl<_Tp, _Sco, _Ref>* __a, _Delta __val, memory_order __order)
{
  NV_IF_TARGET(
    NV_IS_DEVICE,
    (return __atomic_fetch_max_cuda(__cxx_get_underlying_device_atomic(__a),
                                    __val,
                                    static_cast<__memory_order_underlying_t>(__order),
                                    __scope_tag<_Sco>());),
    (return __host::__cxx_atomic_fetch_max(&__a->__a_value, __val, __order);))
}

template <typename _Tp, typename _Delta, int _Sco, bool _Ref>
_CCCL_HOST_DEVICE _Tp __cxx_atomic_fetch_max(
  __cxx_atomic_base_heterogeneous_impl<_Tp, _Sco, _Ref> volatile* __a, _Delta __val, memory_order __order)
{
  NV_IF_TARGET(
    NV_IS_DEVICE,
    (return __atomic_fetch_max_cuda(__cxx_get_underlying_device_atomic(__a),
                                    __val,
                                    static_cast<__memory_order_underlying_t>(__order),
                                    __scope_tag<_Sco>());),
    (return __host::__cxx_atomic_fetch_max(&__a->__a_value, __val, __order);))
}

template <typename _Tp, typename _Delta, int _Sco, bool _Ref>
_CCCL_HOST_DEVICE _Tp
__cxx_atomic_fetch_min(__cxx_atomic_base_heterogeneous_impl<_Tp, _Sco, _Ref>* __a, _Delta __val, memory_order __order)
{
  NV_IF_TARGET(
    NV_IS_DEVICE,
    (return __atomic_fetch_min_cuda(__cxx_get_underlying_device_atomic(__a),
                                    __val,
                                    static_cast<__memory_order_underlying_t>(__order),
                                    __scope_tag<_Sco>());),
    (return __host::__cxx_atomic_fetch_min(&__a->__a_value, __val, __order);))
}

template <typename _Tp, typename _Delta, int _Sco, bool _Ref>
_CCCL_HOST_DEVICE _Tp __cxx_atomic_fetch_min(
  __cxx_atomic_base_heterogeneous_impl<_Tp, _Sco, _Ref> volatile* __a, _Delta __val, memory_order __order)
{
  NV_IF_TARGET(
    NV_IS_DEVICE,
    (return __atomic_fetch_min_cuda(__cxx_get_underlying_device_atomic(__a),
                                    __val,
                                    static_cast<__memory_order_underlying_t>(__order),
                                    __scope_tag<_Sco>());),
    (return __host::__cxx_atomic_fetch_min(&__a->__a_value, __val, __order);))
}

template <typename _Tp, int _Sco>
_CCCL_HOST_DEVICE inline void __cxx_atomic_init(__cxx_atomic_base_small_impl<_Tp, _Sco> volatile* __a, _Tp __val)
{
  __cxx_atomic_init(&__a->__a_value, __cxx_small_to_32(__val));
}

template <typename _Tp, int _Sco>
_CCCL_HOST_DEVICE inline void
__cxx_atomic_store(__cxx_atomic_base_small_impl<_Tp, _Sco> volatile* __a, _Tp __val, memory_order __order)
{
  __cxx_atomic_store(&__a->__a_value, __cxx_small_to_32(__val), __order);
}

template <typename _Tp, int _Sco>
_CCCL_HOST_DEVICE inline _Tp
__cxx_atomic_load(__cxx_atomic_base_small_impl<_Tp, _Sco> const volatile* __a, memory_order __order)
{
  return __cxx_small_from_32<_Tp>(__cxx_atomic_load(&__a->__a_value, __order));
}

template <typename _Tp, int _Sco>
_CCCL_HOST_DEVICE inline _Tp
__cxx_atomic_exchange(__cxx_atomic_base_small_impl<_Tp, _Sco> volatile* __a, _Tp __value, memory_order __order)
{
  return __cxx_small_from_32<_Tp>(__cxx_atomic_exchange(&__a->__a_value, __cxx_small_to_32(__value), __order));
}
_CCCL_HOST_DEVICE inline int __cuda_memcmp(void const* __lhs, void const* __rhs, size_t __count)
{
  NV_DISPATCH_TARGET(
    NV_IS_DEVICE,
    (auto __lhs_c = reinterpret_cast<unsigned char const*>(__lhs);
     auto __rhs_c = reinterpret_cast<unsigned char const*>(__rhs);
     while (__count--) {
       auto const __lhs_v = *__lhs_c++;
       auto const __rhs_v = *__rhs_c++;
       if (__lhs_v < __rhs_v)
       {
         return -1;
       }
       if (__lhs_v > __rhs_v)
       {
         return 1;
       }
     } return 0;),
    NV_IS_HOST,
    (return memcmp(__lhs, __rhs, __count);))
}

template <typename _Tp, int _Sco>
_CCCL_HOST_DEVICE inline bool __cxx_atomic_compare_exchange_weak(
  __cxx_atomic_base_small_impl<_Tp, _Sco> volatile* __a,
  _Tp* __expected,
  _Tp __value,
  memory_order __success,
  memory_order __failure)
{
  auto __temp = __cxx_small_to_32(*__expected);
  auto const __ret =
    __cxx_atomic_compare_exchange_weak(&__a->__a_value, &__temp, __cxx_small_to_32(__value), __success, __failure);
  auto const __actual   = __cxx_small_from_32<_Tp>(__temp);
  constexpr auto __mask = static_cast<decltype(__temp)>((1u << (8 * sizeof(_Tp))) - 1);
  if (!__ret)
  {
    if (0 == __cuda_memcmp(&__actual, __expected, sizeof(_Tp)))
    {
      __cxx_atomic_fetch_and(&__a->__a_value, __mask, memory_order_relaxed);
    }
    else
    {
      *__expected = __actual;
    }
  }
  return __ret;
}

template <typename _Tp, int _Sco>
_CCCL_HOST_DEVICE inline bool __cxx_atomic_compare_exchange_strong(
  __cxx_atomic_base_small_impl<_Tp, _Sco> volatile* __a,
  _Tp* __expected,
  _Tp __value,
  memory_order __success,
  memory_order __failure)
{
  auto const __old = *__expected;
  while (1)
  {
    if (__cxx_atomic_compare_exchange_weak(__a, __expected, __value, __success, __failure))
    {
      return true;
    }
    if (0 != __cuda_memcmp(&__old, __expected, sizeof(_Tp)))
    {
      return false;
    }
  }
}

template <typename _Tp, int _Sco>
_CCCL_HOST_DEVICE inline _Tp
__cxx_atomic_fetch_add(__cxx_atomic_base_small_impl<_Tp, _Sco> volatile* __a, _Tp __delta, memory_order __order)
{
  return __cxx_small_from_32<_Tp>(__cxx_atomic_fetch_add(&__a->__a_value, __cxx_small_to_32(__delta), __order));
}

template <typename _Tp, int _Sco>
_CCCL_HOST_DEVICE inline _Tp
__cxx_atomic_fetch_sub(__cxx_atomic_base_small_impl<_Tp, _Sco> volatile* __a, _Tp __delta, memory_order __order)
{
  return __cxx_small_from_32<_Tp>(__cxx_atomic_fetch_sub(&__a->__a_value, __cxx_small_to_32(__delta), __order));
}

template <typename _Tp, int _Sco>
_CCCL_HOST_DEVICE inline _Tp
__cxx_atomic_fetch_and(__cxx_atomic_base_small_impl<_Tp, _Sco> volatile* __a, _Tp __pattern, memory_order __order)
{
  return __cxx_small_from_32<_Tp>(__cxx_atomic_fetch_and(&__a->__a_value, __cxx_small_to_32(__pattern), __order));
}

template <typename _Tp, int _Sco>
_CCCL_HOST_DEVICE inline _Tp
__cxx_atomic_fetch_or(__cxx_atomic_base_small_impl<_Tp, _Sco> volatile* __a, _Tp __pattern, memory_order __order)
{
  return __cxx_small_from_32<_Tp>(__cxx_atomic_fetch_or(&__a->__a_value, __cxx_small_to_32(__pattern), __order));
}

template <typename _Tp, int _Sco>
_CCCL_HOST_DEVICE inline _Tp
__cxx_atomic_fetch_xor(__cxx_atomic_base_small_impl<_Tp, _Sco> volatile* __a, _Tp __pattern, memory_order __order)
{
  return __cxx_small_from_32<_Tp>(__cxx_atomic_fetch_xor(&__a->__a_value, __cxx_small_to_32(__pattern), __order));
}

template <typename _Tp, typename _Delta, int _Sco>
_CCCL_HOST_DEVICE inline _Tp
__cxx_atomic_fetch_max(__cxx_atomic_base_small_impl<_Tp, _Sco> volatile* __a, _Delta __val, memory_order __order)
{
  return __cxx_small_from_32<_Tp>(__cxx_atomic_fetch_max(&__a->__a_value, __cxx_small_to_32(__val), __order));
}

template <typename _Tp, typename _Delta, int _Sco>
_CCCL_HOST_DEVICE inline _Tp
__cxx_atomic_fetch_min(__cxx_atomic_base_small_impl<_Tp, _Sco> volatile* __a, _Delta __val, memory_order __order)
{
  return __cxx_small_from_32<_Tp>(__cxx_atomic_fetch_min(&__a->__a_value, __cxx_small_to_32(__val), __order));
}
