gasnet_atomicops.h

/*   $Source: github.com:BerkeleyLab/gasnet.git/gasnet_atomicops.h $
 * Description: GASNet header for portable atomic memory operations
 * Copyright 2002, Dan Bonachea <bonachea@cs.berkeley.edu>
 * Terms of use are as specified in license.txt
 */

#if !defined(_IN_GASNET_TOOLS_H) && !defined(_IN_GASNETEX_H)
  #error This file is not meant to be included directly- clients should include gasnetex.h or gasnet_tools.h
#endif

#ifndef _GASNET_ATOMICOPS_H
#define _GASNET_ATOMICOPS_H

/* ------------------------------------------------------------------------------------ */
/* Portable atomic operations
   --------------------------

   see README-tools for general usage information

   Signal safety of atomic operations
   ----------------------------------
   On most, but not all, platforms, operations on gasneti_atomic_t are signal safe.  
   On the few platforms where this is not the case GASNETI_ATOMICOPS_NOT_SIGNALSAFE
   will be defined to 1.

   Similarly, GASNETI_ATOMIC32_NOT_SIGNALSAFE and GASNETI_ATOMIC64_NOT_SIGNALSAFE
   are defined to 1 IFF the implementation of the fixed-width atomics is not signal-safe.
   Note that these two are set independently.

   Mutexes
   -------
   If GASNETI_USE_GENERIC_ATOMICOPS is defined, then the gasnet atomics are
   implemented using mutexes.  Therefore, one may wish to consider using other
   algorithms when this symbol is defined.

   Similarly, GASNETI_USE_GENERIC_ATOMIC32 and GASNETI_USE_GENERIC_ATOMIC64
   are defined to 1 IFF the implementation of the fixed-width atomics uses mutexes.
   Note that these two are set independently.

 */

/* ------------------------------------------------------------------------------------ */
/* Flags for memory fences */
#define GASNETI_ATOMIC_NONE			0x00
#define GASNETI_ATOMIC_RMB_PRE			0x01
#define GASNETI_ATOMIC_WMB_PRE			0x02  // MUST equal GEX_FLAG_AD_REL
#define GASNETI_ATOMIC_RMB_POST			0x04  // MUST equal GEX_FLAG_AD_ACQ
#define GASNETI_ATOMIC_WMB_POST			0x08
#define GASNETI_ATOMIC_RMB_POST_IF_TRUE		0x10
#define GASNETI_ATOMIC_RMB_POST_IF_FALSE	0x20

/* OR into flags to make the weak atomics omit fences in a non-threaded build. */
#define GASNETI_ATOMIC_WEAK_FENCE		0x80000000

#define GASNETI_ATOMIC_MB_PRE		(GASNETI_ATOMIC_WMB_PRE | GASNETI_ATOMIC_RMB_PRE)
#define GASNETI_ATOMIC_MB_POST		(GASNETI_ATOMIC_WMB_POST | GASNETI_ATOMIC_RMB_POST)

#define GASNETI_ATOMIC_REL		GASNETI_ATOMIC_WMB_PRE
#define GASNETI_ATOMIC_ACQ		GASNETI_ATOMIC_RMB_POST
#define GASNETI_ATOMIC_ACQ_IF_TRUE	GASNETI_ATOMIC_RMB_POST_IF_TRUE
#define GASNETI_ATOMIC_ACQ_IF_FALSE	GASNETI_ATOMIC_RMB_POST_IF_FALSE

/* ------------------------------------------------------------------------------------ */
/* Uniform memory fences for GASNet atomics.
 */

#define GASNETI_ATOMIC_MASK_PRE         (GASNETI_ATOMIC_WMB_PRE | GASNETI_ATOMIC_RMB_PRE)
#define GASNETI_ATOMIC_MASK_POST        (GASNETI_ATOMIC_WMB_POST | GASNETI_ATOMIC_RMB_POST)
#define GASNETI_ATOMIC_MASK_BOOL        (GASNETI_ATOMIC_MASK_POST | \
                                         GASNETI_ATOMIC_RMB_POST_IF_TRUE | \
                                         GASNETI_ATOMIC_RMB_POST_IF_FALSE)

#define _gasneti_atomic_cf_before(f)	if (f & GASNETI_ATOMIC_MASK_PRE) gasneti_compiler_fence();
#define _gasneti_atomic_cf_after(f)	if (f & GASNETI_ATOMIC_MASK_POST) gasneti_compiler_fence();
#define _gasneti_atomic_cf_bool(f)	if (f & GASNETI_ATOMIC_MASK_BOOL) gasneti_compiler_fence();

/* Fencing Part 1.  Removal of fences which are redundant on a given platform
 *	_gasneti_atomic_fence_{before,after}(flags)
 *	_gasneti_atomic_fence_bool(flags, value)
 *
 * This level of macros serves to remove at, preprocess-time, any tests
 * which are redundant due to the relationships among fences.  For example,
 * on a platform with a single fence instruction that is mb(), rmb() and
 * wmb() these macros will reduce three conditionals to just one.
 */

#if GASNETI_RMB_IS_MB && GASNETI_WMB_IS_MB
  /* Since mb() == rmb() == wmb() (including case that all are empty), only
   * a single check is needed for all three.
   */
  #define _gasneti_atomic_fence_before(f)       if (f & GASNETI_ATOMIC_MASK_PRE) gasneti_local_mb();
  #define _gasneti_atomic_fence_after(f)        if (f & GASNETI_ATOMIC_MASK_POST) gasneti_local_mb();
#elif GASNETI_MB_IS_SUM
  /* Since mb() == rmb()+wmb(), distinct rmb() and wmb() checks are
   * sufficient to implement a request for mb(), rmb() or wmb().
   * This includes the case where either is just a compiler fence.
   */
  #define _gasneti_atomic_fence_before(f)       if (f & GASNETI_ATOMIC_RMB_PRE) gasneti_local_rmb(); \
                                                if (f & GASNETI_ATOMIC_WMB_PRE) gasneti_local_wmb();
  #define _gasneti_atomic_fence_after(f)        if (f & GASNETI_ATOMIC_RMB_POST) gasneti_local_rmb(); \
                                                if (f & GASNETI_ATOMIC_WMB_POST) gasneti_local_wmb();
#elif GASNETI_RMB_IS_MB
  /* Case with mb() == rmb() and a distinct wmb().
   */
  #define _gasneti_atomic_fence_before(f)	if (f & GASNETI_ATOMIC_RMB_PRE) gasneti_local_rmb(); \
                                                else if (f & GASNETI_ATOMIC_WMB_PRE) gasneti_local_wmb();
  #define _gasneti_atomic_fence_after(f)	if (f & GASNETI_ATOMIC_RMB_POST) gasneti_local_rmb(); \
                                                else if (f & GASNETI_ATOMIC_WMB_POST) gasneti_local_wmb();
#elif GASNETI_WMB_IS_MB
  /* Case with mb() == wmb() and a distinct rmb().
   */
  #define _gasneti_atomic_fence_before(f)	if (f & GASNETI_ATOMIC_WMB_PRE) gasneti_local_wmb(); \
                                                else if (f & GASNETI_ATOMIC_RMB_PRE) gasneti_local_rmb();
  #define _gasneti_atomic_fence_after(f)	if (f & GASNETI_ATOMIC_WMB_POST) gasneti_local_wmb(); \
                                                else if (f & GASNETI_ATOMIC_RMB_POST) gasneti_local_rmb();
#else
  /*  With distinct mb(), rmb() and wmb(), we make the most general 3 checks (like a "switch").
   */
  #define _gasneti_atomic_fence_before(f)	if ((f & GASNETI_ATOMIC_MASK_PRE) == GASNETI_ATOMIC_MB_PRE) gasneti_local_mb(); \
                                                else if (f & GASNETI_ATOMIC_RMB_PRE) gasneti_local_rmb(); \
                                                else if (f & GASNETI_ATOMIC_WMB_PRE) gasneti_local_wmb();
  #define _gasneti_atomic_fence_after(f)	if ((f & GASNETI_ATOMIC_MASK_POST) == GASNETI_ATOMIC_MB_POST) gasneti_local_mb(); \
                                                else if (f & GASNETI_ATOMIC_RMB_POST) gasneti_local_rmb(); \
                                                else if (f & GASNETI_ATOMIC_WMB_POST) gasneti_local_wmb();
#endif

#if 1
  /*
   * Several optimizations are possible when a conditional rmb() is combined
   * with an unconditional POST fence.  Such optimizations would prevent
   * imposing a "double" rmb() in such cases.  However:
   * 1) There are no current callers that mix *MB_POST with a
   *    conditional RMB_POST_IF*, and no likely reason to.
   * 2) Though they all reduce a great deal at compile-time,
   *    such "optimizations" look very large to the inliner
   *    before any dead code can be eliminated.
   * Therefore, they are not currently implemented.
   */
  #define _gasneti_atomic_fence_bool(f, v) \
    if (((f & GASNETI_ATOMIC_RMB_POST_IF_TRUE ) &&  v) || \
        ((f & GASNETI_ATOMIC_RMB_POST_IF_FALSE) && !v)) gasneti_local_rmb();
#endif

/* ------------------------------------------------------------------------------------ */
/* Misc non-public definitions needed in the platform-specific parts */

#define _GASNETI_ATOMIC_CHECKALIGN(_a,_p) \
    gasneti_assert(!(_a) || !(((uintptr_t)(_p))&((_a)-1)))

/* ------------------------------------------------------------------------------------ */
/* All the platform-specific parts */
#include <gasnet_atomic_bits.h>

/* ------------------------------------------------------------------------------------ */
/* Fencing Part 2. Fences for the platform-specific atomics, based on macros in Part 1.
 *	_gasneti_atomic{,32,64}_prologue_{set,read,rmw}(p, flags)
 *	_gasneti_atomic{,32,64}_fence_{before,after}_{set,read,rmw}(p, flags)
 *	_gasneti_atomic{,32,64}_fence_after_bool(p, flags, value)
 *
 * These can/should be predefined by the platform-specific code if there are
 * any fencing side-effects in the unfenced ("_" prefixed) implementaions.
 * Such overrides must be applied to entire groups, as can be seen below.
 *
 * The "atomic_" are used if the platform defines GASNETI_HAVE_PRIVATE_ATOMIC_T.
 * The "atomic32_" are used unless the platform defines GASNETI_USE_GENERIC_ATOMIC32
 * The "atomic64_" are used unless the platform defines GASNETI_USE_GENERIC_ATOMIC64
 */
#ifndef _gasneti_atomic_prologue_set
  #define _gasneti_atomic_prologue_set(p,f)          /*empty*/
  #define _gasneti_atomic_fence_before_set(p,f)      _gasneti_atomic_fence_before(f)
  #define _gasneti_atomic_fence_after_set(p,f)       _gasneti_atomic_fence_after(f)
#endif
#ifndef _gasneti_atomic32_prologue_set
  #define _gasneti_atomic32_prologue_set(p,f)        /*empty*/
  #define _gasneti_atomic32_fence_before_set(p,f)    _gasneti_atomic_fence_before(f)
  #define _gasneti_atomic32_fence_after_set(p,f)     _gasneti_atomic_fence_after(f)
#endif
#ifndef _gasneti_atomic64_prologue_set
  #define _gasneti_atomic64_prologue_set(p,f)        /*empty*/
  #define _gasneti_atomic64_fence_before_set(p,f)    _gasneti_atomic_fence_before(f)
  #define _gasneti_atomic64_fence_after_set(p,f)     _gasneti_atomic_fence_after(f)
#endif
#ifndef _gasneti_atomic_prologue_read
  #define _gasneti_atomic_prologue_read(p,f)         /*empty*/
  #define _gasneti_atomic_fence_before_read(p,f)     _gasneti_atomic_fence_before(f)
  #define _gasneti_atomic_fence_after_read(p,f)      _gasneti_atomic_fence_after(f)
#endif
#ifndef _gasneti_atomic32_prologue_read
  #define _gasneti_atomic32_prologue_read(p,f)       /*empty*/
  #define _gasneti_atomic32_fence_before_read(p,f)   _gasneti_atomic_fence_before(f)
  #define _gasneti_atomic32_fence_after_read(p,f)    _gasneti_atomic_fence_after(f)
#endif
#ifndef _gasneti_atomic64_prologue_read
  #define _gasneti_atomic64_prologue_read(p,f)       /*empty*/
  #define _gasneti_atomic64_fence_before_read(p,f)   _gasneti_atomic_fence_before(f)
  #define _gasneti_atomic64_fence_after_read(p,f)    _gasneti_atomic_fence_after(f)
#endif
#ifndef _gasneti_atomic_prologue_rmw
  #define _gasneti_atomic_prologue_rmw(p,f)          /*empty*/
  #define _gasneti_atomic_fence_before_rmw(p,f)      _gasneti_atomic_fence_before(f)
  #define _gasneti_atomic_fence_after_rmw(p,f)       _gasneti_atomic_fence_after(f)
  #define _gasneti_atomic_fence_after_bool(p,f,v)    _gasneti_atomic_fence_after(f)  \
                                                     _gasneti_atomic_fence_bool(f,v)
#endif
#ifndef _gasneti_atomic32_prologue_rmw
  #define _gasneti_atomic32_prologue_rmw(p,f)        /*empty*/
  #define _gasneti_atomic32_fence_before_rmw(p,f)    _gasneti_atomic_fence_before(f)
  #define _gasneti_atomic32_fence_after_rmw(p,f)     _gasneti_atomic_fence_after(f)
  #define _gasneti_atomic32_fence_after_bool(p,f,v)  _gasneti_atomic_fence_after(f)  \
                                                     _gasneti_atomic_fence_bool(f,v)
#endif
#ifndef _gasneti_atomic64_prologue_rmw
  #define _gasneti_atomic64_prologue_rmw(p,f)        /*empty*/
  #define _gasneti_atomic64_fence_before_rmw(p,f)    _gasneti_atomic_fence_before(f)
  #define _gasneti_atomic64_fence_after_rmw(p,f)     _gasneti_atomic_fence_after(f)
  #define _gasneti_atomic64_fence_after_bool(p,f,v)  _gasneti_atomic_fence_after(f)  \
                                                     _gasneti_atomic_fence_bool(f,v)
#endif

/* ------------------------------------------------------------------------------------ */
/* Fenced atomic templates, using the per-group fencing macros.
 */

#define GASNETI_ATOMIC_CHECKALIGN(stem,p) _GASNETI_ATOMIC_CHECKALIGN(stem##align,p)

#define GASNETI_ATOMIC_FENCED_SET(group,_func,stem,p,v,f)           \
  do {                                                              \
    stem##t * const __p = (p);                                      \
    const int __flags = (f);                                        \
    _gasneti_##group##_prologue_set(__p,__flags)                    \
    GASNETI_ATOMIC_CHECKALIGN(stem,__p);                            \
    _gasneti_##group##_fence_before_set(__p,__flags)                \
    _func(__p,(v));                                                 \
    _gasneti_##group##_fence_after_set(__p,__flags)                 \
  } while (0)
#define GASNETI_ATOMIC_FENCED_INCDEC(group,_func,stem,p,f)          \
  do {                                                              \
    stem##t * const __p = (p);                                      \
    const int __flags = (f);                                        \
    _gasneti_##group##_prologue_rmw(__p,__flags)                    \
    GASNETI_ATOMIC_CHECKALIGN(stem,__p);                            \
    _gasneti_##group##_fence_before_rmw(__p,__flags)                \
    _func(__p);                                                     \
    _gasneti_##group##_fence_after_rmw(__p,__flags)                 \
  } while (0)

#define GASNETI_ATOMIC_FENCED_SET_DEFN_NOT_INLINE(group,func,_func,stem) \
  void func(stem##t *_p, stem##val_t _v, const int _flags) {        \
    GASNETI_ATOMIC_FENCED_SET(group,_func,stem,_p,_v,_flags);       \
  }
#define GASNETI_ATOMIC_FENCED_INCDEC_DEFN_NOT_INLINE(group,func,_func,stem) \
  void func(stem##t *_p, const int _flags) {                        \
    GASNETI_ATOMIC_FENCED_INCDEC(group,_func,stem,_p,_flags);       \
  }
#define GASNETI_ATOMIC_FENCED_READ_DEFN_NOT_INLINE(group,func,_func,stem) \
  stem##val_t func(stem##t *_p, const int _flags) {                 \
    _gasneti_##group##_prologue_read(_p,_flags)                     \
    GASNETI_ATOMIC_CHECKALIGN(stem,_p);                             \
    _gasneti_##group##_fence_before_read(_p,_flags)                 \
    { const stem##val_t _retval = _func(_p);                        \
      _gasneti_##group##_fence_after_read(_p,_flags)                \
      return _retval;                                               \
    }                                                               \
  }
#define GASNETI_ATOMIC_FENCED_DECTEST_DEFN_NOT_INLINE(group,func,_func,stem) \
  int func(stem##t *_p, const int _flags) {                         \
    _gasneti_##group##_prologue_rmw(_p,_flags)                      \
    GASNETI_ATOMIC_CHECKALIGN(stem,_p);                             \
    _gasneti_##group##_fence_before_rmw(_p,_flags)                  \
    { const int _retval = _func(_p);                                \
      _gasneti_##group##_fence_after_bool(_p,_flags, _retval)       \
      return _retval;                                               \
    }                                                               \
  }
#define GASNETI_ATOMIC_FENCED_CAS_DEFN_NOT_INLINE(group,func,_func,stem) \
  int func(stem##t *_p, stem##val_t _oldval,                        \
           stem##val_t _newval, const int _flags) {                 \
    _gasneti_##group##_prologue_rmw(_p,_flags)                      \
    GASNETI_ATOMIC_CHECKALIGN(stem,_p);                             \
    _gasneti_##group##_fence_before_rmw(_p,_flags)                  \
    { const int _retval = _func(_p,_oldval,_newval);                \
      _gasneti_##group##_fence_after_bool(_p,_flags, _retval)       \
      return _retval;                                               \
    }                                                               \
  }
#define GASNETI_ATOMIC_FENCED_SWAP_DEFN_NOT_INLINE(group,func,_func,stem) \
  stem##val_t func(stem##t *_p, stem##val_t _val, const int _flags) { \
    _gasneti_##group##_prologue_rmw(_p,_flags)                      \
    GASNETI_ATOMIC_CHECKALIGN(stem,_p);                             \
    _gasneti_##group##_fence_before_rmw(_p,_flags)                  \
    { const stem##val_t _retval = _func(_p, _val);                  \
      _gasneti_##group##_fence_after_rmw(_p,_flags)                 \
      return _retval;                                               \
    }                                                               \
  }
#define GASNETI_ATOMIC_FENCED_ADDSUB_DEFN_NOT_INLINE(group,func,_func,stem) \
  stem##val_t func(stem##t *_p, stem##val_t _op, const int _flags) { \
    _gasneti_##group##_prologue_rmw(_p,_flags)                      \
    GASNETI_ATOMIC_CHECKALIGN(stem,_p);                             \
    _gasneti_##group##_fence_before_rmw(_p,_flags)                  \
    { const stem##val_t _retval = _func(_p, _op);                   \
      _gasneti_##group##_fence_after_rmw(_p,_flags)                 \
      return _retval;                                               \
    }                                                               \
  }
#define GASNETI_ATOMIC_FENCED_ADDFETCH_DEFN_NOT_INLINE(group,func,_func,stem) \
  stem##val_t func(stem##t *_p, stem##val_t _op, const int _flags) {\
    _gasneti_##group##_prologue_rmw(_p,_flags)                      \
    GASNETI_ATOMIC_CHECKALIGN(stem,_p);                             \
    _gasneti_##group##_fence_before_rmw(_p,_flags)                  \
    { const stem##val_t _retval = _func(_p, _op);                   \
      _gasneti_##group##_fence_after_rmw(_p,_flags)                 \
      return _retval;                                               \
    }                                                               \
  }

#define GASNETI_ATOMIC_FENCED_SET_DEFN(group,func,_func,stem)       \
	GASNETI_INLINE(func)                                        \
	GASNETI_ATOMIC_FENCED_SET_DEFN_NOT_INLINE(group,func,_func,stem)
#define GASNETI_ATOMIC_FENCED_INCDEC_DEFN(group,func,_func,stem)    \
	GASNETI_INLINE(func)                                        \
	GASNETI_ATOMIC_FENCED_INCDEC_DEFN_NOT_INLINE(group,func,_func,stem)
#define GASNETI_ATOMIC_FENCED_READ_DEFN(group,func,_func,stem)      \
	GASNETI_INLINE(func)                                        \
        GASNETI_ATOMIC_FENCED_READ_DEFN_NOT_INLINE(group,func,_func,stem)
#define GASNETI_ATOMIC_FENCED_DECTEST_DEFN(group,func,_func,stem)   \
	GASNETI_INLINE(func)                                        \
        GASNETI_ATOMIC_FENCED_DECTEST_DEFN_NOT_INLINE(group,func,_func,stem)
#define GASNETI_ATOMIC_FENCED_CAS_DEFN(group,func,_func,stem)       \
	GASNETI_INLINE(func)                                        \
        GASNETI_ATOMIC_FENCED_CAS_DEFN_NOT_INLINE(group,func,_func,stem)
#define GASNETI_ATOMIC_FENCED_SWAP_DEFN(group,func,_func,stem)      \
        GASNETI_INLINE(func)                                        \
        GASNETI_ATOMIC_FENCED_SWAP_DEFN_NOT_INLINE(group,func,_func,stem)
#define GASNETI_ATOMIC_FENCED_ADDSUB_DEFN(group,func,_func,stem)    \
	GASNETI_INLINE(func)                                        \
        GASNETI_ATOMIC_FENCED_ADDSUB_DEFN_NOT_INLINE(group,func,_func,stem)
#define GASNETI_ATOMIC_FENCED_ADDFETCH_DEFN(group,func,_func,stem)  \
	GASNETI_INLINE(func)                                        \
        GASNETI_ATOMIC_FENCED_ADDFETCH_DEFN_NOT_INLINE(group,func,_func,stem)

/* ------------------------------------------------------------------------------------ */
/* Typeless unfenced operations on a pointer to a (volatile) scalar */

#define _gasneti_scalar_atomic_init(v)               (v)
#define _gasneti_scalar_atomic_set(p,v)              (*(p) = (v))
#define _gasneti_scalar_atomic_read(p)               (*(p))

/* NOTE: _gasneti_scalar_atomic_compare_and_swap evaluates `p` either once
 * or twice, depending on the initial value.
 * However, it is used only in the body of inline functions in which `p` is
 * a function argument (and thus free of side-effects).
 */
#define _gasneti_scalar_atomic_compare_and_swap(p,oval,nval) \
                                                     (*(p) == (oval) ? (*(p) = (nval), 1) : 0)

#if __cplusplus > 201703L
  // Bug 4060: C++20 deprecates certain "rmw" operations on volatile types
  #define _gasneti_scalar_atomic_increment(p) \
    (([=](auto _p) { auto _tmp = *_p; _tmp++; *_p = _tmp; } )(p))
  #define _gasneti_scalar_atomic_decrement(p) \
    (([=](auto _p) { auto _tmp = *_p; _tmp--; *_p = _tmp; } )(p))
  #define _gasneti_scalar_atomic_add(p,op) \
    (([=](auto _p, auto _op) { auto _tmp = *_p; _tmp += _op; *_p = _tmp; return _tmp;} )(p,op))
  #define _gasneti_scalar_atomic_subtract(p,op) \
    (([=](auto _p, auto _op) { auto _tmp = *_p; _tmp -= _op; *_p = _tmp; return _tmp;} )(p,op))
  #define _gasneti_scalar_atomic_decrement_and_test(p) \
    (_gasneti_scalar_atomic_subtract(p,1) == 0)
#else
  // C or C++ prior to 20
  #define _gasneti_scalar_atomic_increment(p)          ((*(p))++)
  #define _gasneti_scalar_atomic_decrement(p)          ((*(p))--)
  #define _gasneti_scalar_atomic_decrement_and_test(p) ((--(*(p))) == 0)
  #define _gasneti_scalar_atomic_add(p,op)             (*(p) += (op))
  #define _gasneti_scalar_atomic_subtract(p,op)        (*(p) -= (op))
#endif

#define _gasneti_scalar_atomic_addfetch(p,op) \
        _gasneti_scalar_atomic_add(p,op)

/* Swap, as above, but not typeless due to need for a temporary */
#define GASNETI_SCALAR_ATOMIC_SWAP_DEFN(func,stem)                      \
  GASNETI_INLINE(func) stem##val_t func(stem##t *_p, stem##val_t _val) {\
    const stem##val_t _retval = *_p; *_p = _val; return _retval;        \
  }

/* ------------------------------------------------------------------------------------ */
/* Fenced generic atomics, if needed, using per-platform defns and the macros above.
 */

#if defined(GASNETI_BUILD_GENERIC_ATOMIC32) || defined(GASNETI_BUILD_GENERIC_ATOMIC64)
  /* Fences for the generics */
  #if !GASNETI_GENATOMIC_LOCKS
    /* Not locking, so use full fences */
    #define _gasneti_genatomic_prologue_set(p,f)        /*empty*/
    #define _gasneti_genatomic_prologue_read(p,f)       /*empty*/
    #define _gasneti_genatomic_prologue_rmw(p,f)        /*empty*/
    #define _gasneti_genatomic_fence_before_rmw(p,f)    _gasneti_atomic_fence_before(f)
    #define _gasneti_genatomic_fence_after_rmw(p,f)     _gasneti_atomic_fence_after(f)
    #define _gasneti_genatomic_fence_after_bool(p,f,v)  _gasneti_atomic_fence_bool(f,v)
    #define _GASNETI_GENATOMIC_DECL_AND_DEFN(_sz)                                     \
      typedef volatile uint##_sz##_t gasneti_genatomic##_sz##_t;                      \
      typedef uint##_sz##_t gasneti_genatomic##_sz##_val_t;                           \
      typedef int##_sz##_t gasneti_genatomic##_sz##_sval_t;                           \
      GASNETI_ATOMIC_FENCED_SET_DEFN(genatomic,                                       \
                                     gasneti_genatomic##_sz##_set,                    \
                                     _gasneti_scalar_atomic_set,                      \
                                     gasneti_genatomic##_sz##_)                       \
      GASNETI_ATOMIC_FENCED_INCDEC_DEFN(genatomic,                                    \
                                        gasneti_genatomic##_sz##_increment,           \
                                        _gasneti_scalar_atomic_increment,             \
                                        gasneti_genatomic##_sz##_)                    \
      GASNETI_ATOMIC_FENCED_INCDEC_DEFN(genatomic,                                    \
                                        gasneti_genatomic##_sz##_decrement,           \
                                        _gasneti_scalar_atomic_decrement,             \
                                        gasneti_genatomic##_sz##_)                    \
      GASNETI_ATOMIC_FENCED_DECTEST_DEFN(genatomic,                                   \
                                         gasneti_genatomic##_sz##_decrement_and_test, \
                                         _gasneti_scalar_atomic_decrement_and_test,   \
                                         gasneti_genatomic##_sz##_)                   \
      GASNETI_ATOMIC_FENCED_CAS_DEFN(genatomic,                                       \
                                     gasneti_genatomic##_sz##_compare_and_swap,       \
                                     _gasneti_scalar_atomic_compare_and_swap,         \
                                     gasneti_genatomic##_sz##_)                       \
      GASNETI_SCALAR_ATOMIC_SWAP_DEFN(_gasneti_genatomic##_sz##_swap,                 \
                                      gasneti_genatomic##_sz##_)                      \
      GASNETI_ATOMIC_FENCED_SWAP_DEFN(genatomic,                                      \
                                      gasneti_genatomic##_sz##_swap,                  \
                                      _gasneti_genatomic##_sz##_swap,                 \
                                      gasneti_genatomic##_sz##_)                      \
      GASNETI_ATOMIC_FENCED_ADDFETCH_DEFN(genatomic,                                  \
                                          gasneti_genatomic##_sz##_addfetch,          \
                                          _gasneti_scalar_atomic_addfetch,            \
                                          gasneti_genatomic##_sz##_)
  #else /* Mutex-based versions */
    #define GASNETI_GENATOMIC_LOCK_PREP(ptr) \
                gasnett_mutex_t * const _genatomic_lock = gasneti_mutex_atomic_hash_lookup((uintptr_t)ptr)
    #define GASNETI_GENATOMIC_LOCK()   gasnett_mutex_lock(_genatomic_lock)
    #define GASNETI_GENATOMIC_UNLOCK() gasnett_mutex_unlock(_genatomic_lock)

    /* The lock acquire includes RMB and release includes WMB */
    #define _gasneti_genatomic_prologue_set(p,f)        GASNETI_GENATOMIC_LOCK_PREP(p);
    #define _gasneti_genatomic_prologue_read(p,f)       /*empty*/
    #define _gasneti_genatomic_prologue_rmw(p,f)        GASNETI_GENATOMIC_LOCK_PREP(p);
    #define _gasneti_genatomic_fence_before_rmw(p,f)	_gasneti_atomic_fence_before((f&~GASNETI_ATOMIC_RMB_PRE)) \
							GASNETI_GENATOMIC_LOCK();
    #define _gasneti_genatomic_fence_after_rmw(p,f)	GASNETI_GENATOMIC_UNLOCK();   \
							_gasneti_atomic_fence_after((f&~GASNETI_ATOMIC_WMB_POST))
    #define _gasneti_genatomic_fence_after_bool(p,f,v)	GASNETI_GENATOMIC_UNLOCK();   \
							_gasneti_atomic_fence_after((f&~GASNETI_ATOMIC_WMB_POST))\
							_gasneti_atomic_fence_bool(f,v)

    /* Because mutexes are not yet available (bug 693: avoid header dependency cycle),
     * we don't define the lock-acquiring operations as inlines.
     * Therefore, we declared them here but define them in gasnet_tools.c
     */
    #define _GASNETI_GENATOMIC_DECL_AND_DEFN(_sz)                                            \
      typedef volatile uint##_sz##_t gasneti_genatomic##_sz##_t;                             \
      typedef uint##_sz##_t gasneti_genatomic##_sz##_val_t;                                  \
      typedef int##_sz##_t gasneti_genatomic##_sz##_sval_t;                                  \
      extern void gasneti_genatomic##_sz##_set(gasneti_genatomic##_sz##_t *_p,               \
                                               gasneti_genatomic##_sz##_val_t _v,            \
                                               const int _flags);                            \
      extern void gasneti_genatomic##_sz##_increment(gasneti_genatomic##_sz##_t *_p,         \
                                                     const int _flags);                      \
      extern void gasneti_genatomic##_sz##_decrement(gasneti_genatomic##_sz##_t *_p,         \
                                                     const int _flags);                      \
      extern int gasneti_genatomic##_sz##_decrement_and_test(gasneti_genatomic##_sz##_t *_p, \
                                                             const int _flags);              \
      extern int gasneti_genatomic##_sz##_compare_and_swap(gasneti_genatomic##_sz##_t *_p,   \
                                                           uint##_sz##_t _oldval,            \
                                                           uint##_sz##_t _newval,            \
                                                           const int _flags);                \
      extern uint##_sz##_t gasneti_genatomic##_sz##_swap(gasneti_genatomic##_sz##_t *_p,     \
                                                         uint##_sz##_t _val,                 \
                                                         const int _flags);                  \
      extern uint##_sz##_t gasneti_genatomic##_sz##_addfetch(gasneti_genatomic##_sz##_t *_p, \
                                                             uint##_sz##_t _op,              \
                                                             const int _flags);               
    #define _GASNETI_GENATOMIC_DEFN(_sz)                                                         \
      GASNETI_ATOMIC_FENCED_SET_DEFN_NOT_INLINE(genatomic,                                       \
                                                gasneti_genatomic##_sz##_set,                    \
                                                _gasneti_scalar_atomic_set,                      \
                                                gasneti_genatomic##_sz##_)                       \
      GASNETI_ATOMIC_FENCED_INCDEC_DEFN_NOT_INLINE(genatomic,                                    \
                                                   gasneti_genatomic##_sz##_increment,           \
                                                   _gasneti_scalar_atomic_increment,             \
                                                   gasneti_genatomic##_sz##_)                    \
      GASNETI_ATOMIC_FENCED_INCDEC_DEFN_NOT_INLINE(genatomic,                                    \
                                                   gasneti_genatomic##_sz##_decrement,           \
                                                   _gasneti_scalar_atomic_decrement,             \
                                                   gasneti_genatomic##_sz##_)                    \
      GASNETI_ATOMIC_FENCED_DECTEST_DEFN_NOT_INLINE(genatomic,                                   \
                                                    gasneti_genatomic##_sz##_decrement_and_test, \
                                                    _gasneti_scalar_atomic_decrement_and_test,   \
                                                    gasneti_genatomic##_sz##_)                   \
      GASNETI_ATOMIC_FENCED_CAS_DEFN_NOT_INLINE(genatomic,                                       \
                                                gasneti_genatomic##_sz##_compare_and_swap,       \
                                                _gasneti_scalar_atomic_compare_and_swap,         \
                                                gasneti_genatomic##_sz##_)                       \
      GASNETI_SCALAR_ATOMIC_SWAP_DEFN(_gasneti_genatomic##_sz##_swap,                            \
                                      gasneti_genatomic##_sz##_)                                 \
      GASNETI_ATOMIC_FENCED_SWAP_DEFN_NOT_INLINE(genatomic,                                      \
                                                 gasneti_genatomic##_sz##_swap,                  \
                                                 _gasneti_genatomic##_sz##_swap,                 \
                                                 gasneti_genatomic##_sz##_)                      \
      GASNETI_ATOMIC_FENCED_ADDFETCH_DEFN_NOT_INLINE(genatomic,                                  \
                                                     gasneti_genatomic##_sz##_addfetch,          \
                                                     _gasneti_scalar_atomic_addfetch,            \
                                                     gasneti_genatomic##_sz##_)
  #endif
  #define _gasneti_genatomic_fence_before_set		_gasneti_genatomic_fence_before_rmw
  #define _gasneti_genatomic_fence_after_set		_gasneti_genatomic_fence_after_rmw
  /* READ is almost always performed without the lock (if any) held */
  #define _gasneti_genatomic_fence_before_read(p,f)	_gasneti_atomic_fence_before(f)
  #define _gasneti_genatomic_fence_after_read(p,f)	_gasneti_atomic_fence_after(f)

  /* Build the 32-bit generics if needed */
  #ifdef GASNETI_BUILD_GENERIC_ATOMIC32
    _GASNETI_GENATOMIC_DECL_AND_DEFN(32)
    #define gasneti_genatomic32_init          _gasneti_scalar_atomic_init
    GASNETI_ATOMIC_FENCED_READ_DEFN(genatomic,gasneti_genatomic32_read,
                                    _gasneti_scalar_atomic_read,gasneti_genatomic32_)
    #ifdef _GASNETI_GENATOMIC_DEFN
      #define GASNETI_GENATOMIC32_DEFN        _GASNETI_GENATOMIC_DEFN(32)
    #endif
  #endif

  /* Build the 64-bit generics if needed */
  #ifdef GASNETI_BUILD_GENERIC_ATOMIC64
    _GASNETI_GENATOMIC_DECL_AND_DEFN(64)
    #define gasneti_genatomic64_init          _gasneti_scalar_atomic_init
    #if _gasneti_need_genatomic64_read	/* ILP32 or HYBRID for under-aligned ABIs */
      /* Mutex is needed in read to avoid word tearing.
       * Can't use the normal template w/o also forcing a mutex into the 32-bit generics.
       * Note that we use the "rmw" fencing macros here, since the "read" fencing macros
       * assume no lock is taken and thus would potentially double fence.
       */
      extern uint64_t gasneti_genatomic64_read(gasneti_genatomic64_t *_p, const int _flags);
      #define _GASNETI_GENATOMIC64_DEFN_EXTRA \
	uint64_t gasneti_genatomic64_read(gasneti_genatomic64_t *_p, const int _flags) { \
          _gasneti_genatomic_prologue_rmw(_p,_flags)      /* rmw is NOT a typo here */   \
          GASNETI_ATOMIC_CHECKALIGN(gasneti_genatomic64_,_p);                            \
	  _gasneti_genatomic_fence_before_rmw(_p,_flags)  /* rmw is NOT a typo here */   \
	  { const uint64_t _retval = _gasneti_scalar_atomic_read(_p);                    \
	    _gasneti_genatomic_fence_after_rmw(_p,_flags) /* rmw is NOT a typo here */   \
	    return _retval;                                                              \
	  }                                                                              \
	}
    #else
      /* Read is assumed naturally atomic due to word size, or it doesn't matter in a serial build. */
      GASNETI_ATOMIC_FENCED_READ_DEFN(genatomic,gasneti_genatomic64_read,
                                      _gasneti_scalar_atomic_read,gasneti_genatomic64_)
      #define _GASNETI_GENATOMIC64_DEFN_EXTRA /* Empty */
    #endif
    #ifndef _GASNETI_GENATOMIC_DEFN
      #define _GASNETI_GENATOMIC_DEFN(_sz) /* Empty */
    #endif
    #define GASNETI_GENATOMIC64_DEFN        _GASNETI_GENATOMIC_DEFN(64) \
                                            _GASNETI_GENATOMIC64_DEFN_EXTRA
  #endif
#endif

/* ------------------------------------------------------------------------------------ */
/* Define the fully-fenced fixed-width atomics
 */

typedef uint32_t gasneti_atomic32_val_t;	/* For consistency in fencing macros */
typedef int32_t gasneti_atomic32_sval_t;	/* For consistency in fencing macros */
#if defined(GASNETI_USE_GENERIC_ATOMIC32)
  /* Define 32-bit fixed-width atomics in terms of full-fenced generics */
  #define gasneti_atomic32_t                   gasneti_genatomic32_t
  #define gasneti_atomic32_init                gasneti_genatomic32_init
  #define gasneti_genatomic32_add(p,op,f)      ((uint32_t)gasneti_genatomic32_addfetch((p),(op),(f)))
  #define gasneti_genatomic32_subtract(p,op,f) ((uint32_t)gasneti_genatomic32_addfetch((p),(uint32_t)-(op),(f)))
  #define _gasneti_atomic32_cons(_id)          gasneti_genatomic32_##_id
#elif defined(GASNETI_USING_SLOW_ATOMIC32)
  typedef struct { volatile uint32_t gasneti_ctr; } gasneti_atomic32_t;
  #define gasneti_atomic32_init(v) { (v) }
  #define gasneti_slow_atomic32_t     gasneti_atomic32_t
  #define gasneti_slow_atomic32_init  gasneti_atomic32_init
  #define gasneti_slow_atomic32_align gasneti_atomic32_align
  extern uint32_t gasneti_slow_atomic32_read(gasneti_atomic32_t *_p, const int _flags);
  extern void gasneti_slow_atomic32_set(gasneti_atomic32_t *_p, uint32_t _v, const int _flags);
  extern void gasneti_slow_atomic32_increment(gasneti_atomic32_t *_p, const int _flags);
  extern void gasneti_slow_atomic32_decrement(gasneti_atomic32_t *_p, const int _flags);
  extern int gasneti_slow_atomic32_decrement_and_test(gasneti_atomic32_t *_p, const int _flags);
  extern int gasneti_slow_atomic32_compare_and_swap(gasneti_atomic32_t *_p, uint32_t _oldval, uint32_t _newval, const int _flags);
  extern uint32_t gasneti_slow_atomic32_swap(gasneti_atomic32_t *_p, uint32_t _val, const int _flags);
  extern uint32_t gasneti_slow_atomic32_add(gasneti_atomic32_t *_p, uint32_t _op, const int _flags);
  extern uint32_t gasneti_slow_atomic32_subtract(gasneti_atomic32_t *_p, uint32_t _op, const int _flags);
  #define _gasneti_atomic32_cons(_id)    gasneti_slow_atomic32_##_id
#else
  /* Define 32-bit fixed-width atomics in terms of un-fenced native atomics */

  /* First define a fully-fenced addfetch if it appears to be needed */
  #if defined(_gasneti_atomic32_addfetch) || defined(_gasneti_atomic32_fetchadd)
    #ifndef _gasneti_atomic32_addfetch
      #define _gasneti_atomic32_addfetch(p,op) ((op) + _gasneti_atomic32_fetchadd((p),(op)))
    #endif
    GASNETI_ATOMIC_FENCED_ADDFETCH_DEFN(atomic32,                   \
                                        gasneti_atomic32_addfetch,  \
                                        _gasneti_atomic32_addfetch, \
                                        gasneti_atomic32_)
  #elif defined(gasneti_atomic32_fetchadd)
    GASNETI_INLINE(gasneti_atomic32_addfetch)
    uint32_t gasneti_atomic32_addfetch(gasneti_atomic32_t *_p, uint32_t _op, int _f) {
      return (uint32_t)(gasneti_atomic32_fetchadd(_p,_op,_f) + _op);
    }
  #endif

  #ifdef _gasneti_atomic32_set
    #define gasneti_atomic32_set(p,v,f) GASNETI_ATOMIC_FENCED_SET(atomic32,_gasneti_atomic32_set,gasneti_atomic32_,p,v,f)
  #endif
  #ifdef _gasneti_atomic32_read
    GASNETI_ATOMIC_FENCED_READ_DEFN(atomic32,gasneti_atomic32_read,_gasneti_atomic32_read,gasneti_atomic32_)
  #endif
  #ifdef _gasneti_atomic32_increment
    #define gasneti_atomic32_increment(p,f) GASNETI_ATOMIC_FENCED_INCDEC(atomic32,_gasneti_atomic32_increment,gasneti_atomic32_,p,f)
  #elif !defined(gasneti_atomic32_increment)
    #define gasneti_atomic32_increment(p,f) ((void)gasneti_atomic32_addfetch((p),1,(f)))
  #endif
  #ifdef _gasneti_atomic32_decrement
    #define gasneti_atomic32_decrement(p,f) GASNETI_ATOMIC_FENCED_INCDEC(atomic32,_gasneti_atomic32_decrement,gasneti_atomic32_,p,f)
  #elif !defined(gasneti_atomic32_decrement)
    #define gasneti_atomic32_decrement(p,f) ((void)gasneti_atomic32_addfetch((p),(uint32_t)-1,(f)))
  #endif
  #ifndef gasneti_atomic32_decrement_and_test
    #ifndef _gasneti_atomic32_decrement_and_test
      #define _gasneti_atomic32_decrement_and_test(p) (0==_gasneti_atomic32_addfetch((p),(uint32_t)-1))
    #endif
    GASNETI_ATOMIC_FENCED_DECTEST_DEFN(atomic32,gasneti_atomic32_decrement_and_test,_gasneti_atomic32_decrement_and_test,gasneti_atomic32_)
  #endif
  #ifdef _gasneti_atomic32_compare_and_swap
    GASNETI_ATOMIC_FENCED_CAS_DEFN(atomic32,gasneti_atomic32_compare_and_swap,_gasneti_atomic32_compare_and_swap,gasneti_atomic32_)
  #endif
  #ifdef _gasneti_atomic32_swap
    GASNETI_ATOMIC_FENCED_SWAP_DEFN(atomic32,gasneti_atomic32_swap,_gasneti_atomic32_swap,gasneti_atomic32_)
  #endif
  #ifdef _gasneti_atomic32_add
    GASNETI_ATOMIC_FENCED_ADDSUB_DEFN(atomic32,gasneti_atomic32_add,_gasneti_atomic32_add,gasneti_atomic32_)
  #elif !defined(gasneti_atomic32_add)
    #define gasneti_atomic32_add(p,op,f) ((uint32_t)gasneti_atomic32_addfetch((p),(uint32_t)(op),(f)))
  #endif
  #ifdef _gasneti_atomic32_subtract
    GASNETI_ATOMIC_FENCED_ADDSUB_DEFN(atomic32,gasneti_atomic32_subtract,_gasneti_atomic32_subtract,gasneti_atomic32_)
  #elif !defined(gasneti_atomic32_subtract)
    #define gasneti_atomic32_subtract(p,op,f) ((uint32_t)gasneti_atomic32_addfetch((p),(uint32_t)-(op),(f)))
  #endif
#endif
#ifdef _gasneti_atomic32_cons
  #define gasneti_atomic32_read                 _gasneti_atomic32_cons(read)
  #define gasneti_atomic32_set                  _gasneti_atomic32_cons(set)
  #define gasneti_atomic32_increment            _gasneti_atomic32_cons(increment)
  #define gasneti_atomic32_decrement            _gasneti_atomic32_cons(decrement)
  #define gasneti_atomic32_decrement_and_test   _gasneti_atomic32_cons(decrement_and_test)
  #define gasneti_atomic32_compare_and_swap     _gasneti_atomic32_cons(compare_and_swap)
  #define gasneti_atomic32_swap                 _gasneti_atomic32_cons(swap)
  #define gasneti_atomic32_add                  _gasneti_atomic32_cons(add)
  #define gasneti_atomic32_subtract             _gasneti_atomic32_cons(subtract)
#endif

typedef uint64_t gasneti_atomic64_val_t;	/* For consistency in fencing macros */
typedef int64_t gasneti_atomic64_sval_t;	/* For consistency in fencing macros */
#if defined(GASNETI_USE_GENERIC_ATOMIC64)
  /* Define 64-bit fixed-width atomics in terms of full-fenced generics */
  #define gasneti_atomic64_t                   gasneti_genatomic64_t
  #define gasneti_atomic64_init                gasneti_genatomic64_init
  #define gasneti_genatomic64_add(p,op,f)      ((uint64_t)gasneti_genatomic64_addfetch((p),(uint64_t)(op),(f)))
  #define gasneti_genatomic64_subtract(p,op,f) ((uint64_t)gasneti_genatomic64_addfetch((p),(uint64_t)-(op),(f)))
  #define _gasneti_atomic64_cons(_id)          gasneti_genatomic64_##_id
#elif defined(GASNETI_USING_SLOW_ATOMIC64)
  typedef struct { volatile uint64_t gasneti_ctr; } gasneti_atomic64_t;
  #define gasneti_atomic64_init(v) { (v) }
  #define gasneti_slow_atomic64_t     gasneti_atomic64_t
  #define gasneti_slow_atomic64_init  gasneti_atomic64_init
  #define gasneti_slow_atomic64_align gasneti_atomic64_align
  extern uint64_t gasneti_slow_atomic64_read(gasneti_atomic64_t *_p, const int _flags);
  extern void gasneti_slow_atomic64_set(gasneti_atomic64_t *_p, uint64_t _v, const int _flags);
  extern void gasneti_slow_atomic64_increment(gasneti_atomic64_t *_p, const int _flags);
  extern void gasneti_slow_atomic64_decrement(gasneti_atomic64_t *_p, const int _flags);
  extern int gasneti_slow_atomic64_decrement_and_test(gasneti_atomic64_t *_p, const int _flags);
  extern int gasneti_slow_atomic64_compare_and_swap(gasneti_atomic64_t *_p, uint64_t _oldval, uint64_t _newval, const int _flags);
  extern uint64_t gasneti_slow_atomic64_swap(gasneti_atomic64_t *_p, uint64_t _val, const int _flags);
  extern uint64_t gasneti_slow_atomic64_add(gasneti_atomic64_t *_p, uint64_t _op, const int _flags);
  extern uint64_t gasneti_slow_atomic64_subtract(gasneti_atomic64_t *_p, uint64_t _op, const int _flags);
  #define _gasneti_atomic64_cons(_id)    gasneti_slow_atomic64_##_id
#elif defined(GASNETI_HYBRID_ATOMIC64)
  /* Hybrid: need to runtime select between native and generic, based on alignment.
   *
   * Note that this code is currently only used for ILP32 on PPC64 h/w on Darwin.
   * Consequently, it is less general in its assumptions about what thas been
   * implemented in the platform-specific code.  In particular we require the
   * following in the current revision of "hybid" atomics support:
   *     _gasneti_atomic64_{set,read,compare_and_swap,swap,fetchadd}
   */
  
  #define __gasneti_atomic64_set(p,v,f) GASNETI_ATOMIC_FENCED_SET(atomic64,_gasneti_atomic64_set,gasneti_atomic64_,p,v,f)
  #define gasneti_atomic64_set(p,v,f) do {                                   \
      const int _f = (f);                                                    \
      const uint64_t _v = (v);                                               \
      gasneti_atomic64_t * const _p = (p);                                   \
      if_pt (!((uintptr_t)_p & 0x7)) {                                       \
        __gasneti_atomic64_set(_p,_v,_f);                                    \
      } else {                                                               \
	gasneti_genatomic64_set((gasneti_genatomic64_t *)_p,_v,_f);          \
      }                                                                      \
    } while (0)
  GASNETI_ATOMIC_FENCED_READ_DEFN(atomic64,__gasneti_atomic64_read,_gasneti_atomic64_read,gasneti_atomic64_)
  GASNETI_INLINE(gasneti_atomic64_read)
  uint64_t gasneti_atomic64_read(gasneti_atomic64_t *_p, const int _flags) {
    if_pt (!((uintptr_t)_p & 0x7)) {
      return __gasneti_atomic64_read(_p, _flags);
    } else {
      return gasneti_genatomic64_read((gasneti_genatomic64_t *)_p, _flags);
    }
  }
  GASNETI_ATOMIC_FENCED_CAS_DEFN(atomic64,__gasneti_atomic64_compare_and_swap,_gasneti_atomic64_compare_and_swap,gasneti_atomic64_)
  GASNETI_INLINE(gasneti_atomic64_compare_and_swap)
  int gasneti_atomic64_compare_and_swap(gasneti_atomic64_t *_p, uint64_t _oldval,
					uint64_t _newval, const int _flags) {
    if_pt (!((uintptr_t)_p & 0x7)) {
      return __gasneti_atomic64_compare_and_swap(_p,_oldval,_newval,_flags);
    } else {
      return gasneti_genatomic64_compare_and_swap((gasneti_genatomic64_t *)_p,_oldval,_newval,_flags);
    }
  }
  GASNETI_ATOMIC_FENCED_SWAP_DEFN(atomic64,__gasneti_atomic64_swap,_gasneti_atomic64_swap,gasneti_atomic64_)
  GASNETI_INLINE(gasneti_atomic64_swap)
  uint64_t gasneti_atomic64_swap(gasneti_atomic64_t *_p, uint64_t _op, const int _flags) {
    if_pt (!((uintptr_t)_p & 0x7)) {
      return __gasneti_atomic64_swap(_p,_op,_flags);
    } else {
      return gasneti_genatomic64_swap((gasneti_genatomic64_t *)_p,_op,_flags);
    }
  }
  GASNETI_ATOMIC_FENCED_ADDFETCH_DEFN(atomic64,__gasneti_atomic64_fetchadd,_gasneti_atomic64_fetchadd,gasneti_atomic64_)
  GASNETI_INLINE(gasneti_atomic64_addfetch)
  uint64_t gasneti_atomic64_addfetch(gasneti_atomic64_t *_p, uint64_t _op, const int _flags) {
    if_pt (!((uintptr_t)_p & 0x7)) {
      return _op + __gasneti_atomic64_fetchadd(_p,_op,_flags);
    } else {
      return gasneti_genatomic64_addfetch((gasneti_genatomic64_t *)_p,_op,_flags);
    }
  }
  GASNETI_INLINE(gasneti_atomic64_decrement_and_test)
  int gasneti_atomic64_decrement_and_test(gasneti_atomic64_t *_p, const int _flags) {
    const int _mask = (GASNETI_ATOMIC_RMB_POST_IF_TRUE|GASNETI_ATOMIC_RMB_POST_IF_FALSE);
    if_pt (!((uintptr_t)_p & 0x7)) {
      const int _result = (1 == __gasneti_atomic64_fetchadd(_p,(uint64_t)-1,_flags));
      _gasneti_atomic64_fence_after_bool(_p,(_flags&_mask),_result);
      return _result;
    } else {
      const int _result = (0 == gasneti_genatomic64_addfetch((gasneti_genatomic64_t *)_p,(uint64_t)-1,_flags));
      _gasneti_atomic_fence_after_bool(_p,(_flags&_mask),_result);
      return _result;
    }
  }
  #define gasneti_atomic64_increment(p,f) ((void)gasneti_atomic64_addfetch((p),1,(f)))
  #define gasneti_atomic64_decrement(p,f) ((void)gasneti_atomic64_addfetch((p),(uint64_t)-1,(f)))
  #define gasneti_atomic64_add(p,op,f) ((uint64_t)gasneti_atomic64_addfetch((p),(uint64_t)(op),(f)))
  #define gasneti_atomic64_subtract(p,op,f) ((uint64_t)gasneti_atomic64_addfetch((p),(uint64_t)-(op),(f)))
#else
  /* Define 64-bit fixed-width atomics in terms of un-fenced native atomics */

  /* First define a fully-fenced addfetch if it appears to be needed */
  #if defined(_gasneti_atomic64_addfetch) || defined(_gasneti_atomic64_fetchadd)
    #ifndef _gasneti_atomic64_addfetch
      #define _gasneti_atomic64_addfetch(p,op) ((op) + _gasneti_atomic64_fetchadd((p),(op)))
    #endif
    GASNETI_ATOMIC_FENCED_ADDFETCH_DEFN(atomic64,                   \
                                        gasneti_atomic64_addfetch,  \
                                        _gasneti_atomic64_addfetch, \
                                        gasneti_atomic64_)
  #elif defined(gasneti_atomic64_fetchadd)
    GASNETI_INLINE(gasneti_atomic64_addfetch)
    uint64_t gasneti_atomic64_addfetch(gasneti_atomic64_t *_p, uint64_t _op, int _f) {
      return (uint64_t)(gasneti_atomic64_fetchadd(_p,_op,_f) + _op);
    }
  #endif

  #ifdef _gasneti_atomic64_set
    #define gasneti_atomic64_set(p,v,f) GASNETI_ATOMIC_FENCED_SET(atomic64,_gasneti_atomic64_set,gasneti_atomic64_,p,v,f)
  #endif
  #ifdef _gasneti_atomic64_read
    GASNETI_ATOMIC_FENCED_READ_DEFN(atomic64,gasneti_atomic64_read,_gasneti_atomic64_read,gasneti_atomic64_)
  #endif
  #ifdef _gasneti_atomic64_increment
    #define gasneti_atomic64_increment(p,f) GASNETI_ATOMIC_FENCED_INCDEC(atomic64,_gasneti_atomic64_increment,gasneti_atomic64_,p,f)
  #elif !defined(gasneti_atomic64_increment)
    #define gasneti_atomic64_increment(p,f) ((void)gasneti_atomic64_addfetch((p),1,(f)))
  #endif
  #ifdef _gasneti_atomic64_decrement
    #define gasneti_atomic64_decrement(p,f) GASNETI_ATOMIC_FENCED_INCDEC(atomic64,_gasneti_atomic64_decrement,gasneti_atomic64_,p,f)
  #elif !defined(gasneti_atomic64_decrement)
    #define gasneti_atomic64_decrement(p,f) ((void)gasneti_atomic64_addfetch((p),(uint64_t)-1,(f)))
  #endif
  #ifndef gasneti_atomic64_decrement_and_test
    #ifndef _gasneti_atomic64_decrement_and_test
      #define _gasneti_atomic64_decrement_and_test(p) (0==_gasneti_atomic64_addfetch((p),(uint64_t)-1))
    #endif
    GASNETI_ATOMIC_FENCED_DECTEST_DEFN(atomic64,gasneti_atomic64_decrement_and_test,_gasneti_atomic64_decrement_and_test,gasneti_atomic64_)
  #endif
  #ifdef _gasneti_atomic64_compare_and_swap
    GASNETI_ATOMIC_FENCED_CAS_DEFN(atomic64,gasneti_atomic64_compare_and_swap,_gasneti_atomic64_compare_and_swap,gasneti_atomic64_)
  #endif
  #ifdef _gasneti_atomic64_swap
    GASNETI_ATOMIC_FENCED_SWAP_DEFN(atomic64,gasneti_atomic64_swap,_gasneti_atomic64_swap,gasneti_atomic64_)
  #endif
  #ifdef _gasneti_atomic64_add
    GASNETI_ATOMIC_FENCED_ADDSUB_DEFN(atomic64,gasneti_atomic64_add,_gasneti_atomic64_add,gasneti_atomic64_)
  #elif !defined(gasneti_atomic64_add)
    #define gasneti_atomic64_add(p,op,f) ((uint64_t)gasneti_atomic64_addfetch((p),(uint64_t)(op),(f)))
  #endif
  #ifdef _gasneti_atomic64_subtract
    GASNETI_ATOMIC_FENCED_ADDSUB_DEFN(atomic64,gasneti_atomic64_subtract,_gasneti_atomic64_subtract,gasneti_atomic64_)
  #elif !defined(gasneti_atomic64_subtract)
    #define gasneti_atomic64_subtract(p,op,f) ((uint64_t)gasneti_atomic64_addfetch((p),(uint64_t)-(op),(f)))
  #endif
#endif
#ifdef _gasneti_atomic64_cons
  #define gasneti_atomic64_read                 _gasneti_atomic64_cons(read)
  #define gasneti_atomic64_set                  _gasneti_atomic64_cons(set)
  #define gasneti_atomic64_increment            _gasneti_atomic64_cons(increment)
  #define gasneti_atomic64_decrement            _gasneti_atomic64_cons(decrement)
  #define gasneti_atomic64_decrement_and_test   _gasneti_atomic64_cons(decrement_and_test)
  #define gasneti_atomic64_compare_and_swap     _gasneti_atomic64_cons(compare_and_swap)
  #define gasneti_atomic64_swap                 _gasneti_atomic64_cons(swap)
  #define gasneti_atomic64_add                  _gasneti_atomic64_cons(add)
  #define gasneti_atomic64_subtract             _gasneti_atomic64_cons(subtract)
#endif

/* ------------------------------------------------------------------------------------ */
/* Define the "normal" atomics, which might be a private type, or either the 32-bit or 64-
 * bit fixed-width types (which in turn may have been derived from the generics, above.)
 */

#if defined(GASNETI_HAVE_PRIVATE_ATOMIC_T)
  /* "Private" atomics, using per-platform defns and the fencing macros.
   * These definitions add fencing around non-fenced implementations, but defer to any
   * platform-specific fully-fenced definitions which may exisit.
   */
  #ifndef gasneti_atomic_set
  #define gasneti_atomic_set(p,v,f) GASNETI_ATOMIC_FENCED_SET(atomic,_gasneti_atomic_set,gasneti_atomic_,p,v,f)
  #endif
  #ifndef gasneti_atomic_increment
    #define gasneti_atomic_increment(p,f) GASNETI_ATOMIC_FENCED_INCDEC(atomic,_gasneti_atomic_increment,gasneti_atomic_,p,f)
  #endif
  #ifndef gasneti_atomic_decrement
    #define gasneti_atomic_decrement(p,f) GASNETI_ATOMIC_FENCED_INCDEC(atomic,_gasneti_atomic_decrement,gasneti_atomic_,p,f)
  #endif
  #ifndef gasneti_atomic_read
    GASNETI_ATOMIC_FENCED_READ_DEFN(atomic,gasneti_atomic_read,_gasneti_atomic_read,gasneti_atomic_)
  #endif
  #ifndef gasneti_atomic_decrement_and_test
    GASNETI_ATOMIC_FENCED_DECTEST_DEFN(atomic,gasneti_atomic_decrement_and_test,_gasneti_atomic_decrement_and_test,gasneti_atomic_)
  #endif
  #if defined(GASNETI_HAVE_ATOMIC_CAS) && !defined(gasneti_atomic_compare_and_swap)
    GASNETI_ATOMIC_FENCED_CAS_DEFN(atomic,gasneti_atomic_compare_and_swap,_gasneti_atomic_compare_and_swap,gasneti_atomic_)
  #endif
  #if defined(GASNETI_HAVE_ATOMIC_CAS) && !defined(gasneti_atomic_swap)
    GASNETI_ATOMIC_FENCED_SWAP_DEFN(atomic,gasneti_atomic_swap,_gasneti_atomic_swap,gasneti_atomic_)
  #endif
  #if defined(GASNETI_HAVE_ATOMIC_ADD_SUB) && !defined(gasneti_atomic_add)
    GASNETI_ATOMIC_FENCED_ADDSUB_DEFN(atomic,gasneti_atomic_add,_gasneti_atomic_add,gasneti_atomic_)
  #endif
  #if defined(GASNETI_HAVE_ATOMIC_ADD_SUB) && !defined(gasneti_atomic_subtract)
    GASNETI_ATOMIC_FENCED_ADDSUB_DEFN(atomic,gasneti_atomic_subtract,_gasneti_atomic_subtract,gasneti_atomic_)
  #endif
#elif !defined(GASNETI_FORCE_64BIT_ATOMICOPS) && /* Not forcing 64-bits */ \
      (!defined(GASNETI_USE_GENERIC_ATOMIC32) || defined(GASNETI_USE_GENERIC_ATOMIC64)) /* No worse than 64 bit */
  #define GASNETI_USE_32BIT_ATOMICS
#else
  #define GASNETI_USE_64BIT_ATOMICS
#endif

#if defined(GASNETI_USE_32BIT_ATOMICS)
  typedef uint32_t				gasneti_atomic_val_t;
  typedef int32_t				gasneti_atomic_sval_t;
  #define GASNETI_ATOMIC_MAX			((gasneti_atomic_val_t)0xFFFFFFFFU)
  #define GASNETI_ATOMIC_SIGNED_MIN		((gasneti_atomic_sval_t)0x80000000)
  #define GASNETI_ATOMIC_SIGNED_MAX		((gasneti_atomic_sval_t)0x7FFFFFFF)

  #if defined(GASNETI_USE_GENERIC_ATOMIC32)
    #ifndef GASNETI_USE_GENERIC_ATOMICOPS
      #define GASNETI_USE_GENERIC_ATOMICOPS 1
    #endif
  #endif
  #ifdef GASNETI_ATOMIC32_NOT_SIGNALSAFE
    #define GASNETI_ATOMICOPS_NOT_SIGNALSAFE 1
  #endif

  #define GASNETI_HAVE_ATOMIC_CAS               1
  #define GASNETI_HAVE_ATOMIC_ADD_SUB           1

  #if defined(GASNETI_USING_SLOW_ATOMIC32)
    #define _gasneti_atomic_cons(_id)    gasneti_slow_atomic32_##_id
  #else
    #define _gasneti_atomic_cons(_id)    gasneti_atomic32_##_id
  #endif
#elif defined(GASNETI_USE_64BIT_ATOMICS)
  typedef uint64_t			gasneti_atomic_val_t;
  typedef int64_t			gasneti_atomic_sval_t;
  #define GASNETI_ATOMIC_MAX		((gasneti_atomic_val_t)0xFFFFFFFFFFFFFFFFLLU)
  #define GASNETI_ATOMIC_SIGNED_MIN	((gasneti_atomic_sval_t)0x8000000000000000LL)
  #define GASNETI_ATOMIC_SIGNED_MAX	((gasneti_atomic_sval_t)0x7FFFFFFFFFFFFFFFLL)

  #if defined(GASNETI_USE_GENERIC_ATOMIC64)
    #ifndef GASNETI_USE_GENERIC_ATOMICOPS
      #define GASNETI_USE_GENERIC_ATOMICOPS 1
    #endif
  #endif
  #ifdef GASNETI_ATOMIC64_NOT_SIGNALSAFE
    #define GASNETI_ATOMICOPS_NOT_SIGNALSAFE 1
  #endif

  #define GASNETI_HAVE_ATOMIC_CAS               1
  #define GASNETI_HAVE_ATOMIC_ADD_SUB           1

  #if defined(GASNETI_USING_SLOW_ATOMIC64)
    #define _gasneti_atomic_cons(_id)    gasneti_slow_atomic64_##_id
  #else
    #define _gasneti_atomic_cons(_id)    gasneti_atomic64_##_id
  #endif
#elif defined(GASNETI_USING_SLOW_ATOMICOPS) // GASNETI_HAVE_PRIVATE_ATOMIC_T case
  extern gasneti_atomic_val_t gasneti_slow_atomic_read(gasneti_atomic_t *_p, const int _flags);
  extern void gasneti_slow_atomic_set(gasneti_atomic_t *_p, gasneti_atomic_val_t _v, const int _flags);
  extern void gasneti_slow_atomic_increment(gasneti_atomic_t *_p, const int _flags);
  extern void gasneti_slow_atomic_decrement(gasneti_atomic_t *_p, const int _flags);
  extern int gasneti_slow_atomic_decrement_and_test(gasneti_atomic_t *_p, const int _flags);
  #if defined(GASNETI_HAVE_ATOMIC_CAS)
    extern int gasneti_slow_atomic_compare_and_swap(gasneti_atomic_t *_p, gasneti_atomic_val_t _oldval, gasneti_atomic_val_t _newval, const int _flags);
    extern gasneti_atomic_val_t gasneti_slow_atomic_swap(gasneti_atomic_t *_p, gasneti_atomic_val_t _val, const int _flags);
  #endif
  #if defined(GASNETI_HAVE_ATOMIC_ADD_SUB)
    extern gasneti_atomic_val_t gasneti_slow_atomic_add(gasneti_atomic_t *_p, gasneti_atomic_val_t _op, const int _flags);
    extern gasneti_atomic_val_t gasneti_slow_atomic_subtract(gasneti_atomic_t *_p, gasneti_atomic_val_t _op, const int _flags);
  #endif

  #define _gasneti_atomic_cons(_id)    gasneti_slow_atomic_##_id
#endif
#ifndef gasneti_atomic_signed
  #define gasneti_atomic_signed(val) ((gasneti_atomic_sval_t)(val))
#endif
#ifdef _gasneti_atomic_cons
  #define gasneti_atomic_t                      _gasneti_atomic_cons(t)
  #define gasneti_atomic_align			_gasneti_atomic_cons(align)
  #define gasneti_atomic_init			_gasneti_atomic_cons(init)
  #define gasneti_atomic_set                    _gasneti_atomic_cons(set)
  #define gasneti_atomic_read                   _gasneti_atomic_cons(read)
  #define gasneti_atomic_increment              _gasneti_atomic_cons(increment)
  #define gasneti_atomic_decrement              _gasneti_atomic_cons(decrement)
  #define gasneti_atomic_decrement_and_test     _gasneti_atomic_cons(decrement_and_test)
  #if defined(GASNETI_HAVE_ATOMIC_CAS)
    #define gasneti_atomic_compare_and_swap     _gasneti_atomic_cons(compare_and_swap)
    #define gasneti_atomic_swap                 _gasneti_atomic_cons(swap)
  #endif
  #if defined(GASNETI_HAVE_ATOMIC_ADD_SUB)
    #define gasneti_atomic_add                  _gasneti_atomic_cons(add)
    #define gasneti_atomic_subtract             _gasneti_atomic_cons(subtract)
  #endif
#endif

/* ------------------------------------------------------------------------------------ */
/* GASNet "non-atomics" - these implement the same operations as the interfaces without
 *   "non" in the names, only in a non-threadsafe manner.
 *
 * On SEQ build the weak atomics will reduce to this implementation, but it is made
 *   available unconditionally to allow use by conduits under appropriate circumstances.
 *
 * Do not need any exclusion mechanism, but we still want to include any fences that
 *   the caller has requested, since any memory in the gasnet segment "protected" by a
 *   fenced atomic may be written by a network adapter.
 */
#define _GASNETI_NONATOMIC_DEFN(_type,_sz)            \
  typedef volatile uint##_sz##_t gasneti_##_type##_t; \
  typedef uint##_sz##_t gasneti_##_type##_val_t;      \
  typedef int##_sz##_t gasneti_##_type##_sval_t;

/* Fencing macros for nonatomics
 *	_gasneti_nonatomic_prologue_{set,read,rmw}(p, flags)
 *	_gasneti_nonatomic_fence_{before,after}_{set,read,rmw}(p, flags)
 *	_gasneti_nonatomic_fence_after_bool(p, flags, value)
 *
 * These are defined defined unconditionally because presently there are no
 * fencing side-effects for the non-atomic code, and thus no reason to override.
 */
#if GASNETI_THREADS || defined(GASNETI_FORCE_TRUE_WEAKATOMICS)
  /* Always apply the fences */
  #define _gasneti_weak_fence_check(f)	0
#else
  /* Apply fences unless "GASNETI_ATOMIC_WEAK_FENCE" is present */
  #define _gasneti_weak_fence_check(f)	(f & GASNETI_ATOMIC_WEAK_FENCE)
#endif
#define _gasneti_nonatomic_fence_before(p,f) \
            if (!_gasneti_weak_fence_check(f)) { _gasneti_atomic_fence_before(f); }
#define _gasneti_nonatomic_fence_after(p,f) \
            if (!_gasneti_weak_fence_check(f)) { _gasneti_atomic_fence_after(f); }
#define _gasneti_nonatomic_fence_after_bool(p,f,v) \
            if (!_gasneti_weak_fence_check(f)) { _gasneti_atomic_fence_after(f)  \
                                                 _gasneti_atomic_fence_bool(f,v) }
#define _gasneti_nonatomic_prologue_set(p,f)  /*empty*/
#define _gasneti_nonatomic_fence_before_set  _gasneti_nonatomic_fence_before
#define _gasneti_nonatomic_fence_after_set   _gasneti_nonatomic_fence_after
#define _gasneti_nonatomic_prologue_read(p,f) /*empty*/
#define _gasneti_nonatomic_fence_before_read _gasneti_nonatomic_fence_before
#define _gasneti_nonatomic_fence_after_read  _gasneti_nonatomic_fence_after
#define _gasneti_nonatomic_prologue_rmw(p,f)  /*empty*/
#define _gasneti_nonatomic_fence_before_rmw  _gasneti_nonatomic_fence_before
#define _gasneti_nonatomic_fence_after_rmw   _gasneti_nonatomic_fence_after

/* Build gasneti_nonatomic_t to match width of "normal" atomic (unless custom) */
#if defined(GASNETI_USE_64BIT_ATOMICS)
  _GASNETI_NONATOMIC_DEFN(nonatomic,64)
  #define GASNETI_NONATOMIC_MAX            ((gasneti_nonatomic_val_t)0xFFFFFFFFFFFFFFFFLLU)
  #define GASNETI_NONATOMIC_SIGNED_MIN     ((gasneti_nonatomic_sval_t)0x8000000000000000LL)
  #define GASNETI_NONATOMIC_SIGNED_MAX     ((gasneti_nonatomic_sval_t)0x7FFFFFFFFFFFFFFFLL)
#else
  _GASNETI_NONATOMIC_DEFN(nonatomic,32)
  #define GASNETI_NONATOMIC_MAX            ((gasneti_nonatomic_val_t)0xFFFFFFFFU)
  #define GASNETI_NONATOMIC_SIGNED_MIN     ((gasneti_nonatomic_sval_t)0x80000000)
  #define GASNETI_NONATOMIC_SIGNED_MAX     ((gasneti_nonatomic_sval_t)0x7FFFFFFF)
#endif

#ifndef gasneti_nonatomic_align
  #define gasneti_nonatomic_align       gasneti_atomic_align
#endif
#ifndef gasneti_nonatomic32_align
  #define gasneti_nonatomic32_align     gasneti_atomic32_align
#endif
#ifndef gasneti_nonatomic64_align
  #define gasneti_nonatomic64_align     gasneti_atomic64_align
#endif

#define gasneti_nonatomic_init            _gasneti_scalar_atomic_init
#define gasneti_nonatomic_signed(v)       ((gasneti_nonatomic_sval_t)(val))
#define gasneti_nonatomic_set(p,v,f)      GASNETI_ATOMIC_FENCED_SET(nonatomic,_gasneti_scalar_atomic_set,gasneti_nonatomic_,p,v,f)
#define gasneti_nonatomic_increment(p,f)  GASNETI_ATOMIC_FENCED_INCDEC(nonatomic,_gasneti_scalar_atomic_increment,gasneti_nonatomic_,p,f)
#define gasneti_nonatomic_decrement(p,f)  GASNETI_ATOMIC_FENCED_INCDEC(nonatomic,_gasneti_scalar_atomic_decrement,gasneti_nonatomic_,p,f)
GASNETI_ATOMIC_FENCED_READ_DEFN(nonatomic,gasneti_nonatomic_read,_gasneti_scalar_atomic_read,gasneti_nonatomic_)
GASNETI_ATOMIC_FENCED_DECTEST_DEFN(nonatomic,gasneti_nonatomic_decrement_and_test,_gasneti_scalar_atomic_decrement_and_test,gasneti_nonatomic_)
GASNETI_ATOMIC_FENCED_CAS_DEFN(nonatomic,gasneti_nonatomic_compare_and_swap,_gasneti_scalar_atomic_compare_and_swap,gasneti_nonatomic_)
GASNETI_SCALAR_ATOMIC_SWAP_DEFN(_gasneti_scalar_atomic_swap, gasneti_nonatomic_)
GASNETI_ATOMIC_FENCED_SWAP_DEFN(nonatomic,gasneti_nonatomic_swap,_gasneti_scalar_atomic_swap,gasneti_nonatomic_)
GASNETI_ATOMIC_FENCED_ADDSUB_DEFN(nonatomic,gasneti_nonatomic_add,_gasneti_scalar_atomic_add,gasneti_nonatomic_)
GASNETI_ATOMIC_FENCED_ADDSUB_DEFN(nonatomic,gasneti_nonatomic_subtract,_gasneti_scalar_atomic_subtract,gasneti_nonatomic_)
#define GASNETI_HAVE_NONATOMIC_CAS 1
#define GASNETI_HAVE_NONATOMIC_ADD_SUB 1

/* Build gasneti_nonatomic32_t */
_GASNETI_NONATOMIC_DEFN(nonatomic32,32)
#define gasneti_nonatomic32_init            _gasneti_scalar_atomic_init
#define gasneti_nonatomic32_set(p,v,f)      GASNETI_ATOMIC_FENCED_SET(nonatomic,_gasneti_scalar_atomic_set,gasneti_nonatomic32_,p,v,f)
#define gasneti_nonatomic32_increment(p,f)  GASNETI_ATOMIC_FENCED_INCDEC(nonatomic,_gasneti_scalar_atomic_increment,gasneti_nonatomic32_,p,f)
#define gasneti_nonatomic32_decrement(p,f)  GASNETI_ATOMIC_FENCED_INCDEC(nonatomic,_gasneti_scalar_atomic_decrement,gasneti_nonatomic32_,p,f)
GASNETI_ATOMIC_FENCED_READ_DEFN(nonatomic,gasneti_nonatomic32_read,_gasneti_scalar_atomic_read,gasneti_nonatomic32_)
GASNETI_ATOMIC_FENCED_DECTEST_DEFN(nonatomic,gasneti_nonatomic32_decrement_and_test,_gasneti_scalar_atomic_decrement_and_test,gasneti_nonatomic32_)
GASNETI_ATOMIC_FENCED_CAS_DEFN(nonatomic,gasneti_nonatomic32_compare_and_swap,_gasneti_scalar_atomic_compare_and_swap,gasneti_nonatomic32_)
GASNETI_SCALAR_ATOMIC_SWAP_DEFN(_gasneti_scalar_atomic32_swap, gasneti_nonatomic32_)
GASNETI_ATOMIC_FENCED_SWAP_DEFN(nonatomic,gasneti_nonatomic32_swap,_gasneti_scalar_atomic32_swap,gasneti_nonatomic32_)
GASNETI_ATOMIC_FENCED_ADDSUB_DEFN(nonatomic,gasneti_nonatomic32_add,_gasneti_scalar_atomic_add,gasneti_nonatomic32_)
GASNETI_ATOMIC_FENCED_ADDSUB_DEFN(nonatomic,gasneti_nonatomic32_subtract,_gasneti_scalar_atomic_subtract,gasneti_nonatomic32_)

/* Build gasneti_nonatomic64_t */
_GASNETI_NONATOMIC_DEFN(nonatomic64,64)
#define gasneti_nonatomic64_init            _gasneti_scalar_atomic_init
#define gasneti_nonatomic64_set(p,v,f)      GASNETI_ATOMIC_FENCED_SET(nonatomic,_gasneti_scalar_atomic_set,gasneti_nonatomic64_,p,v,f)
#define gasneti_nonatomic64_increment(p,f)  GASNETI_ATOMIC_FENCED_INCDEC(nonatomic,_gasneti_scalar_atomic_increment,gasneti_nonatomic64_,p,f)
#define gasneti_nonatomic64_decrement(p,f)  GASNETI_ATOMIC_FENCED_INCDEC(nonatomic,_gasneti_scalar_atomic_decrement,gasneti_nonatomic64_,p,f)
GASNETI_ATOMIC_FENCED_READ_DEFN(nonatomic,gasneti_nonatomic64_read,_gasneti_scalar_atomic_read,gasneti_nonatomic64_)
GASNETI_ATOMIC_FENCED_DECTEST_DEFN(nonatomic,gasneti_nonatomic64_decrement_and_test,_gasneti_scalar_atomic_decrement_and_test,gasneti_nonatomic64_)
GASNETI_ATOMIC_FENCED_CAS_DEFN(nonatomic,gasneti_nonatomic64_compare_and_swap,_gasneti_scalar_atomic_compare_and_swap,gasneti_nonatomic64_)
GASNETI_SCALAR_ATOMIC_SWAP_DEFN(_gasneti_scalar_atomic64_swap, gasneti_nonatomic64_)
GASNETI_ATOMIC_FENCED_SWAP_DEFN(nonatomic,gasneti_nonatomic64_swap,_gasneti_scalar_atomic64_swap,gasneti_nonatomic64_)
GASNETI_ATOMIC_FENCED_ADDSUB_DEFN(nonatomic,gasneti_nonatomic64_add,_gasneti_scalar_atomic_add,gasneti_nonatomic64_)
GASNETI_ATOMIC_FENCED_ADDSUB_DEFN(nonatomic,gasneti_nonatomic64_subtract,_gasneti_scalar_atomic_subtract,gasneti_nonatomic64_)

/* ------------------------------------------------------------------------------------ */
/* GASNet weak atomics - these operations are guaranteed to be atomic if and only if
    the sole updates are from the host processor(s), with no signals involved.
   if !GASNETI_THREADS, they compile away to the non-atomic implementation
    thereby saving the overhead of unnecessary atomic-memory CPU instructions.
   Otherwise, they expand to regular gasneti_atomic_t's
 */
#if GASNETI_THREADS || defined(GASNETI_FORCE_TRUE_WEAKATOMICS)
  #define _GASNETI_WEAKATOMIC_ID(_id)     _CONCAT(GASNETI_ATOMIC,_id)
  #define _gasneti_weakatomic_id(_id)     _CONCAT(gasneti_atomic,_id)
  #ifdef GASNETI_HAVE_ATOMIC_CAS
    #define GASNETI_HAVE_WEAKATOMIC_CAS     1
  #endif
  #ifdef GASNETI_HAVE_ATOMIC_ADD_SUB
    #define GASNETI_HAVE_WEAKATOMIC_ADD_SUB 1
  #endif
#else
  #define _GASNETI_WEAKATOMIC_ID(_id)     _CONCAT(GASNETI_NONATOMIC,_id)
  #define _gasneti_weakatomic_id(_id)     _CONCAT(gasneti_nonatomic,_id)
  #define GASNETI_HAVE_WEAKATOMIC_CAS     1
  #define GASNETI_HAVE_WEAKATOMIC_ADD_SUB 1
#endif

typedef _gasneti_weakatomic_id(_t)             gasneti_weakatomic_t;
typedef _gasneti_weakatomic_id(_val_t)         gasneti_weakatomic_val_t;
typedef _gasneti_weakatomic_id(_sval_t)        gasneti_weakatomic_sval_t;
#define gasneti_weakatomic_init                _gasneti_weakatomic_id(_init)
#define gasneti_weakatomic_signed              _gasneti_weakatomic_id(_signed)
#define gasneti_weakatomic_set                 _gasneti_weakatomic_id(_set)
#define gasneti_weakatomic_read                _gasneti_weakatomic_id(_read)
#define gasneti_weakatomic_increment           _gasneti_weakatomic_id(_increment)
#define gasneti_weakatomic_decrement           _gasneti_weakatomic_id(_decrement)
#define gasneti_weakatomic_decrement_and_test  _gasneti_weakatomic_id(_decrement_and_test)
#ifdef GASNETI_HAVE_WEAKATOMIC_CAS
  #define gasneti_weakatomic_compare_and_swap  _gasneti_weakatomic_id(_compare_and_swap)
  #define gasneti_weakatomic_swap              _gasneti_weakatomic_id(_swap)
#endif
#ifdef GASNETI_HAVE_WEAKATOMIC_ADD_SUB
  #define gasneti_weakatomic_add               _gasneti_weakatomic_id(_add)
  #define gasneti_weakatomic_subtract          _gasneti_weakatomic_id(_subtract)
#endif

typedef _gasneti_weakatomic_id(32_t)           gasneti_weakatomic32_t;
#define gasneti_weakatomic32_init              _gasneti_weakatomic_id(32_init)
#define gasneti_weakatomic32_set               _gasneti_weakatomic_id(32_set)
#define gasneti_weakatomic32_read              _gasneti_weakatomic_id(32_read)
#define gasneti_weakatomic32_increment         _gasneti_weakatomic_id(32_increment)
#define gasneti_weakatomic32_decrement         _gasneti_weakatomic_id(32_decrement)
#define gasneti_weakatomic32_decrement_and_test _gasneti_weakatomic_id(32_decrement_and_test)
#define gasneti_weakatomic32_compare_and_swap  _gasneti_weakatomic_id(32_compare_and_swap)
#define gasneti_weakatomic32_swap              _gasneti_weakatomic_id(32_swap)
#define gasneti_weakatomic32_add               _gasneti_weakatomic_id(32_add)
#define gasneti_weakatomic32_subtract          _gasneti_weakatomic_id(32_subtract)

typedef _gasneti_weakatomic_id(64_t)           gasneti_weakatomic64_t;
#define gasneti_weakatomic64_init              _gasneti_weakatomic_id(64_init)
#define gasneti_weakatomic64_set               _gasneti_weakatomic_id(64_set)
#define gasneti_weakatomic64_read              _gasneti_weakatomic_id(64_read)
#define gasneti_weakatomic64_increment         _gasneti_weakatomic_id(64_increment)
#define gasneti_weakatomic64_decrement         _gasneti_weakatomic_id(64_decrement)
#define gasneti_weakatomic64_decrement_and_test _gasneti_weakatomic_id(64_decrement_and_test)
#define gasneti_weakatomic64_compare_and_swap  _gasneti_weakatomic_id(64_compare_and_swap)
#define gasneti_weakatomic64_swap              _gasneti_weakatomic_id(64_swap)
#define gasneti_weakatomic64_add               _gasneti_weakatomic_id(64_add)
#define gasneti_weakatomic64_subtract          _gasneti_weakatomic_id(64_subtract)

/* ------------------------------------------------------------------------------------ */
/* Configuration strings */

#if defined(GASNETI_FORCE_GENERIC_ATOMICOPS)
  #define GASNETI_ATOMIC_CONFIG   atomics_forced_mutex
#elif defined(GASNETI_FORCE_OS_ATOMICOPS)
  #define GASNETI_ATOMIC_CONFIG   atomics_forced_os
#elif defined(GASNETI_FORCE_COMPILER_ATOMICOPS)
  #define GASNETI_ATOMIC_CONFIG   atomics_forced_compiler
#elif (GASNETI_ATOMIC_IMPL_CONFIGURE == GASNETI_ATOMIC_IMPL_GENERIC)
  #define GASNETI_ATOMIC_CONFIG   atomics_mutex
#elif (GASNETI_ATOMIC_IMPL_CONFIGURE == GASNETI_ATOMIC_IMPL_COMPILER)
  #define GASNETI_ATOMIC_CONFIG   atomics_compiler
#elif (GASNETI_ATOMIC_IMPL_CONFIGURE == GASNETI_ATOMIC_IMPL_OS)
  #define GASNETI_ATOMIC_CONFIG   atomics_os
#else
  #define GASNETI_ATOMIC_CONFIG   atomics_native
#endif

#if defined(GASNETI_FORCE_GENERIC_ATOMICOPS)
  #define GASNETI_ATOMIC32_CONFIG   atomic32_forced_mutex
#elif defined(GASNETI_FORCE_OS_ATOMICOPS)
  #define GASNETI_ATOMIC32_CONFIG   atomic32_forced_os
#elif defined(GASNETI_FORCE_COMPILER_ATOMICOPS)
  #define GASNETI_ATOMIC32_CONFIG   atomic32_forced_compiler
#elif (GASNETI_ATOMIC32_IMPL_CONFIGURE == GASNETI_ATOMIC_IMPL_GENERIC)
  #define GASNETI_ATOMIC32_CONFIG   atomic32_mutex
#elif (GASNETI_ATOMIC32_IMPL_CONFIGURE == GASNETI_ATOMIC_IMPL_COMPILER)
  #define GASNETI_ATOMIC32_CONFIG   atomic32_compiler
#elif (GASNETI_ATOMIC32_IMPL_CONFIGURE == GASNETI_ATOMIC_IMPL_OS)
  #define GASNETI_ATOMIC32_CONFIG   atomic32_os
#else
  #define GASNETI_ATOMIC32_CONFIG   atomic32_native
#endif

#if defined(GASNETI_FORCE_GENERIC_ATOMICOPS)
  #define GASNETI_ATOMIC64_CONFIG   atomic64_forced_mutex
#elif defined(GASNETI_FORCE_OS_ATOMICOPS)
  #define GASNETI_ATOMIC64_CONFIG   atomic64_forced_os
#elif defined(GASNETI_FORCE_COMPILER_ATOMICOPS) && PLATFORM_ARCH_64
  #define GASNETI_ATOMIC64_CONFIG   atomic64_forced_compiler
#elif (GASNETI_ATOMIC64_IMPL_CONFIGURE == GASNETI_ATOMIC_IMPL_GENERIC)
  #define GASNETI_ATOMIC64_CONFIG   atomic64_mutex
#elif (GASNETI_ATOMIC64_IMPL_CONFIGURE == GASNETI_ATOMIC_IMPL_COMPILER)
  #define GASNETI_ATOMIC64_CONFIG   atomic64_compiler
#elif (GASNETI_ATOMIC64_IMPL_CONFIGURE == GASNETI_ATOMIC_IMPL_HYBRID)
  #define GASNETI_ATOMIC64_CONFIG   atomic64_hybrid
#elif (GASNETI_ATOMIC64_IMPL_CONFIGURE == GASNETI_ATOMIC_IMPL_OS)
  #define GASNETI_ATOMIC64_CONFIG   atomic64_os
#else
  #define GASNETI_ATOMIC64_CONFIG   atomic64_native
#endif

/* ------------------------------------------------------------------------------------ */
#endif