tsan_assist.h 6.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144
  1. /*
  2. * Copyright 2018 The OpenSSL Project Authors. All Rights Reserved.
  3. *
  4. * Licensed under the OpenSSL license (the "License"). You may not use
  5. * this file except in compliance with the License. You can obtain a copy
  6. * in the file LICENSE in the source distribution or at
  7. * https://www.openssl.org/source/license.html
  8. */
  9. /*
  10. * Contemporary compilers implement lock-free atomic memory access
  11. * primitives that facilitate writing "thread-opportunistic" or even real
  12. * multi-threading low-overhead code. "Thread-opportunistic" is when
  13. * exact result is not required, e.g. some statistics, or execution flow
  14. * doesn't have to be unambiguous. Simplest example is lazy "constant"
  15. * initialization when one can synchronize on variable itself, e.g.
  16. *
  17. * if (var == NOT_YET_INITIALIZED)
  18. * var = function_returning_same_value();
  19. *
  20. * This does work provided that loads and stores are single-instuction
  21. * operations (and integer ones are on *all* supported platforms), but
  22. * it upsets Thread Sanitizer. Suggested solution is
  23. *
  24. * if (tsan_load(&var) == NOT_YET_INITIALIZED)
  25. * tsan_store(&var, function_returning_same_value());
  26. *
  27. * Production machine code would be the same, so one can wonder why
  28. * bother. Having Thread Sanitizer accept "thread-opportunistic" code
  29. * allows to move on trouble-shooting real bugs.
  30. *
  31. * Resolving Thread Sanitizer nits was the initial purpose for this module,
  32. * but it was later extended with more nuanced primitives that are useful
  33. * even in "non-opportunistic" scenarios. Most notably verifying if a shared
  34. * structure is fully initialized and bypassing the initialization lock.
  35. * It's suggested to view macros defined in this module as "annotations" for
  36. * thread-safe lock-free code, "Thread-Safe ANnotations"...
  37. *
  38. * It's assumed that ATOMIC_{LONG|INT}_LOCK_FREE are assigned same value as
  39. * ATOMIC_POINTER_LOCK_FREE. And check for >= 2 ensures that corresponding
  40. * code is inlined. It should be noted that statistics counters become
  41. * accurate in such case.
  42. *
  43. * Special note about TSAN_QUALIFIER. It might be undesired to use it in
  44. * a shared header. Because whether operation on specific variable or member
  45. * is atomic or not might be irrelevant in other modules. In such case one
  46. * can use TSAN_QUALIFIER in cast specifically when it has to count.
  47. */
  48. #if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L \
  49. && !defined(__STDC_NO_ATOMICS__)
  50. # include <stdatomic.h>
  51. # if defined(ATOMIC_POINTER_LOCK_FREE) \
  52. && ATOMIC_POINTER_LOCK_FREE >= 2
  53. # define TSAN_QUALIFIER _Atomic
  54. # define tsan_load(ptr) atomic_load_explicit((ptr), memory_order_relaxed)
  55. # define tsan_store(ptr, val) atomic_store_explicit((ptr), (val), memory_order_relaxed)
  56. # define tsan_counter(ptr) atomic_fetch_add_explicit((ptr), 1, memory_order_relaxed)
  57. # define tsan_decr(ptr) atomic_fetch_add_explicit((ptr), -1, memory_order_relaxed)
  58. # define tsan_ld_acq(ptr) atomic_load_explicit((ptr), memory_order_acquire)
  59. # define tsan_st_rel(ptr, val) atomic_store_explicit((ptr), (val), memory_order_release)
  60. # endif
  61. #elif defined(__GNUC__) && defined(__ATOMIC_RELAXED)
  62. # if defined(__GCC_ATOMIC_POINTER_LOCK_FREE) \
  63. && __GCC_ATOMIC_POINTER_LOCK_FREE >= 2
  64. # define TSAN_QUALIFIER volatile
  65. # define tsan_load(ptr) __atomic_load_n((ptr), __ATOMIC_RELAXED)
  66. # define tsan_store(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_RELAXED)
  67. # define tsan_counter(ptr) __atomic_fetch_add((ptr), 1, __ATOMIC_RELAXED)
  68. # define tsan_decr(ptr) __atomic_fetch_add((ptr), -1, __ATOMIC_RELAXED)
  69. # define tsan_ld_acq(ptr) __atomic_load_n((ptr), __ATOMIC_ACQUIRE)
  70. # define tsan_st_rel(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_RELEASE)
  71. # endif
  72. #elif defined(_MSC_VER) && _MSC_VER>=1200 \
  73. && (defined(_M_IX86) || defined(_M_AMD64) || defined(_M_X64) || \
  74. defined(_M_ARM64) || (defined(_M_ARM) && _M_ARM >= 7))
  75. /*
  76. * There is subtle dependency on /volatile:<iso|ms> command-line option.
  77. * "ms" implies same semantic as memory_order_acquire for loads and
  78. * memory_order_release for stores, while "iso" - memory_order_relaxed for
  79. * either. Real complication is that defaults are different on x86 and ARM.
  80. * There is explanation for that, "ms" is backward compatible with earlier
  81. * compiler versions, while multi-processor ARM can be viewed as brand new
  82. * platform to MSC and its users, and with non-relaxed semantic taking toll
  83. * with additional instructions and penalties, it kind of makes sense to
  84. * default to "iso"...
  85. */
  86. # define TSAN_QUALIFIER volatile
  87. # if defined(_M_ARM) || defined(_M_ARM64)
  88. # define _InterlockedExchangeAdd _InterlockedExchangeAdd_nf
  89. # pragma intrinsic(_InterlockedExchangeAdd_nf)
  90. # pragma intrinsic(__iso_volatile_load32, __iso_volatile_store32)
  91. # ifdef _WIN64
  92. # define _InterlockedExchangeAdd64 _InterlockedExchangeAdd64_nf
  93. # pragma intrinsic(_InterlockedExchangeAdd64_nf)
  94. # pragma intrinsic(__iso_volatile_load64, __iso_volatile_store64)
  95. # define tsan_load(ptr) (sizeof(*(ptr)) == 8 ? __iso_volatile_load64(ptr) \
  96. : __iso_volatile_load32(ptr))
  97. # define tsan_store(ptr, val) (sizeof(*(ptr)) == 8 ? __iso_volatile_store64((ptr), (val)) \
  98. : __iso_volatile_store32((ptr), (val)))
  99. # else
  100. # define tsan_load(ptr) __iso_volatile_load32(ptr)
  101. # define tsan_store(ptr, val) __iso_volatile_store32((ptr), (val))
  102. # endif
  103. # else
  104. # define tsan_load(ptr) (*(ptr))
  105. # define tsan_store(ptr, val) (*(ptr) = (val))
  106. # endif
  107. # pragma intrinsic(_InterlockedExchangeAdd)
  108. # ifdef _WIN64
  109. # pragma intrinsic(_InterlockedExchangeAdd64)
  110. # define tsan_counter(ptr) (sizeof(*(ptr)) == 8 ? _InterlockedExchangeAdd64((ptr), 1) \
  111. : _InterlockedExchangeAdd((ptr), 1))
  112. # define tsan_decr(ptr) (sizeof(*(ptr)) == 8 ? _InterlockedExchangeAdd64((ptr), -1) \
  113. : _InterlockedExchangeAdd((ptr), -1))
  114. # else
  115. # define tsan_counter(ptr) _InterlockedExchangeAdd((ptr), 1)
  116. # define tsan_decr(ptr) _InterlockedExchangeAdd((ptr), -1)
  117. # endif
  118. # if !defined(_ISO_VOLATILE)
  119. # define tsan_ld_acq(ptr) (*(ptr))
  120. # define tsan_st_rel(ptr, val) (*(ptr) = (val))
  121. # endif
  122. #endif
  123. #ifndef TSAN_QUALIFIER
  124. # define TSAN_QUALIFIER volatile
  125. # define tsan_load(ptr) (*(ptr))
  126. # define tsan_store(ptr, val) (*(ptr) = (val))
  127. # define tsan_counter(ptr) ((*(ptr))++)
  128. # define tsan_decr(ptr) ((*(ptr))--)
  129. /*
  130. * Lack of tsan_ld_acq and tsan_ld_rel means that compiler support is not
  131. * sophisticated enough to support them. Code that relies on them should be
  132. * protected with #ifdef tsan_ld_acq with locked fallback.
  133. */
  134. #endif