Squashed 'third_party/abseil/' changes from ddf8e52a2..384af0e91
384af0e91 Export of internal Abseil changes
e7ca23aca Export of internal Abseil changes
4611a601a Export of internal Abseil changes
8a9ef3c5d Export of internal Abseil changes
9f8b87b71 Add missing word 'library' in the 'status' description (#868)
e2b1bab19 Export of internal Abseil changes
1bae23e32 Export of internal Abseil changes
6df644c56 Include the status library into the main README. (#863)
68f1ad932 Export of internal Abseil changes
52acfe6fc Export of internal Abseil changes
1918ad2ae Export of internal Abseil changes
938fd0f4e Export of internal Abseil changes
fbdff6f3a Export of internal Abseil changes
acf3390ca Export of internal Abseil changes
592924480 Export of internal Abseil changes
e80c0b353 Export of internal Abseil changes
5d8fc9192 Export of internal Abseil changes
e19260fd7 Export of internal Abseil changes
4fd9a1ec5 Export of internal Abseil changes
4ae673067 Export of internal Abseil changes
1b465af3b Export of internal Abseil changes
6b03bf543 fix build dll (#797)
0bbebc85c Export of internal Abseil changes
0453e1653 Export of internal Abseil changes
a4798817e Export of internal Abseil changes
e96d49687 Export of internal Abseil changes
731852f10 Fix stacktrace on aarch64 architecture. Fixes #805 (#827)
2e5f2bcfd moved deleted functions to public for better compiler errors. (#828)
e9e9b9fc7 Export of internal Abseil changes
b8e890f95 Export of internal Abseil changes
c9894d1dc Export of internal Abseil changes
e9b9e38f6 Export of internal Abseil changes
962b06754 Export of internal Abseil changes
5bf048b84 Export of internal Abseil changes
1e3d25b26 Export of internal Abseil changes
eb317a701 Export of internal Abseil changes
4b915e709 Export of internal Abseil changes
8f1c34a77 Export of internal Abseil changes
60d00a582 Export of internal Abseil changes
f3f785ab5 Export of internal Abseil changes
4b2fbb4ad Export of internal Abseil changes
e493d6acb fix compile fails with asan and -Wredundant-decls (#801)
e63a5a610 Export of internal Abseil changes
c678d6c6b Export of internal Abseil changes
4b4f9aae7 Export of internal Abseil changes
887d0eee6 Export of internal Abseil changes
b978fc02f Export of internal Abseil changes
093cc2760 Export of internal Abseil changes
40fdb59d3 btree: fix sign-compare warnings (#800)
1fd58b69c Export of internal Abseil changes
d1de75bf5 Export of internal Abseil changes
cad3f30b4 Export of internal Abseil changes
9927a0989 Export of internal Abseil changes
7680a5f8e Added missing asserts for seq.index() < capacity_ and unified their usage based on has_element(). (#781)
d3614de61 Export of internal Abseil changes
20feb1cdb Export of internal Abseil changes
c1ae0a497 Export of internal Abseil changes
6af91b351 Export of internal Abseil changes
f2c9c663d Export of internal Abseil changes
3c8b5d758 Export of internal Abseil changes
7ba8cdb56 Export of internal Abseil changes
930fbec75 Export of internal Abseil changes
0e9921b75 Export of internal Abseil changes
a4cbb5f69 Export of internal Abseil changes
4d2ff381a Export of internal Abseil changes
c03c18e7f Export of internal Abseil changes
b321ad86c Export of internal Abseil changes
fbf0fdab6 Export of internal Abseil changes
dc969f34a Export of internal Abseil changes
d0c433455 Export of internal Abseil changes
c6b3f2cf5 Export of internal Abseil changes
1beb3191c Export of internal Abseil changes
1b7e751e5 Export of internal Abseil changes
ce4bc9277 Export of internal Abseil changes
f72cc3516 Export of internal Abseil changes
f66bc7492 Export of internal Abseil changes
1995c6a3c Export of internal Abseil changes
184cf2524 Export of internal Abseil changes
82302f1e0 Export of internal Abseil changes
dea76486c Export of internal Abseil changes
d39fe6cd6 Export of internal Abseil changes
2c8a5b0d8 Export of internal Abseil changes
41a6263fd Export of internal Abseil changes
3c2bed2e7 Export of internal Abseil changes
ea8a689cf fix build on P9 (#739)
672d9e0ae Export of internal Abseil changes
f624790b7 Export of internal Abseil changes
55c04eb92 Export of internal Abseil changes
302b250e1 Disable pthread for standalone wasm build support (#721)
61d8bc057 Merge branch 'master' of https://github.com/abseil/abseil-cpp into master
4b5b25a28 cmake: remove unneeded enable_testing() cmd (#736)
63f2c695e cmake: flag conformance_testing as TESTONLY (#737)
d5269a8b6 Export of internal Abseil changes
23f1f9cf6 Typo in comment (#733)
259301a52 Fix CMake path for INSTALL_INCLUDEDIR (#723)
dc464c1dc Allow overriding Abseil IDE folder (#724)
bf655de09 Export of internal Abseil changes
38db52adb Export of internal Abseil changes
81f34df83 Export of internal Abseil changes
b86fff162 Export of internal Abseil changes
10cb35e45 Export of internal Abseil changes
4ccc0fce0 Export of internal Abseil changes
4a851046a Export of internal Abseil changes
ccdbb5941 Export of internal Abseil changes
01f5f81f9 Export of internal Abseil changes
2c92bdc7c Export of internal Abseil changes
e7ebf9803 Export of internal Abseil changes
2eba343b5 Export of internal Abseil changes
a8b03d90e Export of internal Abseil changes
1d31b5c36 Export of internal Abseil changes
da3a87690 Export of internal Abseil changes
8faf20461 Exclude empty directories (#697)
2069dc796 Export of internal Abseil changes
4832bf6bf Added a BUILD file in root to expose license. (#695)
af8f994af Export of internal Abseil changes
33caf1097 Export of internal Abseil changes
cf1a02e2d Export of internal Abseil changes
768eb2ca2 Export of internal Abseil changes
3f347c462 Fix build on riscv32 (#675)
62cf6a704 Export of internal Abseil changes
d118d4bb1 Export of internal Abseil changes
f2bc9d11e Fix public target name of the random library (#684)
0fecf0e63 Export of internal Abseil changes
cbfd0f0fe Export of internal Abseil changes
c45d1c09d Export of internal Abseil changes
a35ef8a62 Export of internal Abseil changes
bd317cae3 Export of internal Abseil changes
b11574465 fix MSVC warning 4245: conversion signed => unsigned during initialization (#678)
d85783fd0 Export of internal Abseil changes
a1d668990 Export of internal Abseil changes
ca9856cab Export of internal Abseil changes
6e18c7115 Export of internal Abseil changes
3f48ce1c4 init (#673)
cde2e2410 Export of internal Abseil changes
68494aae9 Fix CMake Threads dependency issue
902909a43 Export of internal Abseil changes
cb52b05ea Export of internal Abseil changes
1a02b7a20 Use "-lrt" instead of the resolved find_library result when linking librt (#665)
df60c82df Export of internal Abseil changes
b35973e3e Export of internal Abseil changes
db5773a72 Export of internal Abseil changes
71079e42c Export of internal Abseil changes
2946ac0de Use base_internal::AtomicHook instead of std::atomic (#661)
567bee2f7 Fix ABSL_RANDOM_RANDEN_COPTS setting on FreeBSD (#664)
bf6166a63 Export of internal Abseil changes
111260963 Export of internal Abseil changes
73ea9a957 Export of internal Abseil changes
c01b9916e Add option to use an externally provided GoogleTest target (for usage of abseil as add_subdirectory target) (#647)
d43b7997c Export of internal Abseil changes
62f05b1f5 Export of internal Abseil changes
fba8a316c Export of internal Abseil changes
79e0dc115 Export of internal Abseil changes
132d791b4 bazel: Add missing load statements for cc_binary (#645)
518f17501 Export of internal Abseil changes
092ed9793 Export of internal Abseil changes
2d2a8aea2 Export of internal Abseil changes
7853a7586 Export of internal Abseil changes
c6954897f Export of internal Abseil changes
b92f35f65 Fix CompressedTuple move constructor on MSVC (#637)
a877af1f2 Export of internal Abseil changes
d936052d3 Export of internal Abseil changes
238b9a59c Skip the .exe suffix in the helpshort filter on Windows (#629)
417ea99cb UWP doesn't allow reading regkeys (#594)
40a0e58eb Export of internal Abseil changes
cf3a1998e Export of internal Abseil changes
b19ba9676 Export of internal Abseil changes
06f0e767d BuildBreak: UWP apps can't call GetModuleHandle (#596)
bcefbdcdf Export of internal Abseil changes
0033c9ea9 Fix build on FreeBSD/powerpc (#616)
0d5ce2797 Export of internal Abseil changes
b69c7d880 Export of internal Abseil changes
2a5633fc0 Merge "Export of internal Abseil changes"
f9b3d6e49 Add RISCV support to GetProgramCounter() (#621)
914ff4451 Export of internal Abseil changes
0232c87f2 Add missing ABSL_HAVE_VDSO_SUPPORT conditional (#622)
3c8141051 Export of internal Abseil changes
c44657f55 Export of internal Abseil changes
98eb410c9 Export of internal Abseil changes
bf78e9773 Export of internal Abseil changes
d95d15671 Export of internal Abseil changes
24713a703 Export of internal Abseil changes
72382c21f Export of internal Abseil changes
08a7e7bf9 Export of internal Abseil changes
36bcd9599 Fix pointer format specifier in documentation (#614)
0f86336b6 Export of internal Abseil changes
c512f118d Export of internal Abseil changes
37dd2562e Export of internal Abseil changes
444277026 fix: Add support for more ARM processors detection (#608)
159bf2bf6 Export of internal Abseil changes
a2e6adecc Use https links. (#586)
564001ae5 Export of internal Abseil changes
b3aaac8a3 Export of internal Abseil changes
63ee2f887 Export of internal Abseil changes
a048203a8 Export of internal Abseil changes
1de016636 Export of internal Abseil changes
ad904b6cd Export of internal Abseil changes
7bd1935dc cmake: Fix x86_64 check on Windows for random copts (#518)
292351391 Export of internal Abseil changes
bf86cfe16 Export of internal Abseil changes
12bc53e03 Export of internal Abseil changes
1e39f8626 Export of internal Abseil changes
77f87009a Export of internal Abseil changes
d659fe54b Export of internal Abseil changes
a4b757b5d Export of internal Abseil changes
0514227d2 Export of internal Abseil changes
7f4fe64af Export of internal Abseil changes
16d9fd58a Export of internal Abseil changes
bcaae6009 Export of internal Abseil changes
8ba96a824 Export of internal Abseil changes
2103fd9ac Export of internal Abseil changes
3df7b52a6 Export of internal Abseil changes
fa8c75182 Export of internal Abseil changes
85092b4b6 Fix Conan builds (#400)
e96ae2203 Export of internal Abseil changes
20de2db74 Export of internal Abseil changes
846e5dbed Export of internal Abseil changes
83880e3d8 Merge branch 'master' of https://github.com/abseil/abseil-cpp
8207907f4 Export of internal Abseil changes
39d68a422 docs: fix typo (#397)
078b89b3c Export of internal Abseil changes
19b021cb3 Export of internal Abseil changes
ecc0033b5 Always enable proper symbolize implementation on Windows (#257)
2796d500a Export of internal Abseil changes
e4c8d0eb8 Export of internal Abseil changes
a15364ce4 Export of internal Abseil changes
ab3552a18 Export of internal Abseil changes
e9f9000c7 Fix ABSL_WAITER_MODE detection for mingw (#342)
abea769b5 Fix ABSL_HAVE_ALARM check on mingw (#341)
25597bdfc Export of internal Abseil changes
aad33fefa Export of internal Abseil changes
8fe7214fe Export of internal Abseil changes
debac94cf Export of internal Abseil changes
882b3501a Fix spelling errors (#384)
502efe6d7 Export of internal Abseil changes
ccdd1d57b Export of internal Abseil changes
Change-Id: I59864a0053f6e03d88cc9d5e2a92757039a05484
git-subtree-dir: third_party/abseil
git-subtree-split: 384af0e9141283172e2bff3210dae79fb7130d9c
diff --git a/absl/synchronization/BUILD.bazel b/absl/synchronization/BUILD.bazel
index 36dc98f..cd4009a 100644
--- a/absl/synchronization/BUILD.bazel
+++ b/absl/synchronization/BUILD.bazel
@@ -24,7 +24,7 @@
package(default_visibility = ["//visibility:public"])
-licenses(["notice"]) # Apache 2.0
+licenses(["notice"])
# Internal data structure for efficiently detecting mutex dependency cycles
cc_library(
@@ -43,6 +43,7 @@
deps = [
"//absl/base",
"//absl/base:base_internal",
+ "//absl/base:config",
"//absl/base:core_headers",
"//absl/base:malloc_internal",
"//absl/base:raw_logging_internal",
@@ -72,15 +73,14 @@
"internal/create_thread_identity.cc",
"internal/per_thread_sem.cc",
"internal/waiter.cc",
+ "mutex.cc",
"notification.cc",
- ] + select({
- "//conditions:default": ["mutex.cc"],
- }),
+ ],
hdrs = [
"barrier.h",
"blocking_counter.h",
"internal/create_thread_identity.h",
- "internal/mutex_nonprod.inc",
+ "internal/futex.h",
"internal/per_thread_sem.h",
"internal/waiter.h",
"mutex.h",
@@ -89,6 +89,7 @@
copts = ABSL_DEFAULT_COPTS,
linkopts = select({
"//absl:windows": [],
+ "//absl:wasm": [],
"//conditions:default": ["-pthread"],
}) + ABSL_DEFAULT_LINKOPTS,
deps = [
@@ -188,6 +189,7 @@
":synchronization",
":thread_pool",
"//absl/base",
+ "//absl/base:config",
"//absl/base:core_headers",
"//absl/base:raw_logging_internal",
"//absl/memory",
@@ -209,6 +211,7 @@
":synchronization",
":thread_pool",
"//absl/base",
+ "//absl/base:config",
"@com_github_google_benchmark//:benchmark_main",
],
alwayslink = 1,
@@ -247,6 +250,7 @@
deps = [
":synchronization",
"//absl/base",
+ "//absl/base:config",
"//absl/strings",
"//absl/time",
"@com_google_googletest//:gtest",
diff --git a/absl/synchronization/CMakeLists.txt b/absl/synchronization/CMakeLists.txt
index 3c47a1b..e633d0b 100644
--- a/absl/synchronization/CMakeLists.txt
+++ b/absl/synchronization/CMakeLists.txt
@@ -26,6 +26,7 @@
DEPS
absl::base
absl::base_internal
+ absl::config
absl::core_headers
absl::malloc_internal
absl::raw_logging_internal
@@ -51,7 +52,7 @@
"barrier.h"
"blocking_counter.h"
"internal/create_thread_identity.h"
- "internal/mutex_nonprod.inc"
+ "internal/futex.h"
"internal/per_thread_sem.h"
"internal/waiter.h"
"mutex.h"
@@ -148,6 +149,7 @@
absl::synchronization
absl::thread_pool
absl::base
+ absl::config
absl::core_headers
absl::memory
absl::raw_logging_internal
@@ -178,6 +180,7 @@
DEPS
absl::synchronization
absl::base
+ absl::config
absl::strings
absl::time
gmock
diff --git a/absl/synchronization/barrier.cc b/absl/synchronization/barrier.cc
index c2c539a..0dfd795 100644
--- a/absl/synchronization/barrier.cc
+++ b/absl/synchronization/barrier.cc
@@ -18,6 +18,7 @@
#include "absl/synchronization/mutex.h"
namespace absl {
+ABSL_NAMESPACE_BEGIN
// Return whether int *arg is zero.
static bool IsZero(void *arg) {
@@ -47,4 +48,5 @@
return this->num_to_exit_ == 0;
}
+ABSL_NAMESPACE_END
} // namespace absl
diff --git a/absl/synchronization/barrier.h b/absl/synchronization/barrier.h
index cb5d821..d8e7544 100644
--- a/absl/synchronization/barrier.h
+++ b/absl/synchronization/barrier.h
@@ -23,6 +23,7 @@
#include "absl/synchronization/mutex.h"
namespace absl {
+ABSL_NAMESPACE_BEGIN
// Barrier
//
@@ -73,5 +74,6 @@
int num_to_exit_ ABSL_GUARDED_BY(lock_);
};
+ABSL_NAMESPACE_END
} // namespace absl
#endif // ABSL_SYNCHRONIZATION_BARRIER_H_
diff --git a/absl/synchronization/blocking_counter.cc b/absl/synchronization/blocking_counter.cc
index 481a06b..3cea7ae 100644
--- a/absl/synchronization/blocking_counter.cc
+++ b/absl/synchronization/blocking_counter.cc
@@ -17,6 +17,7 @@
#include "absl/base/internal/raw_logging.h"
namespace absl {
+ABSL_NAMESPACE_BEGIN
// Return whether int *arg is zero.
static bool IsZero(void *arg) {
@@ -52,4 +53,5 @@
// after we return from this method.
}
+ABSL_NAMESPACE_END
} // namespace absl
diff --git a/absl/synchronization/blocking_counter.h b/absl/synchronization/blocking_counter.h
index 77560fc..1f53f9f 100644
--- a/absl/synchronization/blocking_counter.h
+++ b/absl/synchronization/blocking_counter.h
@@ -24,6 +24,7 @@
#include "absl/synchronization/mutex.h"
namespace absl {
+ABSL_NAMESPACE_BEGIN
// BlockingCounter
//
@@ -92,6 +93,7 @@
int num_waiting_ ABSL_GUARDED_BY(lock_);
};
+ABSL_NAMESPACE_END
} // namespace absl
#endif // ABSL_SYNCHRONIZATION_BLOCKING_COUNTER_H_
diff --git a/absl/synchronization/blocking_counter_test.cc b/absl/synchronization/blocking_counter_test.cc
index c63e339..2926224 100644
--- a/absl/synchronization/blocking_counter_test.cc
+++ b/absl/synchronization/blocking_counter_test.cc
@@ -22,6 +22,7 @@
#include "absl/time/time.h"
namespace absl {
+ABSL_NAMESPACE_BEGIN
namespace {
void PauseAndDecreaseCounter(BlockingCounter* counter, int* done) {
@@ -63,4 +64,5 @@
}
} // namespace
+ABSL_NAMESPACE_END
} // namespace absl
diff --git a/absl/synchronization/internal/create_thread_identity.cc b/absl/synchronization/internal/create_thread_identity.cc
index ce3676a..53a71b3 100644
--- a/absl/synchronization/internal/create_thread_identity.cc
+++ b/absl/synchronization/internal/create_thread_identity.cc
@@ -27,17 +27,18 @@
#include "absl/synchronization/internal/per_thread_sem.h"
namespace absl {
+ABSL_NAMESPACE_BEGIN
namespace synchronization_internal {
// ThreadIdentity storage is persistent, we maintain a free-list of previously
// released ThreadIdentity objects.
-static base_internal::SpinLock freelist_lock(
- base_internal::kLinkerInitialized);
-static base_internal::ThreadIdentity* thread_identity_freelist;
+ABSL_CONST_INIT static base_internal::SpinLock freelist_lock(
+ absl::kConstInit, base_internal::SCHEDULE_KERNEL_ONLY);
+ABSL_CONST_INIT static base_internal::ThreadIdentity* thread_identity_freelist;
// A per-thread destructor for reclaiming associated ThreadIdentity objects.
// Since we must preserve their storage we cache them for re-use.
-static void ReclaimThreadIdentity(void* v) {
+void ReclaimThreadIdentity(void* v) {
base_internal::ThreadIdentity* identity =
static_cast<base_internal::ThreadIdentity*>(v);
@@ -47,6 +48,8 @@
base_internal::LowLevelAlloc::Free(identity->per_thread_synch.all_locks);
}
+ PerThreadSem::Destroy(identity);
+
// We must explicitly clear the current thread's identity:
// (a) Subsequent (unrelated) per-thread destructors may require an identity.
// We must guarantee a new identity is used in this case (this instructor
@@ -84,7 +87,6 @@
pts->wake = false;
pts->cond_waiter = false;
pts->all_locks = nullptr;
- identity->waiter_state = {};
identity->blocked_count_ptr = nullptr;
identity->ticker.store(0, std::memory_order_relaxed);
identity->wait_start.store(0, std::memory_order_relaxed);
@@ -132,6 +134,7 @@
}
} // namespace synchronization_internal
+ABSL_NAMESPACE_END
} // namespace absl
#endif // ABSL_LOW_LEVEL_ALLOC_MISSING
diff --git a/absl/synchronization/internal/create_thread_identity.h b/absl/synchronization/internal/create_thread_identity.h
index ebb16c5..e121f68 100644
--- a/absl/synchronization/internal/create_thread_identity.h
+++ b/absl/synchronization/internal/create_thread_identity.h
@@ -29,12 +29,17 @@
#include "absl/base/port.h"
namespace absl {
+ABSL_NAMESPACE_BEGIN
namespace synchronization_internal {
// Allocates and attaches a ThreadIdentity object for the calling thread.
// For private use only.
base_internal::ThreadIdentity* CreateThreadIdentity();
+// A per-thread destructor for reclaiming associated ThreadIdentity objects.
+// For private use only.
+void ReclaimThreadIdentity(void* v);
+
// Returns the ThreadIdentity object representing the calling thread; guaranteed
// to be unique for its lifetime. The returned object will remain valid for the
// program's lifetime; although it may be re-assigned to a subsequent thread.
@@ -49,6 +54,7 @@
}
} // namespace synchronization_internal
+ABSL_NAMESPACE_END
} // namespace absl
#endif // ABSL_SYNCHRONIZATION_INTERNAL_CREATE_THREAD_IDENTITY_H_
diff --git a/absl/synchronization/internal/futex.h b/absl/synchronization/internal/futex.h
new file mode 100644
index 0000000..06fbd6d
--- /dev/null
+++ b/absl/synchronization/internal/futex.h
@@ -0,0 +1,154 @@
+// Copyright 2020 The Abseil Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#ifndef ABSL_SYNCHRONIZATION_INTERNAL_FUTEX_H_
+#define ABSL_SYNCHRONIZATION_INTERNAL_FUTEX_H_
+
+#include "absl/base/config.h"
+
+#ifdef _WIN32
+#include <windows.h>
+#else
+#include <sys/time.h>
+#include <unistd.h>
+#endif
+
+#ifdef __linux__
+#include <linux/futex.h>
+#include <sys/syscall.h>
+#endif
+
+#include <errno.h>
+#include <stdio.h>
+#include <time.h>
+
+#include <atomic>
+#include <cstdint>
+
+#include "absl/base/optimization.h"
+#include "absl/synchronization/internal/kernel_timeout.h"
+
+#ifdef ABSL_INTERNAL_HAVE_FUTEX
+#error ABSL_INTERNAL_HAVE_FUTEX may not be set on the command line
+#elif defined(__BIONIC__)
+// Bionic supports all the futex operations we need even when some of the futex
+// definitions are missing.
+#define ABSL_INTERNAL_HAVE_FUTEX
+#elif defined(__linux__) && defined(FUTEX_CLOCK_REALTIME)
+// FUTEX_CLOCK_REALTIME requires Linux >= 2.6.28.
+#define ABSL_INTERNAL_HAVE_FUTEX
+#endif
+
+#ifdef ABSL_INTERNAL_HAVE_FUTEX
+
+namespace absl {
+ABSL_NAMESPACE_BEGIN
+namespace synchronization_internal {
+
+// Some Android headers are missing these definitions even though they
+// support these futex operations.
+#ifdef __BIONIC__
+#ifndef SYS_futex
+#define SYS_futex __NR_futex
+#endif
+#ifndef FUTEX_WAIT_BITSET
+#define FUTEX_WAIT_BITSET 9
+#endif
+#ifndef FUTEX_PRIVATE_FLAG
+#define FUTEX_PRIVATE_FLAG 128
+#endif
+#ifndef FUTEX_CLOCK_REALTIME
+#define FUTEX_CLOCK_REALTIME 256
+#endif
+#ifndef FUTEX_BITSET_MATCH_ANY
+#define FUTEX_BITSET_MATCH_ANY 0xFFFFFFFF
+#endif
+#endif
+
+#if defined(__NR_futex_time64) && !defined(SYS_futex_time64)
+#define SYS_futex_time64 __NR_futex_time64
+#endif
+
+#if defined(SYS_futex_time64) && !defined(SYS_futex)
+#define SYS_futex SYS_futex_time64
+#endif
+
+class FutexImpl {
+ public:
+ static int WaitUntil(std::atomic<int32_t> *v, int32_t val,
+ KernelTimeout t) {
+ int err = 0;
+ if (t.has_timeout()) {
+ // https://locklessinc.com/articles/futex_cheat_sheet/
+ // Unlike FUTEX_WAIT, FUTEX_WAIT_BITSET uses absolute time.
+ struct timespec abs_timeout = t.MakeAbsTimespec();
+ // Atomically check that the futex value is still 0, and if it
+ // is, sleep until abs_timeout or until woken by FUTEX_WAKE.
+ err = syscall(
+ SYS_futex, reinterpret_cast<int32_t *>(v),
+ FUTEX_WAIT_BITSET | FUTEX_PRIVATE_FLAG | FUTEX_CLOCK_REALTIME, val,
+ &abs_timeout, nullptr, FUTEX_BITSET_MATCH_ANY);
+ } else {
+ // Atomically check that the futex value is still 0, and if it
+ // is, sleep until woken by FUTEX_WAKE.
+ err = syscall(SYS_futex, reinterpret_cast<int32_t *>(v),
+ FUTEX_WAIT | FUTEX_PRIVATE_FLAG, val, nullptr);
+ }
+ if (ABSL_PREDICT_FALSE(err != 0)) {
+ err = -errno;
+ }
+ return err;
+ }
+
+ static int WaitBitsetAbsoluteTimeout(std::atomic<int32_t> *v, int32_t val,
+ int32_t bits,
+ const struct timespec *abstime) {
+ int err = syscall(SYS_futex, reinterpret_cast<int32_t *>(v),
+ FUTEX_WAIT_BITSET | FUTEX_PRIVATE_FLAG, val, abstime,
+ nullptr, bits);
+ if (ABSL_PREDICT_FALSE(err != 0)) {
+ err = -errno;
+ }
+ return err;
+ }
+
+ static int Wake(std::atomic<int32_t> *v, int32_t count) {
+ int err = syscall(SYS_futex, reinterpret_cast<int32_t *>(v),
+ FUTEX_WAKE | FUTEX_PRIVATE_FLAG, count);
+ if (ABSL_PREDICT_FALSE(err < 0)) {
+ err = -errno;
+ }
+ return err;
+ }
+
+ // FUTEX_WAKE_BITSET
+ static int WakeBitset(std::atomic<int32_t> *v, int32_t count, int32_t bits) {
+ int err = syscall(SYS_futex, reinterpret_cast<int32_t *>(v),
+ FUTEX_WAKE_BITSET | FUTEX_PRIVATE_FLAG, count, nullptr,
+ nullptr, bits);
+ if (ABSL_PREDICT_FALSE(err < 0)) {
+ err = -errno;
+ }
+ return err;
+ }
+};
+
+class Futex : public FutexImpl {};
+
+} // namespace synchronization_internal
+ABSL_NAMESPACE_END
+} // namespace absl
+
+#endif // ABSL_INTERNAL_HAVE_FUTEX
+
+#endif // ABSL_SYNCHRONIZATION_INTERNAL_FUTEX_H_
diff --git a/absl/synchronization/internal/graphcycles.cc b/absl/synchronization/internal/graphcycles.cc
index 0c8c756..27fec21 100644
--- a/absl/synchronization/internal/graphcycles.cc
+++ b/absl/synchronization/internal/graphcycles.cc
@@ -37,6 +37,7 @@
#include <algorithm>
#include <array>
+#include <limits>
#include "absl/base/internal/hide_ptr.h"
#include "absl/base/internal/raw_logging.h"
#include "absl/base/internal/spinlock.h"
@@ -44,15 +45,16 @@
// Do not use STL. This module does not use standard memory allocation.
namespace absl {
+ABSL_NAMESPACE_BEGIN
namespace synchronization_internal {
namespace {
// Avoid LowLevelAlloc's default arena since it calls malloc hooks in
// which people are doing things like acquiring Mutexes.
-static absl::base_internal::SpinLock arena_mu(
- absl::base_internal::kLinkerInitialized);
-static base_internal::LowLevelAlloc::Arena* arena;
+ABSL_CONST_INIT static absl::base_internal::SpinLock arena_mu(
+ absl::kConstInit, base_internal::SCHEDULE_KERNEL_ONLY);
+ABSL_CONST_INIT static base_internal::LowLevelAlloc::Arena* arena;
static void InitArenaIfNecessary() {
arena_mu.Lock();
@@ -690,6 +692,7 @@
}
} // namespace synchronization_internal
+ABSL_NAMESPACE_END
} // namespace absl
#endif // ABSL_LOW_LEVEL_ALLOC_MISSING
diff --git a/absl/synchronization/internal/graphcycles.h b/absl/synchronization/internal/graphcycles.h
index e5bde00..ceba33e 100644
--- a/absl/synchronization/internal/graphcycles.h
+++ b/absl/synchronization/internal/graphcycles.h
@@ -36,11 +36,14 @@
// InsertEdge() is very fast when the edge already exists, and reasonably fast
// otherwise.
// FindPath() is linear in the size of the graph.
-// The current implemenation uses O(|V|+|E|) space.
+// The current implementation uses O(|V|+|E|) space.
#include <cstdint>
+#include "absl/base/config.h"
+
namespace absl {
+ABSL_NAMESPACE_BEGIN
namespace synchronization_internal {
// Opaque identifier for a graph node.
@@ -132,6 +135,7 @@
};
} // namespace synchronization_internal
+ABSL_NAMESPACE_END
} // namespace absl
#endif
diff --git a/absl/synchronization/internal/graphcycles_test.cc b/absl/synchronization/internal/graphcycles_test.cc
index 58e8477..74eaffe 100644
--- a/absl/synchronization/internal/graphcycles_test.cc
+++ b/absl/synchronization/internal/graphcycles_test.cc
@@ -25,6 +25,7 @@
#include "absl/base/macros.h"
namespace absl {
+ABSL_NAMESPACE_BEGIN
namespace synchronization_internal {
// We emulate a GraphCycles object with a node vector and an edge vector.
@@ -459,4 +460,5 @@
}
} // namespace synchronization_internal
+ABSL_NAMESPACE_END
} // namespace absl
diff --git a/absl/synchronization/internal/kernel_timeout.h b/absl/synchronization/internal/kernel_timeout.h
index 61c72e7..bbd4d2d 100644
--- a/absl/synchronization/internal/kernel_timeout.h
+++ b/absl/synchronization/internal/kernel_timeout.h
@@ -26,6 +26,7 @@
#define ABSL_SYNCHRONIZATION_INTERNAL_KERNEL_TIMEOUT_H_
#include <time.h>
+
#include <algorithm>
#include <limits>
@@ -34,6 +35,7 @@
#include "absl/time/time.h"
namespace absl {
+ABSL_NAMESPACE_BEGIN
namespace synchronization_internal {
class Futex;
@@ -56,6 +58,10 @@
bool has_timeout() const { return ns_ != 0; }
+ // Convert to parameter for sem_timedwait/futex/similar. Only for approved
+ // users. Do not call if !has_timeout.
+ struct timespec MakeAbsTimespec();
+
private:
// internal rep, not user visible: ns after unix epoch.
// zero = no timeout.
@@ -81,34 +87,6 @@
return x;
}
- // Convert to parameter for sem_timedwait/futex/similar. Only for approved
- // users. Do not call if !has_timeout.
- struct timespec MakeAbsTimespec() {
- int64_t n = ns_;
- static const int64_t kNanosPerSecond = 1000 * 1000 * 1000;
- if (n == 0) {
- ABSL_RAW_LOG(
- ERROR,
- "Tried to create a timespec from a non-timeout; never do this.");
- // But we'll try to continue sanely. no-timeout ~= saturated timeout.
- n = (std::numeric_limits<int64_t>::max)();
- }
-
- // Kernel APIs validate timespecs as being at or after the epoch,
- // despite the kernel time type being signed. However, no one can
- // tell the difference between a timeout at or before the epoch (since
- // all such timeouts have expired!)
- if (n < 0) n = 0;
-
- struct timespec abstime;
- int64_t seconds = (std::min)(n / kNanosPerSecond,
- int64_t{(std::numeric_limits<time_t>::max)()});
- abstime.tv_sec = static_cast<time_t>(seconds);
- abstime.tv_nsec =
- static_cast<decltype(abstime.tv_nsec)>(n % kNanosPerSecond);
- return abstime;
- }
-
#ifdef _WIN32
// Converts to milliseconds from now, or INFINITE when
// !has_timeout(). For use by SleepConditionVariableSRW on
@@ -147,7 +125,32 @@
friend class Waiter;
};
+inline struct timespec KernelTimeout::MakeAbsTimespec() {
+ int64_t n = ns_;
+ static const int64_t kNanosPerSecond = 1000 * 1000 * 1000;
+ if (n == 0) {
+ ABSL_RAW_LOG(
+ ERROR, "Tried to create a timespec from a non-timeout; never do this.");
+ // But we'll try to continue sanely. no-timeout ~= saturated timeout.
+ n = (std::numeric_limits<int64_t>::max)();
+ }
+
+ // Kernel APIs validate timespecs as being at or after the epoch,
+ // despite the kernel time type being signed. However, no one can
+ // tell the difference between a timeout at or before the epoch (since
+ // all such timeouts have expired!)
+ if (n < 0) n = 0;
+
+ struct timespec abstime;
+ int64_t seconds = (std::min)(n / kNanosPerSecond,
+ int64_t{(std::numeric_limits<time_t>::max)()});
+ abstime.tv_sec = static_cast<time_t>(seconds);
+ abstime.tv_nsec = static_cast<decltype(abstime.tv_nsec)>(n % kNanosPerSecond);
+ return abstime;
+}
+
} // namespace synchronization_internal
+ABSL_NAMESPACE_END
} // namespace absl
#endif // ABSL_SYNCHRONIZATION_INTERNAL_KERNEL_TIMEOUT_H_
diff --git a/absl/synchronization/internal/mutex_nonprod.cc b/absl/synchronization/internal/mutex_nonprod.cc
deleted file mode 100644
index 267deaf..0000000
--- a/absl/synchronization/internal/mutex_nonprod.cc
+++ /dev/null
@@ -1,318 +0,0 @@
-// Copyright 2017 The Abseil Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// https://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// Implementation of a small subset of Mutex and CondVar functionality
-// for platforms where the production implementation hasn't been fully
-// ported yet.
-
-#include "absl/synchronization/mutex.h"
-
-#if defined(_WIN32)
-#include <chrono> // NOLINT(build/c++11)
-#else
-#include <sys/time.h>
-#include <time.h>
-#endif
-
-#include <algorithm>
-
-#include "absl/base/internal/raw_logging.h"
-#include "absl/time/time.h"
-
-namespace absl {
-namespace synchronization_internal {
-
-namespace {
-
-// Return the current time plus the timeout.
-absl::Time DeadlineFromTimeout(absl::Duration timeout) {
- return absl::Now() + timeout;
-}
-
-// Limit the deadline to a positive, 32-bit time_t value to accommodate
-// implementation restrictions. This also deals with InfinitePast and
-// InfiniteFuture.
-absl::Time LimitedDeadline(absl::Time deadline) {
- deadline = std::max(absl::FromTimeT(0), deadline);
- deadline = std::min(deadline, absl::FromTimeT(0x7fffffff));
- return deadline;
-}
-
-} // namespace
-
-#if defined(_WIN32)
-
-MutexImpl::MutexImpl() {}
-
-MutexImpl::~MutexImpl() {
- if (locked_) {
- std_mutex_.unlock();
- }
-}
-
-void MutexImpl::Lock() {
- std_mutex_.lock();
- locked_ = true;
-}
-
-bool MutexImpl::TryLock() {
- bool locked = std_mutex_.try_lock();
- if (locked) locked_ = true;
- return locked;
-}
-
-void MutexImpl::Unlock() {
- locked_ = false;
- released_.SignalAll();
- std_mutex_.unlock();
-}
-
-CondVarImpl::CondVarImpl() {}
-
-CondVarImpl::~CondVarImpl() {}
-
-void CondVarImpl::Signal() { std_cv_.notify_one(); }
-
-void CondVarImpl::SignalAll() { std_cv_.notify_all(); }
-
-void CondVarImpl::Wait(MutexImpl* mu) {
- mu->released_.SignalAll();
- std_cv_.wait(mu->std_mutex_);
-}
-
-bool CondVarImpl::WaitWithDeadline(MutexImpl* mu, absl::Time deadline) {
- mu->released_.SignalAll();
- time_t when = ToTimeT(deadline);
- int64_t nanos = ToInt64Nanoseconds(deadline - absl::FromTimeT(when));
- std::chrono::system_clock::time_point deadline_tp =
- std::chrono::system_clock::from_time_t(when) +
- std::chrono::duration_cast<std::chrono::system_clock::duration>(
- std::chrono::nanoseconds(nanos));
- auto deadline_since_epoch =
- std::chrono::duration_cast<std::chrono::duration<double>>(
- deadline_tp - std::chrono::system_clock::from_time_t(0));
- return std_cv_.wait_until(mu->std_mutex_, deadline_tp) ==
- std::cv_status::timeout;
-}
-
-#else // ! _WIN32
-
-MutexImpl::MutexImpl() {
- ABSL_RAW_CHECK(pthread_mutex_init(&pthread_mutex_, nullptr) == 0,
- "pthread error");
-}
-
-MutexImpl::~MutexImpl() {
- if (locked_) {
- ABSL_RAW_CHECK(pthread_mutex_unlock(&pthread_mutex_) == 0, "pthread error");
- }
- ABSL_RAW_CHECK(pthread_mutex_destroy(&pthread_mutex_) == 0, "pthread error");
-}
-
-void MutexImpl::Lock() {
- ABSL_RAW_CHECK(pthread_mutex_lock(&pthread_mutex_) == 0, "pthread error");
- locked_ = true;
-}
-
-bool MutexImpl::TryLock() {
- bool locked = (0 == pthread_mutex_trylock(&pthread_mutex_));
- if (locked) locked_ = true;
- return locked;
-}
-
-void MutexImpl::Unlock() {
- locked_ = false;
- released_.SignalAll();
- ABSL_RAW_CHECK(pthread_mutex_unlock(&pthread_mutex_) == 0, "pthread error");
-}
-
-CondVarImpl::CondVarImpl() {
- ABSL_RAW_CHECK(pthread_cond_init(&pthread_cv_, nullptr) == 0,
- "pthread error");
-}
-
-CondVarImpl::~CondVarImpl() {
- ABSL_RAW_CHECK(pthread_cond_destroy(&pthread_cv_) == 0, "pthread error");
-}
-
-void CondVarImpl::Signal() {
- ABSL_RAW_CHECK(pthread_cond_signal(&pthread_cv_) == 0, "pthread error");
-}
-
-void CondVarImpl::SignalAll() {
- ABSL_RAW_CHECK(pthread_cond_broadcast(&pthread_cv_) == 0, "pthread error");
-}
-
-void CondVarImpl::Wait(MutexImpl* mu) {
- mu->released_.SignalAll();
- ABSL_RAW_CHECK(pthread_cond_wait(&pthread_cv_, &mu->pthread_mutex_) == 0,
- "pthread error");
-}
-
-bool CondVarImpl::WaitWithDeadline(MutexImpl* mu, absl::Time deadline) {
- mu->released_.SignalAll();
- struct timespec ts = ToTimespec(deadline);
- int rc = pthread_cond_timedwait(&pthread_cv_, &mu->pthread_mutex_, &ts);
- if (rc == ETIMEDOUT) return true;
- ABSL_RAW_CHECK(rc == 0, "pthread error");
- return false;
-}
-
-#endif // ! _WIN32
-
-void MutexImpl::Await(const Condition& cond) {
- if (cond.Eval()) return;
- released_.SignalAll();
- do {
- released_.Wait(this);
- } while (!cond.Eval());
-}
-
-bool MutexImpl::AwaitWithDeadline(const Condition& cond, absl::Time deadline) {
- if (cond.Eval()) return true;
- released_.SignalAll();
- while (true) {
- if (released_.WaitWithDeadline(this, deadline)) return false;
- if (cond.Eval()) return true;
- }
-}
-
-} // namespace synchronization_internal
-
-Mutex::Mutex() {}
-
-Mutex::~Mutex() {}
-
-void Mutex::Lock() { impl()->Lock(); }
-
-void Mutex::Unlock() { impl()->Unlock(); }
-
-bool Mutex::TryLock() { return impl()->TryLock(); }
-
-void Mutex::ReaderLock() { Lock(); }
-
-void Mutex::ReaderUnlock() { Unlock(); }
-
-void Mutex::Await(const Condition& cond) { impl()->Await(cond); }
-
-void Mutex::LockWhen(const Condition& cond) {
- Lock();
- Await(cond);
-}
-
-bool Mutex::AwaitWithDeadline(const Condition& cond, absl::Time deadline) {
- return impl()->AwaitWithDeadline(
- cond, synchronization_internal::LimitedDeadline(deadline));
-}
-
-bool Mutex::AwaitWithTimeout(const Condition& cond, absl::Duration timeout) {
- return AwaitWithDeadline(
- cond, synchronization_internal::DeadlineFromTimeout(timeout));
-}
-
-bool Mutex::LockWhenWithDeadline(const Condition& cond, absl::Time deadline) {
- Lock();
- return AwaitWithDeadline(cond, deadline);
-}
-
-bool Mutex::LockWhenWithTimeout(const Condition& cond, absl::Duration timeout) {
- return LockWhenWithDeadline(
- cond, synchronization_internal::DeadlineFromTimeout(timeout));
-}
-
-void Mutex::ReaderLockWhen(const Condition& cond) {
- ReaderLock();
- Await(cond);
-}
-
-bool Mutex::ReaderLockWhenWithTimeout(const Condition& cond,
- absl::Duration timeout) {
- return LockWhenWithTimeout(cond, timeout);
-}
-bool Mutex::ReaderLockWhenWithDeadline(const Condition& cond,
- absl::Time deadline) {
- return LockWhenWithDeadline(cond, deadline);
-}
-
-void Mutex::EnableDebugLog(const char*) {}
-void Mutex::EnableInvariantDebugging(void (*)(void*), void*) {}
-void Mutex::ForgetDeadlockInfo() {}
-void Mutex::AssertHeld() const {}
-void Mutex::AssertReaderHeld() const {}
-void Mutex::AssertNotHeld() const {}
-
-CondVar::CondVar() {}
-
-CondVar::~CondVar() {}
-
-void CondVar::Signal() { impl()->Signal(); }
-
-void CondVar::SignalAll() { impl()->SignalAll(); }
-
-void CondVar::Wait(Mutex* mu) { return impl()->Wait(mu->impl()); }
-
-bool CondVar::WaitWithDeadline(Mutex* mu, absl::Time deadline) {
- return impl()->WaitWithDeadline(
- mu->impl(), synchronization_internal::LimitedDeadline(deadline));
-}
-
-bool CondVar::WaitWithTimeout(Mutex* mu, absl::Duration timeout) {
- return WaitWithDeadline(mu, absl::Now() + timeout);
-}
-
-void CondVar::EnableDebugLog(const char*) {}
-
-#ifdef THREAD_SANITIZER
-extern "C" void __tsan_read1(void *addr);
-#else
-#define __tsan_read1(addr) // do nothing if TSan not enabled
-#endif
-
-// A function that just returns its argument, dereferenced
-static bool Dereference(void *arg) {
- // ThreadSanitizer does not instrument this file for memory accesses.
- // This function dereferences a user variable that can participate
- // in a data race, so we need to manually tell TSan about this memory access.
- __tsan_read1(arg);
- return *(static_cast<bool *>(arg));
-}
-
-Condition::Condition() {} // null constructor, used for kTrue only
-const Condition Condition::kTrue;
-
-Condition::Condition(bool (*func)(void *), void *arg)
- : eval_(&CallVoidPtrFunction),
- function_(func),
- method_(nullptr),
- arg_(arg) {}
-
-bool Condition::CallVoidPtrFunction(const Condition *c) {
- return (*c->function_)(c->arg_);
-}
-
-Condition::Condition(const bool *cond)
- : eval_(CallVoidPtrFunction),
- function_(Dereference),
- method_(nullptr),
- // const_cast is safe since Dereference does not modify arg
- arg_(const_cast<bool *>(cond)) {}
-
-bool Condition::Eval() const {
- // eval_ == null for kTrue
- return (this->eval_ == nullptr) || (*this->eval_)(this);
-}
-
-void RegisterSymbolizer(bool (*)(const void*, char*, int)) {}
-
-} // namespace absl
diff --git a/absl/synchronization/internal/mutex_nonprod.inc b/absl/synchronization/internal/mutex_nonprod.inc
deleted file mode 100644
index b8d5af7..0000000
--- a/absl/synchronization/internal/mutex_nonprod.inc
+++ /dev/null
@@ -1,259 +0,0 @@
-// Do not include. This is an implementation detail of base/mutex.h.
-//
-// Declares three classes:
-//
-// base::internal::MutexImpl - implementation helper for Mutex
-// base::internal::CondVarImpl - implementation helper for CondVar
-// base::internal::SynchronizationStorage<T> - implementation helper for
-// Mutex, CondVar
-
-#include <type_traits>
-
-#if defined(_WIN32)
-#include <condition_variable>
-#include <mutex>
-#else
-#include <pthread.h>
-#endif
-
-#include "absl/base/call_once.h"
-#include "absl/time/time.h"
-
-// Declare that Mutex::ReaderLock is actually Lock(). Intended primarily
-// for tests, and even then as a last resort.
-#ifdef ABSL_MUTEX_READER_LOCK_IS_EXCLUSIVE
-#error ABSL_MUTEX_READER_LOCK_IS_EXCLUSIVE cannot be directly set
-#else
-#define ABSL_MUTEX_READER_LOCK_IS_EXCLUSIVE 1
-#endif
-
-// Declare that Mutex::EnableInvariantDebugging is not implemented.
-// Intended primarily for tests, and even then as a last resort.
-#ifdef ABSL_MUTEX_ENABLE_INVARIANT_DEBUGGING_NOT_IMPLEMENTED
-#error ABSL_MUTEX_ENABLE_INVARIANT_DEBUGGING_NOT_IMPLEMENTED cannot be directly set
-#else
-#define ABSL_MUTEX_ENABLE_INVARIANT_DEBUGGING_NOT_IMPLEMENTED 1
-#endif
-
-namespace absl {
-class Condition;
-
-namespace synchronization_internal {
-
-class MutexImpl;
-
-// Do not use this implementation detail of CondVar. Provides most of the
-// implementation, but should not be placed directly in static storage
-// because it will not linker initialize properly. See
-// SynchronizationStorage<T> below for what we mean by linker
-// initialization.
-class CondVarImpl {
- public:
- CondVarImpl();
- CondVarImpl(const CondVarImpl&) = delete;
- CondVarImpl& operator=(const CondVarImpl&) = delete;
- ~CondVarImpl();
-
- void Signal();
- void SignalAll();
- void Wait(MutexImpl* mutex);
- bool WaitWithDeadline(MutexImpl* mutex, absl::Time deadline);
-
- private:
-#if defined(_WIN32)
- std::condition_variable_any std_cv_;
-#else
- pthread_cond_t pthread_cv_;
-#endif
-};
-
-// Do not use this implementation detail of Mutex. Provides most of the
-// implementation, but should not be placed directly in static storage
-// because it will not linker initialize properly. See
-// SynchronizationStorage<T> below for what we mean by linker
-// initialization.
-class MutexImpl {
- public:
- MutexImpl();
- MutexImpl(const MutexImpl&) = delete;
- MutexImpl& operator=(const MutexImpl&) = delete;
- ~MutexImpl();
-
- void Lock();
- bool TryLock();
- void Unlock();
- void Await(const Condition& cond);
- bool AwaitWithDeadline(const Condition& cond, absl::Time deadline);
-
- private:
- friend class CondVarImpl;
-
-#if defined(_WIN32)
- std::mutex std_mutex_;
-#else
- pthread_mutex_t pthread_mutex_;
-#endif
-
- // True if the underlying mutex is locked. If the destructor is entered
- // while locked_, the underlying mutex is unlocked. Mutex supports
- // destruction while locked, but the same is undefined behavior for both
- // pthread_mutex_t and std::mutex.
- bool locked_ = false;
-
- // Signaled before releasing the lock, in support of Await.
- CondVarImpl released_;
-};
-
-// Do not use this implementation detail of CondVar and Mutex. A storage
-// space for T that supports a LinkerInitialized constructor. T must
-// have a default constructor, which is called by the first call to
-// get(). T's destructor is never called if the LinkerInitialized
-// constructor is called.
-//
-// Objects constructed with the default constructor are constructed and
-// destructed like any other object, and should never be allocated in
-// static storage.
-//
-// Objects constructed with the LinkerInitialized constructor should
-// always be in static storage. For such objects, calls to get() are always
-// valid, except from signal handlers.
-//
-// Note that this implementation relies on undefined language behavior that
-// are known to hold for the set of supported compilers. An analysis
-// follows.
-//
-// From the C++11 standard:
-//
-// [basic.life] says an object has non-trivial initialization if it is of
-// class type and it is initialized by a constructor other than a trivial
-// default constructor. (the LinkerInitialized constructor is
-// non-trivial)
-//
-// [basic.life] says the lifetime of an object with a non-trivial
-// constructor begins when the call to the constructor is complete.
-//
-// [basic.life] says the lifetime of an object with non-trivial destructor
-// ends when the call to the destructor begins.
-//
-// [basic.life] p5 specifies undefined behavior when accessing non-static
-// members of an instance outside its
-// lifetime. (SynchronizationStorage::get() access non-static members)
-//
-// So, LinkerInitialized object of SynchronizationStorage uses a
-// non-trivial constructor, which is called at some point during dynamic
-// initialization, and is therefore subject to order of dynamic
-// initialization bugs, where get() is called before the object's
-// constructor is, resulting in undefined behavior.
-//
-// Similarly, a LinkerInitialized SynchronizationStorage object has a
-// non-trivial destructor, and so its lifetime ends at some point during
-// destruction of objects with static storage duration [basic.start.term]
-// p4. There is a window where other exit code could call get() after this
-// occurs, resulting in undefined behavior.
-//
-// Combined, these statements imply that LinkerInitialized instances
-// of SynchronizationStorage<T> rely on undefined behavior.
-//
-// However, in practice, the implementation works on all supported
-// compilers. Specifically, we rely on:
-//
-// a) zero-initialization being sufficient to initialize
-// LinkerInitialized instances for the purposes of calling
-// get(), regardless of when the constructor is called. This is
-// because the is_dynamic_ boolean is correctly zero-initialized to
-// false.
-//
-// b) the LinkerInitialized constructor is a NOP, and immaterial to
-// even to concurrent calls to get().
-//
-// c) the destructor being a NOP for LinkerInitialized objects
-// (guaranteed by a check for !is_dynamic_), and so any concurrent and
-// subsequent calls to get() functioning as if the destructor were not
-// called, by virtue of the instances' storage remaining valid after the
-// destructor runs.
-//
-// d) That a-c apply transitively when SynchronizationStorage<T> is the
-// only member of a class allocated in static storage.
-//
-// Nothing in the language standard guarantees that a-d hold. In practice,
-// these hold in all supported compilers.
-//
-// Future direction:
-//
-// Ideally, we would simply use std::mutex or a similar class, which when
-// allocated statically would support use immediately after static
-// initialization up until static storage is reclaimed (i.e. the properties
-// we require of all "linker initialized" instances).
-//
-// Regarding construction in static storage, std::mutex is required to
-// provide a constexpr default constructor [thread.mutex.class], which
-// ensures the instance's lifetime begins with static initialization
-// [basic.start.init], and so is immune to any problems caused by the order
-// of dynamic initialization. However, as of this writing Microsoft's
-// Visual Studio does not provide a constexpr constructor for std::mutex.
-// See
-// https://blogs.msdn.microsoft.com/vcblog/2015/06/02/constexpr-complete-for-vs-2015-rtm-c11-compiler-c17-stl/
-//
-// Regarding destruction of instances in static storage, [basic.life] does
-// say an object ends when storage in which the occupies is released, in
-// the case of non-trivial destructor. However, std::mutex is not specified
-// to have a trivial destructor.
-//
-// So, we would need a class with a constexpr default constructor and a
-// trivial destructor. Today, we can achieve neither desired property using
-// std::mutex directly.
-template <typename T>
-class SynchronizationStorage {
- public:
- // Instances allocated on the heap or on the stack should use the default
- // constructor.
- SynchronizationStorage()
- : is_dynamic_(true), once_() {}
-
- // Instances allocated in static storage (not on the heap, not on the
- // stack) should use this constructor.
- explicit SynchronizationStorage(base_internal::LinkerInitialized) {}
-
- constexpr explicit SynchronizationStorage(absl::ConstInitType)
- : is_dynamic_(false), once_(), space_{{0}} {}
-
- SynchronizationStorage(SynchronizationStorage&) = delete;
- SynchronizationStorage& operator=(SynchronizationStorage&) = delete;
-
- ~SynchronizationStorage() {
- if (is_dynamic_) {
- get()->~T();
- }
- }
-
- // Retrieve the object in storage. This is fast and thread safe, but does
- // incur the cost of absl::call_once().
- //
- // For instances in static storage constructed with the
- // LinkerInitialized constructor, may be called at any time without
- // regard for order of dynamic initialization or destruction of objects
- // in static storage. See the class comment for caveats.
- T* get() {
- absl::call_once(once_, SynchronizationStorage::Construct, this);
- return reinterpret_cast<T*>(&space_);
- }
-
- private:
- static void Construct(SynchronizationStorage<T>* self) {
- new (&self->space_) T();
- }
-
- // When true, T's destructor is run when this is destructed.
- //
- // The LinkerInitialized constructor assumes this value will be set
- // false by static initialization.
- bool is_dynamic_;
-
- absl::once_flag once_;
-
- // An aligned space for T.
- typename std::aligned_storage<sizeof(T), alignof(T)>::type space_;
-};
-
-} // namespace synchronization_internal
-} // namespace absl
diff --git a/absl/synchronization/internal/per_thread_sem.cc b/absl/synchronization/internal/per_thread_sem.cc
index b7014fb..821ca9b 100644
--- a/absl/synchronization/internal/per_thread_sem.cc
+++ b/absl/synchronization/internal/per_thread_sem.cc
@@ -25,6 +25,7 @@
#include "absl/synchronization/internal/waiter.h"
namespace absl {
+ABSL_NAMESPACE_BEGIN
namespace synchronization_internal {
void PerThreadSem::SetThreadBlockedCounter(std::atomic<int> *counter) {
@@ -40,12 +41,16 @@
}
void PerThreadSem::Init(base_internal::ThreadIdentity *identity) {
- Waiter::GetWaiter(identity)->Init();
+ new (Waiter::GetWaiter(identity)) Waiter();
identity->ticker.store(0, std::memory_order_relaxed);
identity->wait_start.store(0, std::memory_order_relaxed);
identity->is_idle.store(false, std::memory_order_relaxed);
}
+void PerThreadSem::Destroy(base_internal::ThreadIdentity *identity) {
+ Waiter::GetWaiter(identity)->~Waiter();
+}
+
void PerThreadSem::Tick(base_internal::ThreadIdentity *identity) {
const int ticker =
identity->ticker.fetch_add(1, std::memory_order_relaxed) + 1;
@@ -58,6 +63,7 @@
}
} // namespace synchronization_internal
+ABSL_NAMESPACE_END
} // namespace absl
extern "C" {
diff --git a/absl/synchronization/internal/per_thread_sem.h b/absl/synchronization/internal/per_thread_sem.h
index e7da070..2228b6e 100644
--- a/absl/synchronization/internal/per_thread_sem.h
+++ b/absl/synchronization/internal/per_thread_sem.h
@@ -32,6 +32,7 @@
#include "absl/synchronization/internal/kernel_timeout.h"
namespace absl {
+ABSL_NAMESPACE_BEGIN
class Mutex;
@@ -65,6 +66,10 @@
// REQUIRES: May only be called by ThreadIdentity.
static void Init(base_internal::ThreadIdentity* identity);
+ // Destroy the PerThreadSem associated with "identity".
+ // REQUIRES: May only be called by ThreadIdentity.
+ static void Destroy(base_internal::ThreadIdentity* identity);
+
// Increments "identity"'s count.
static inline void Post(base_internal::ThreadIdentity* identity);
@@ -73,13 +78,15 @@
// !t.has_timeout() => Wait(t) will return true.
static inline bool Wait(KernelTimeout t);
- // White-listed callers.
+ // Permitted callers.
friend class PerThreadSemTest;
friend class absl::Mutex;
friend absl::base_internal::ThreadIdentity* CreateThreadIdentity();
+ friend void ReclaimThreadIdentity(void* v);
};
} // namespace synchronization_internal
+ABSL_NAMESPACE_END
} // namespace absl
// In some build configurations we pass --detect-odr-violations to the
diff --git a/absl/synchronization/internal/per_thread_sem_test.cc b/absl/synchronization/internal/per_thread_sem_test.cc
index dba7239..8cf59e6 100644
--- a/absl/synchronization/internal/per_thread_sem_test.cc
+++ b/absl/synchronization/internal/per_thread_sem_test.cc
@@ -23,6 +23,7 @@
#include <thread> // NOLINT(build/c++11)
#include "gtest/gtest.h"
+#include "absl/base/config.h"
#include "absl/base/internal/cycleclock.h"
#include "absl/base/internal/thread_identity.h"
#include "absl/strings/str_cat.h"
@@ -33,6 +34,7 @@
// primitives which might use PerThreadSem, most notably absl::Mutex.
namespace absl {
+ABSL_NAMESPACE_BEGIN
namespace synchronization_internal {
class SimpleSemaphore {
@@ -175,4 +177,5 @@
} // namespace
} // namespace synchronization_internal
+ABSL_NAMESPACE_END
} // namespace absl
diff --git a/absl/synchronization/internal/thread_pool.h b/absl/synchronization/internal/thread_pool.h
index a00f2be..0cb96da 100644
--- a/absl/synchronization/internal/thread_pool.h
+++ b/absl/synchronization/internal/thread_pool.h
@@ -26,6 +26,7 @@
#include "absl/synchronization/mutex.h"
namespace absl {
+ABSL_NAMESPACE_BEGIN
namespace synchronization_internal {
// A simple ThreadPool implementation for tests.
@@ -86,6 +87,7 @@
};
} // namespace synchronization_internal
+ABSL_NAMESPACE_END
} // namespace absl
#endif // ABSL_SYNCHRONIZATION_INTERNAL_THREAD_POOL_H_
diff --git a/absl/synchronization/internal/waiter.cc b/absl/synchronization/internal/waiter.cc
index f17ea56..2123be6 100644
--- a/absl/synchronization/internal/waiter.cc
+++ b/absl/synchronization/internal/waiter.cc
@@ -48,7 +48,9 @@
#include "absl/base/optimization.h"
#include "absl/synchronization/internal/kernel_timeout.h"
+
namespace absl {
+ABSL_NAMESPACE_BEGIN
namespace synchronization_internal {
static void MaybeBecomeIdle() {
@@ -65,73 +67,21 @@
#if ABSL_WAITER_MODE == ABSL_WAITER_MODE_FUTEX
-// Some Android headers are missing these definitions even though they
-// support these futex operations.
-#ifdef __BIONIC__
-#ifndef SYS_futex
-#define SYS_futex __NR_futex
-#endif
-#ifndef FUTEX_WAIT_BITSET
-#define FUTEX_WAIT_BITSET 9
-#endif
-#ifndef FUTEX_PRIVATE_FLAG
-#define FUTEX_PRIVATE_FLAG 128
-#endif
-#ifndef FUTEX_CLOCK_REALTIME
-#define FUTEX_CLOCK_REALTIME 256
-#endif
-#ifndef FUTEX_BITSET_MATCH_ANY
-#define FUTEX_BITSET_MATCH_ANY 0xFFFFFFFF
-#endif
-#endif
-
-class Futex {
- public:
- static int WaitUntil(std::atomic<int32_t> *v, int32_t val,
- KernelTimeout t) {
- int err = 0;
- if (t.has_timeout()) {
- // https://locklessinc.com/articles/futex_cheat_sheet/
- // Unlike FUTEX_WAIT, FUTEX_WAIT_BITSET uses absolute time.
- struct timespec abs_timeout = t.MakeAbsTimespec();
- // Atomically check that the futex value is still 0, and if it
- // is, sleep until abs_timeout or until woken by FUTEX_WAKE.
- err = syscall(
- SYS_futex, reinterpret_cast<int32_t *>(v),
- FUTEX_WAIT_BITSET | FUTEX_PRIVATE_FLAG | FUTEX_CLOCK_REALTIME, val,
- &abs_timeout, nullptr, FUTEX_BITSET_MATCH_ANY);
- } else {
- // Atomically check that the futex value is still 0, and if it
- // is, sleep until woken by FUTEX_WAKE.
- err = syscall(SYS_futex, reinterpret_cast<int32_t *>(v),
- FUTEX_WAIT | FUTEX_PRIVATE_FLAG, val, nullptr);
- }
- if (err != 0) {
- err = -errno;
- }
- return err;
- }
-
- static int Wake(std::atomic<int32_t> *v, int32_t count) {
- int err = syscall(SYS_futex, reinterpret_cast<int32_t *>(v),
- FUTEX_WAKE | FUTEX_PRIVATE_FLAG, count);
- if (ABSL_PREDICT_FALSE(err < 0)) {
- err = -errno;
- }
- return err;
- }
-};
-
-void Waiter::Init() {
+Waiter::Waiter() {
futex_.store(0, std::memory_order_relaxed);
}
+Waiter::~Waiter() = default;
+
bool Waiter::Wait(KernelTimeout t) {
// Loop until we can atomically decrement futex from a positive
// value, waiting on a futex while we believe it is zero.
+ // Note that, since the thread ticker is just reset, we don't need to check
+ // whether the thread is idle on the very first pass of the loop.
+ bool first_pass = true;
while (true) {
int32_t x = futex_.load(std::memory_order_relaxed);
- if (x != 0) {
+ while (x != 0) {
if (!futex_.compare_exchange_weak(x, x - 1,
std::memory_order_acquire,
std::memory_order_relaxed)) {
@@ -140,6 +90,8 @@
return true; // Consumed a wakeup, we are done.
}
+
+ if (!first_pass) MaybeBecomeIdle();
const int err = Futex::WaitUntil(&futex_, 0, t);
if (err != 0) {
if (err == -EINTR || err == -EWOULDBLOCK) {
@@ -150,14 +102,13 @@
ABSL_RAW_LOG(FATAL, "Futex operation failed with error %d\n", err);
}
}
-
- MaybeBecomeIdle();
+ first_pass = false;
}
}
void Waiter::Post() {
if (futex_.fetch_add(1, std::memory_order_release) == 0) {
- // We incremented from 0, need to wake a potential waker.
+ // We incremented from 0, need to wake a potential waiter.
Poke();
}
}
@@ -195,7 +146,7 @@
pthread_mutex_t *mu_;
};
-void Waiter::Init() {
+Waiter::Waiter() {
const int err = pthread_mutex_init(&mu_, 0);
if (err != 0) {
ABSL_RAW_LOG(FATAL, "pthread_mutex_init failed: %d", err);
@@ -206,8 +157,20 @@
ABSL_RAW_LOG(FATAL, "pthread_cond_init failed: %d", err2);
}
- waiter_count_.store(0, std::memory_order_relaxed);
- wakeup_count_.store(0, std::memory_order_relaxed);
+ waiter_count_ = 0;
+ wakeup_count_ = 0;
+}
+
+Waiter::~Waiter() {
+ const int err = pthread_mutex_destroy(&mu_);
+ if (err != 0) {
+ ABSL_RAW_LOG(FATAL, "pthread_mutex_destroy failed: %d", err);
+ }
+
+ const int err2 = pthread_cond_destroy(&cv_);
+ if (err2 != 0) {
+ ABSL_RAW_LOG(FATAL, "pthread_cond_destroy failed: %d", err2);
+ }
}
bool Waiter::Wait(KernelTimeout t) {
@@ -217,21 +180,13 @@
}
PthreadMutexHolder h(&mu_);
- waiter_count_.fetch_add(1, std::memory_order_relaxed);
+ ++waiter_count_;
// Loop until we find a wakeup to consume or timeout.
- while (true) {
- int x = wakeup_count_.load(std::memory_order_relaxed);
- if (x != 0) {
- if (!wakeup_count_.compare_exchange_weak(x, x - 1,
- std::memory_order_acquire,
- std::memory_order_relaxed)) {
- continue; // Raced with someone, retry.
- }
- // Successfully consumed a wakeup, we're done.
- waiter_count_.fetch_sub(1, std::memory_order_relaxed);
- return true;
- }
-
+ // Note that, since the thread ticker is just reset, we don't need to check
+ // whether the thread is idle on the very first pass of the loop.
+ bool first_pass = true;
+ while (wakeup_count_ == 0) {
+ if (!first_pass) MaybeBecomeIdle();
// No wakeups available, time to wait.
if (!t.has_timeout()) {
const int err = pthread_cond_wait(&cv_, &mu_);
@@ -241,46 +196,56 @@
} else {
const int err = pthread_cond_timedwait(&cv_, &mu_, &abs_timeout);
if (err == ETIMEDOUT) {
- waiter_count_.fetch_sub(1, std::memory_order_relaxed);
+ --waiter_count_;
return false;
}
if (err != 0) {
- ABSL_RAW_LOG(FATAL, "pthread_cond_wait failed: %d", err);
+ ABSL_RAW_LOG(FATAL, "pthread_cond_timedwait failed: %d", err);
}
}
- MaybeBecomeIdle();
+ first_pass = false;
}
+ // Consume a wakeup and we're done.
+ --wakeup_count_;
+ --waiter_count_;
+ return true;
}
void Waiter::Post() {
- wakeup_count_.fetch_add(1, std::memory_order_release);
- Poke();
+ PthreadMutexHolder h(&mu_);
+ ++wakeup_count_;
+ InternalCondVarPoke();
}
void Waiter::Poke() {
- if (waiter_count_.load(std::memory_order_relaxed) == 0) {
- return;
- }
- // Potentially a waker. Take the lock and check again.
PthreadMutexHolder h(&mu_);
- if (waiter_count_.load(std::memory_order_relaxed) == 0) {
- return;
- }
- const int err = pthread_cond_signal(&cv_);
- if (err != 0) {
- ABSL_RAW_LOG(FATAL, "pthread_cond_signal failed: %d", err);
+ InternalCondVarPoke();
+}
+
+void Waiter::InternalCondVarPoke() {
+ if (waiter_count_ != 0) {
+ const int err = pthread_cond_signal(&cv_);
+ if (ABSL_PREDICT_FALSE(err != 0)) {
+ ABSL_RAW_LOG(FATAL, "pthread_cond_signal failed: %d", err);
+ }
}
}
#elif ABSL_WAITER_MODE == ABSL_WAITER_MODE_SEM
-void Waiter::Init() {
+Waiter::Waiter() {
if (sem_init(&sem_, 0, 0) != 0) {
ABSL_RAW_LOG(FATAL, "sem_init failed with errno %d\n", errno);
}
wakeups_.store(0, std::memory_order_relaxed);
}
+Waiter::~Waiter() {
+ if (sem_destroy(&sem_) != 0) {
+ ABSL_RAW_LOG(FATAL, "sem_destroy failed with errno %d\n", errno);
+ }
+}
+
bool Waiter::Wait(KernelTimeout t) {
struct timespec abs_timeout;
if (t.has_timeout()) {
@@ -288,9 +253,12 @@
}
// Loop until we timeout or consume a wakeup.
+ // Note that, since the thread ticker is just reset, we don't need to check
+ // whether the thread is idle on the very first pass of the loop.
+ bool first_pass = true;
while (true) {
int x = wakeups_.load(std::memory_order_relaxed);
- if (x != 0) {
+ while (x != 0) {
if (!wakeups_.compare_exchange_weak(x, x - 1,
std::memory_order_acquire,
std::memory_order_relaxed)) {
@@ -300,6 +268,7 @@
return true;
}
+ if (!first_pass) MaybeBecomeIdle();
// Nothing to consume, wait (looping on EINTR).
while (true) {
if (!t.has_timeout()) {
@@ -313,13 +282,16 @@
ABSL_RAW_LOG(FATAL, "sem_timedwait failed: %d", errno);
}
}
- MaybeBecomeIdle();
+ first_pass = false;
}
}
void Waiter::Post() {
- wakeups_.fetch_add(1, std::memory_order_release); // Post a wakeup.
- Poke();
+ // Post a wakeup.
+ if (wakeups_.fetch_add(1, std::memory_order_release) == 0) {
+ // We incremented from 0, need to wake a potential waiter.
+ Poke();
+ }
}
void Waiter::Poke() {
@@ -340,31 +312,29 @@
return reinterpret_cast<CONDITION_VARIABLE *>(&w->cv_storage_);
}
- static_assert(sizeof(SRWLOCK) == sizeof(Waiter::SRWLockStorage),
- "SRWLockStorage does not have the same size as SRWLOCK");
+ static_assert(sizeof(SRWLOCK) == sizeof(void *),
+ "`mu_storage_` does not have the same size as SRWLOCK");
+ static_assert(alignof(SRWLOCK) == alignof(void *),
+ "`mu_storage_` does not have the same alignment as SRWLOCK");
+
+ static_assert(sizeof(CONDITION_VARIABLE) == sizeof(void *),
+ "`ABSL_CONDITION_VARIABLE_STORAGE` does not have the same size "
+ "as `CONDITION_VARIABLE`");
static_assert(
- alignof(SRWLOCK) == alignof(Waiter::SRWLockStorage),
- "SRWLockStorage does not have the same alignment as SRWLOCK");
+ alignof(CONDITION_VARIABLE) == alignof(void *),
+ "`cv_storage_` does not have the same alignment as `CONDITION_VARIABLE`");
- static_assert(sizeof(CONDITION_VARIABLE) ==
- sizeof(Waiter::ConditionVariableStorage),
- "ABSL_CONDITION_VARIABLE_STORAGE does not have the same size "
- "as CONDITION_VARIABLE");
- static_assert(alignof(CONDITION_VARIABLE) ==
- alignof(Waiter::ConditionVariableStorage),
- "ConditionVariableStorage does not have the same "
- "alignment as CONDITION_VARIABLE");
-
- // The SRWLOCK and CONDITION_VARIABLE types must be trivially constuctible
+ // The SRWLOCK and CONDITION_VARIABLE types must be trivially constructible
// and destructible because we never call their constructors or destructors.
static_assert(std::is_trivially_constructible<SRWLOCK>::value,
- "The SRWLOCK type must be trivially constructible");
- static_assert(std::is_trivially_constructible<CONDITION_VARIABLE>::value,
- "The CONDITION_VARIABLE type must be trivially constructible");
+ "The `SRWLOCK` type must be trivially constructible");
+ static_assert(
+ std::is_trivially_constructible<CONDITION_VARIABLE>::value,
+ "The `CONDITION_VARIABLE` type must be trivially constructible");
static_assert(std::is_trivially_destructible<SRWLOCK>::value,
- "The SRWLOCK type must be trivially destructible");
+ "The `SRWLOCK` type must be trivially destructible");
static_assert(std::is_trivially_destructible<CONDITION_VARIABLE>::value,
- "The CONDITION_VARIABLE type must be trivially destructible");
+ "The `CONDITION_VARIABLE` type must be trivially destructible");
};
class LockHolder {
@@ -384,36 +354,33 @@
SRWLOCK* mu_;
};
-void Waiter::Init() {
+Waiter::Waiter() {
auto *mu = ::new (static_cast<void *>(&mu_storage_)) SRWLOCK;
auto *cv = ::new (static_cast<void *>(&cv_storage_)) CONDITION_VARIABLE;
InitializeSRWLock(mu);
InitializeConditionVariable(cv);
- waiter_count_.store(0, std::memory_order_relaxed);
- wakeup_count_.store(0, std::memory_order_relaxed);
+ waiter_count_ = 0;
+ wakeup_count_ = 0;
}
+// SRW locks and condition variables do not need to be explicitly destroyed.
+// https://docs.microsoft.com/en-us/windows/win32/api/synchapi/nf-synchapi-initializesrwlock
+// https://stackoverflow.com/questions/28975958/why-does-windows-have-no-deleteconditionvariable-function-to-go-together-with
+Waiter::~Waiter() = default;
+
bool Waiter::Wait(KernelTimeout t) {
SRWLOCK *mu = WinHelper::GetLock(this);
CONDITION_VARIABLE *cv = WinHelper::GetCond(this);
LockHolder h(mu);
- waiter_count_.fetch_add(1, std::memory_order_relaxed);
+ ++waiter_count_;
// Loop until we find a wakeup to consume or timeout.
- while (true) {
- int x = wakeup_count_.load(std::memory_order_relaxed);
- if (x != 0) {
- if (!wakeup_count_.compare_exchange_weak(x, x - 1,
- std::memory_order_acquire,
- std::memory_order_relaxed)) {
- continue; // Raced with someone, retry.
- }
- // Successfully consumed a wakeup, we're done.
- waiter_count_.fetch_sub(1, std::memory_order_relaxed);
- return true;
- }
-
+ // Note that, since the thread ticker is just reset, we don't need to check
+ // whether the thread is idle on the very first pass of the loop.
+ bool first_pass = true;
+ while (wakeup_count_ == 0) {
+ if (!first_pass) MaybeBecomeIdle();
// No wakeups available, time to wait.
if (!SleepConditionVariableSRW(cv, mu, t.InMillisecondsFromNow(), 0)) {
// GetLastError() returns a Win32 DWORD, but we assign to
@@ -421,32 +388,35 @@
// initialization guarantees this is not a narrowing conversion.
const unsigned long err{GetLastError()}; // NOLINT(runtime/int)
if (err == ERROR_TIMEOUT) {
- waiter_count_.fetch_sub(1, std::memory_order_relaxed);
+ --waiter_count_;
return false;
} else {
ABSL_RAW_LOG(FATAL, "SleepConditionVariableSRW failed: %lu", err);
}
}
-
- MaybeBecomeIdle();
+ first_pass = false;
}
+ // Consume a wakeup and we're done.
+ --wakeup_count_;
+ --waiter_count_;
+ return true;
}
void Waiter::Post() {
- wakeup_count_.fetch_add(1, std::memory_order_release);
- Poke();
+ LockHolder h(WinHelper::GetLock(this));
+ ++wakeup_count_;
+ InternalCondVarPoke();
}
void Waiter::Poke() {
- if (waiter_count_.load(std::memory_order_relaxed) == 0) {
- return;
- }
- // Potentially a waker. Take the lock and check again.
LockHolder h(WinHelper::GetLock(this));
- if (waiter_count_.load(std::memory_order_relaxed) == 0) {
- return;
+ InternalCondVarPoke();
+}
+
+void Waiter::InternalCondVarPoke() {
+ if (waiter_count_ != 0) {
+ WakeConditionVariable(WinHelper::GetCond(this));
}
- WakeConditionVariable(WinHelper::GetCond(this));
}
#else
@@ -454,4 +424,5 @@
#endif
} // namespace synchronization_internal
+ABSL_NAMESPACE_END
} // namespace absl
diff --git a/absl/synchronization/internal/waiter.h b/absl/synchronization/internal/waiter.h
index a3e3124..be3df18 100644
--- a/absl/synchronization/internal/waiter.h
+++ b/absl/synchronization/internal/waiter.h
@@ -18,10 +18,16 @@
#include "absl/base/config.h"
-#ifndef _WIN32
+#ifdef _WIN32
+#include <sdkddkver.h>
+#else
#include <pthread.h>
#endif
+#ifdef __linux__
+#include <linux/futex.h>
+#endif
+
#ifdef ABSL_HAVE_SEMAPHORE_H
#include <semaphore.h>
#endif
@@ -30,6 +36,7 @@
#include <cstdint>
#include "absl/base/internal/thread_identity.h"
+#include "absl/synchronization/internal/futex.h"
#include "absl/synchronization/internal/kernel_timeout.h"
// May be chosen at compile time via -DABSL_FORCE_WAITER_MODE=<index>
@@ -40,9 +47,9 @@
#if defined(ABSL_FORCE_WAITER_MODE)
#define ABSL_WAITER_MODE ABSL_FORCE_WAITER_MODE
-#elif defined(_WIN32)
+#elif defined(_WIN32) && _WIN32_WINNT >= _WIN32_WINNT_VISTA
#define ABSL_WAITER_MODE ABSL_WAITER_MODE_WIN32
-#elif defined(__linux__)
+#elif defined(ABSL_INTERNAL_HAVE_FUTEX)
#define ABSL_WAITER_MODE ABSL_WAITER_MODE_FUTEX
#elif defined(ABSL_HAVE_SEMAPHORE_H)
#define ABSL_WAITER_MODE ABSL_WAITER_MODE_SEM
@@ -51,19 +58,21 @@
#endif
namespace absl {
+ABSL_NAMESPACE_BEGIN
namespace synchronization_internal {
// Waiter is an OS-specific semaphore.
class Waiter {
public:
- // No constructor, instances use the reserved space in ThreadIdentity.
- // All initialization logic belongs in `Init()`.
- Waiter() = delete;
+ // Prepare any data to track waits.
+ Waiter();
+
+ // Not copyable or movable
Waiter(const Waiter&) = delete;
Waiter& operator=(const Waiter&) = delete;
- // Prepare any data to track waits.
- void Init();
+ // Destroy any data to track waits.
+ ~Waiter();
// Blocks the calling thread until a matching call to `Post()` or
// `t` has passed. Returns `true` if woken (`Post()` called),
@@ -87,8 +96,8 @@
}
// How many periods to remain idle before releasing resources
-#ifndef THREAD_SANITIZER
- static const int kIdlePeriods = 60;
+#ifndef ABSL_HAVE_THREAD_SANITIZER
+ static constexpr int kIdlePeriods = 60;
#else
// Memory consumption under ThreadSanitizer is a serious concern,
// so we release resources sooner. The value of 1 leads to 1 to 2 second
@@ -104,10 +113,13 @@
static_assert(sizeof(int32_t) == sizeof(futex_), "Wrong size for futex");
#elif ABSL_WAITER_MODE == ABSL_WAITER_MODE_CONDVAR
+ // REQUIRES: mu_ must be held.
+ void InternalCondVarPoke();
+
pthread_mutex_t mu_;
pthread_cond_t cv_;
- std::atomic<int> waiter_count_;
- std::atomic<int> wakeup_count_; // Unclaimed wakeups, written under lock.
+ int waiter_count_;
+ int wakeup_count_; // Unclaimed wakeups.
#elif ABSL_WAITER_MODE == ABSL_WAITER_MODE_SEM
sem_t sem_;
@@ -117,26 +129,19 @@
std::atomic<int> wakeups_;
#elif ABSL_WAITER_MODE == ABSL_WAITER_MODE_WIN32
- // The Windows API has lots of choices for synchronization
- // primivitives. We are using SRWLOCK and CONDITION_VARIABLE
- // because they don't require a destructor to release system
- // resources.
- //
- // However, we can't include Windows.h in our headers, so we use aligned
- // storage buffers to define the storage.
- using SRWLockStorage =
- typename std::aligned_storage<sizeof(void*), alignof(void*)>::type;
- using ConditionVariableStorage =
- typename std::aligned_storage<sizeof(void*), alignof(void*)>::type;
-
// WinHelper - Used to define utilities for accessing the lock and
// condition variable storage once the types are complete.
class WinHelper;
- SRWLockStorage mu_storage_;
- ConditionVariableStorage cv_storage_;
- std::atomic<int> waiter_count_;
- std::atomic<int> wakeup_count_;
+ // REQUIRES: WinHelper::GetLock(this) must be held.
+ void InternalCondVarPoke();
+
+ // We can't include Windows.h in our headers, so we use aligned charachter
+ // buffers to define the storage of SRWLOCK and CONDITION_VARIABLE.
+ alignas(void*) unsigned char mu_storage_[sizeof(void*)];
+ alignas(void*) unsigned char cv_storage_[sizeof(void*)];
+ int waiter_count_;
+ int wakeup_count_;
#else
#error Unknown ABSL_WAITER_MODE
@@ -144,6 +149,7 @@
};
} // namespace synchronization_internal
+ABSL_NAMESPACE_END
} // namespace absl
#endif // ABSL_SYNCHRONIZATION_INTERNAL_WAITER_H_
diff --git a/absl/synchronization/lifetime_test.cc b/absl/synchronization/lifetime_test.cc
index 34b8875..cc973a3 100644
--- a/absl/synchronization/lifetime_test.cc
+++ b/absl/synchronization/lifetime_test.cc
@@ -122,6 +122,11 @@
Function fn_;
};
+// These tests require that the compiler correctly supports C++11 constant
+// initialization... but MSVC has a known regression since v19.10:
+// https://developercommunity.visualstudio.com/content/problem/336946/class-with-constexpr-constructor-not-using-static.html
+// TODO(epastor): Limit the affected range once MSVC fixes this bug.
+#if defined(__clang__) || !(defined(_MSC_VER) && _MSC_VER > 1900)
// kConstInit
// Test early usage. (Declaration comes first; definitions must appear after
// the test runner.)
@@ -151,6 +156,7 @@
const_init_sanity_mutex.AssertHeld();
const_init_sanity_mutex.Unlock();
});
+#endif // defined(__clang__) || !(defined(_MSC_VER) && _MSC_VER > 1900)
// Test shutdown usage. (Declarations come first; definitions must appear after
// the test runner.)
diff --git a/absl/synchronization/mutex.cc b/absl/synchronization/mutex.cc
index 100def2..9e01393 100644
--- a/absl/synchronization/mutex.cc
+++ b/absl/synchronization/mutex.cc
@@ -39,6 +39,7 @@
#include <thread> // NOLINT(build/c++11)
#include "absl/base/attributes.h"
+#include "absl/base/call_once.h"
#include "absl/base/config.h"
#include "absl/base/dynamic_annotations.h"
#include "absl/base/internal/atomic_hook.h"
@@ -49,6 +50,7 @@
#include "absl/base/internal/spinlock.h"
#include "absl/base/internal/sysinfo.h"
#include "absl/base/internal/thread_identity.h"
+#include "absl/base/internal/tsan_mutex_interface.h"
#include "absl/base/port.h"
#include "absl/debugging/stacktrace.h"
#include "absl/debugging/symbolize.h"
@@ -58,6 +60,7 @@
using absl::base_internal::CurrentThreadIdentityIfPresent;
using absl::base_internal::PerThreadSynch;
+using absl::base_internal::SchedulingGuard;
using absl::base_internal::ThreadIdentity;
using absl::synchronization_internal::GetOrCreateCurrentThreadIdentity;
using absl::synchronization_internal::GraphCycles;
@@ -71,10 +74,11 @@
} // extern "C"
namespace absl {
+ABSL_NAMESPACE_BEGIN
namespace {
-#if defined(THREAD_SANITIZER)
+#if defined(ABSL_HAVE_THREAD_SANITIZER)
constexpr OnDeadlockCycle kDeadlockDetectionDefault = OnDeadlockCycle::kIgnore;
#else
constexpr OnDeadlockCycle kDeadlockDetectionDefault = OnDeadlockCycle::kAbort;
@@ -84,35 +88,16 @@
kDeadlockDetectionDefault);
ABSL_CONST_INIT std::atomic<bool> synch_check_invariants(false);
-// ------------------------------------------ spinlock support
-
-// Make sure read-only globals used in the Mutex code are contained on the
-// same cacheline and cacheline aligned to eliminate any false sharing with
-// other globals from this and other modules.
-static struct MutexGlobals {
- MutexGlobals() {
- // Find machine-specific data needed for Delay() and
- // TryAcquireWithSpinning(). This runs in the global constructor
- // sequence, and before that zeros are safe values.
- num_cpus = absl::base_internal::NumCPUs();
- spinloop_iterations = num_cpus > 1 ? 1500 : 0;
- }
- int num_cpus;
- int spinloop_iterations;
- // Pad this struct to a full cacheline to prevent false sharing.
- char padding[ABSL_CACHELINE_SIZE - 2 * sizeof(int)];
-} ABSL_CACHELINE_ALIGNED mutex_globals;
-static_assert(
- sizeof(MutexGlobals) == ABSL_CACHELINE_SIZE,
- "MutexGlobals must occupy an entire cacheline to prevent false sharing");
-
-ABSL_CONST_INIT absl::base_internal::AtomicHook<void (*)(int64_t wait_cycles)>
+ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES
+absl::base_internal::AtomicHook<void (*)(int64_t wait_cycles)>
submit_profile_data;
-ABSL_CONST_INIT absl::base_internal::AtomicHook<
- void (*)(const char *msg, const void *obj, int64_t wait_cycles)> mutex_tracer;
-ABSL_CONST_INIT absl::base_internal::AtomicHook<
- void (*)(const char *msg, const void *cv)> cond_var_tracer;
-ABSL_CONST_INIT absl::base_internal::AtomicHook<
+ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES absl::base_internal::AtomicHook<void (*)(
+ const char *msg, const void *obj, int64_t wait_cycles)>
+ mutex_tracer;
+ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES
+ absl::base_internal::AtomicHook<void (*)(const char *msg, const void *cv)>
+ cond_var_tracer;
+ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES absl::base_internal::AtomicHook<
bool (*)(const void *pc, char *out, int out_size)>
symbolizer(absl::Symbolize);
@@ -139,33 +124,55 @@
symbolizer.Store(fn);
}
-// spinlock delay on iteration c. Returns new c.
+struct ABSL_CACHELINE_ALIGNED MutexGlobals {
+ absl::once_flag once;
+ int num_cpus = 0;
+ int spinloop_iterations = 0;
+};
+
+static const MutexGlobals& GetMutexGlobals() {
+ ABSL_CONST_INIT static MutexGlobals data;
+ absl::base_internal::LowLevelCallOnce(&data.once, [&]() {
+ data.num_cpus = absl::base_internal::NumCPUs();
+ data.spinloop_iterations = data.num_cpus > 1 ? 1500 : 0;
+ });
+ return data;
+}
+
+// Spinlock delay on iteration c. Returns new c.
namespace {
enum DelayMode { AGGRESSIVE, GENTLE };
};
-static int Delay(int32_t c, DelayMode mode) {
+
+namespace synchronization_internal {
+int MutexDelay(int32_t c, int mode) {
// If this a uniprocessor, only yield/sleep. Otherwise, if the mode is
// aggressive then spin many times before yielding. If the mode is
// gentle then spin only a few times before yielding. Aggressive spinning is
// used to ensure that an Unlock() call, which must get the spin lock for
// any thread to make progress gets it without undue delay.
- int32_t limit = (mutex_globals.num_cpus > 1) ?
- ((mode == AGGRESSIVE) ? 5000 : 250) : 0;
+ const int32_t limit =
+ GetMutexGlobals().num_cpus > 1 ? (mode == AGGRESSIVE ? 5000 : 250) : 0;
if (c < limit) {
- c++; // spin
+ // Spin.
+ c++;
} else {
+ SchedulingGuard::ScopedEnable enable_rescheduling;
ABSL_TSAN_MUTEX_PRE_DIVERT(nullptr, 0);
- if (c == limit) { // yield once
+ if (c == limit) {
+ // Yield once.
AbslInternalMutexYield();
c++;
- } else { // then wait
+ } else {
+ // Then wait.
absl::SleepFor(absl::Microseconds(10));
c = 0;
}
ABSL_TSAN_MUTEX_POST_DIVERT(nullptr, 0);
}
- return (c);
+ return c;
}
+} // namespace synchronization_internal
// --------------------------Generic atomic ops
// Ensure that "(*pv & bits) == bits" by doing an atomic update of "*pv" to
@@ -203,12 +210,12 @@
//------------------------------------------------------------------
// Data for doing deadlock detection.
-static absl::base_internal::SpinLock deadlock_graph_mu(
- absl::base_internal::kLinkerInitialized);
+ABSL_CONST_INIT static absl::base_internal::SpinLock deadlock_graph_mu(
+ absl::kConstInit, base_internal::SCHEDULE_KERNEL_ONLY);
-// graph used to detect deadlocks.
-static GraphCycles *deadlock_graph ABSL_GUARDED_BY(deadlock_graph_mu)
- ABSL_PT_GUARDED_BY(deadlock_graph_mu);
+// Graph used to detect deadlocks.
+ABSL_CONST_INIT static GraphCycles *deadlock_graph
+ ABSL_GUARDED_BY(deadlock_graph_mu) ABSL_PT_GUARDED_BY(deadlock_graph_mu);
//------------------------------------------------------------------
// An event mechanism for debugging mutex use.
@@ -269,13 +276,12 @@
{0, "SignalAll on "},
};
-static absl::base_internal::SpinLock synch_event_mu(
- absl::base_internal::kLinkerInitialized);
-// protects synch_event
+ABSL_CONST_INIT static absl::base_internal::SpinLock synch_event_mu(
+ absl::kConstInit, base_internal::SCHEDULE_KERNEL_ONLY);
// Hash table size; should be prime > 2.
// Can't be too small, as it's used for deadlock detection information.
-static const uint32_t kNSynchEvent = 1031;
+static constexpr uint32_t kNSynchEvent = 1031;
static struct SynchEvent { // this is a trivial hash table for the events
// struct is freed when refcount reaches 0
@@ -295,7 +301,7 @@
bool log; // logging turned on
// Constant after initialization
- char name[1]; // actually longer---null-terminated std::string
+ char name[1]; // actually longer---NUL-terminated string
} * synch_event[kNSynchEvent] ABSL_GUARDED_BY(synch_event_mu);
// Ensure that the object at "addr" has a SynchEvent struct associated with it,
@@ -486,7 +492,7 @@
std::atomic<intptr_t> *cv_word;
int64_t contention_start_cycles; // Time (in cycles) when this thread started
- // to contend for the mutex.
+ // to contend for the mutex.
};
struct SynchLocksHeld {
@@ -700,7 +706,7 @@
static constexpr bool kDebugMode = true;
#endif
-#ifdef THREAD_SANITIZER
+#ifdef ABSL_INTERNAL_HAVE_TSAN_INTERFACE
static unsigned TsanFlags(Mutex::MuHow how) {
return how == kShared ? __tsan_mutex_read_lock : 0;
}
@@ -1051,6 +1057,7 @@
// Try to remove thread s from the list of waiters on this mutex.
// Does nothing if s is not on the waiter list.
void Mutex::TryRemove(PerThreadSynch *s) {
+ SchedulingGuard::ScopedDisable disable_rescheduling;
intptr_t v = mu_.load(std::memory_order_relaxed);
// acquire spinlock & lock
if ((v & (kMuWait | kMuSpin | kMuWriter | kMuReader)) == kMuWait &&
@@ -1115,7 +1122,7 @@
this->TryRemove(s);
int c = 0;
while (s->next != nullptr) {
- c = Delay(c, GENTLE);
+ c = synchronization_internal::MutexDelay(c, GENTLE);
this->TryRemove(s);
}
if (kDebugMode) {
@@ -1434,21 +1441,19 @@
// Attempt to acquire *mu, and return whether successful. The implementation
// may spin for a short while if the lock cannot be acquired immediately.
static bool TryAcquireWithSpinning(std::atomic<intptr_t>* mu) {
- int c = mutex_globals.spinloop_iterations;
- int result = -1; // result of operation: 0=false, 1=true, -1=unknown
-
+ int c = GetMutexGlobals().spinloop_iterations;
do { // do/while somewhat faster on AMD
intptr_t v = mu->load(std::memory_order_relaxed);
- if ((v & (kMuReader|kMuEvent)) != 0) { // a reader or tracing -> give up
- result = 0;
+ if ((v & (kMuReader|kMuEvent)) != 0) {
+ return false; // a reader or tracing -> give up
} else if (((v & kMuWriter) == 0) && // no holder -> try to acquire
mu->compare_exchange_strong(v, kMuWriter | v,
std::memory_order_acquire,
std::memory_order_relaxed)) {
- result = 1;
+ return true;
}
- } while (result == -1 && --c > 0);
- return result == 1;
+ } while (--c > 0);
+ return false;
}
ABSL_XRAY_LOG_ARGS(1) void Mutex::Lock() {
@@ -1747,7 +1752,8 @@
};
// Internal version of LockWhen(). See LockSlowWithDeadline()
-void Mutex::LockSlow(MuHow how, const Condition *cond, int flags) {
+ABSL_ATTRIBUTE_NOINLINE void Mutex::LockSlow(MuHow how, const Condition *cond,
+ int flags) {
ABSL_RAW_CHECK(
this->LockSlowWithDeadline(how, cond, KernelTimeout::Never(), flags),
"condition untrue on return from LockSlow");
@@ -1762,7 +1768,7 @@
// All memory accesses are ignored inside of mutex operations + for unlock
// operation tsan considers that we've already released the mutex.
bool res = false;
-#ifdef THREAD_SANITIZER
+#ifdef ABSL_INTERNAL_HAVE_TSAN_INTERFACE
const int flags = read_lock ? __tsan_mutex_read_lock : 0;
const int tryflags = flags | (trylock ? __tsan_mutex_try_lock : 0);
#endif
@@ -1812,9 +1818,9 @@
// So we "divert" (which un-ignores both memory accesses and synchronization)
// and then separately turn on ignores of memory accesses.
ABSL_TSAN_MUTEX_PRE_DIVERT(mu, 0);
- ANNOTATE_IGNORE_READS_AND_WRITES_BEGIN();
+ ABSL_ANNOTATE_IGNORE_READS_AND_WRITES_BEGIN();
bool res = cond->Eval();
- ANNOTATE_IGNORE_READS_AND_WRITES_END();
+ ABSL_ANNOTATE_IGNORE_READS_AND_WRITES_END();
ABSL_TSAN_MUTEX_POST_DIVERT(mu, 0);
static_cast<void>(mu); // Prevent unused param warning in non-TSAN builds.
return res;
@@ -1895,6 +1901,7 @@
}
void Mutex::LockSlowLoop(SynchWaitParams *waitp, int flags) {
+ SchedulingGuard::ScopedDisable disable_rescheduling;
int c = 0;
intptr_t v = mu_.load(std::memory_order_relaxed);
if ((v & kMuEvent) != 0) {
@@ -1996,7 +2003,8 @@
ABSL_RAW_CHECK(
waitp->thread->waitp == nullptr || waitp->thread->suppress_fatal_errors,
"detected illegal recursion into Mutex code");
- c = Delay(c, GENTLE); // delay, then try again
+ // delay, then try again
+ c = synchronization_internal::MutexDelay(c, GENTLE);
}
ABSL_RAW_CHECK(
waitp->thread->waitp == nullptr || waitp->thread->suppress_fatal_errors,
@@ -2013,7 +2021,8 @@
// which holds the lock but is not runnable because its condition is false
// or it is in the process of blocking on a condition variable; it must requeue
// itself on the mutex/condvar to wait for its condition to become true.
-void Mutex::UnlockSlow(SynchWaitParams *waitp) {
+ABSL_ATTRIBUTE_NOINLINE void Mutex::UnlockSlow(SynchWaitParams *waitp) {
+ SchedulingGuard::ScopedDisable disable_rescheduling;
intptr_t v = mu_.load(std::memory_order_relaxed);
this->AssertReaderHeld();
CheckForMutexCorruption(v, "Unlock");
@@ -2290,7 +2299,8 @@
mu_.store(nv, std::memory_order_release);
break; // out of for(;;)-loop
}
- c = Delay(c, AGGRESSIVE); // aggressive here; no one can proceed till we do
+ // aggressive here; no one can proceed till we do
+ c = synchronization_internal::MutexDelay(c, AGGRESSIVE);
} // end of for(;;)-loop
if (wake_list != kPerThreadSynchNull) {
@@ -2302,7 +2312,8 @@
if (!cond_waiter) {
// Sample lock contention events only if the (first) waiter was trying to
// acquire the lock, not waiting on a condition variable or Condition.
- int64_t wait_cycles = base_internal::CycleClock::Now() - enqueue_timestamp;
+ int64_t wait_cycles =
+ base_internal::CycleClock::Now() - enqueue_timestamp;
mutex_tracer("slow release", this, wait_cycles);
ABSL_TSAN_MUTEX_PRE_DIVERT(this, 0);
submit_profile_data(enqueue_timestamp);
@@ -2329,6 +2340,7 @@
// It will later acquire the mutex with high probability. Otherwise, we
// enqueue thread w on this mutex.
void Mutex::Fer(PerThreadSynch *w) {
+ SchedulingGuard::ScopedDisable disable_rescheduling;
int c = 0;
ABSL_RAW_CHECK(w->waitp->cond == nullptr,
"Mutex::Fer while waiting on Condition");
@@ -2378,7 +2390,7 @@
return;
}
}
- c = Delay(c, GENTLE);
+ c = synchronization_internal::MutexDelay(c, GENTLE);
}
}
@@ -2427,6 +2439,7 @@
// Remove thread s from the list of waiters on this condition variable.
void CondVar::Remove(PerThreadSynch *s) {
+ SchedulingGuard::ScopedDisable disable_rescheduling;
intptr_t v;
int c = 0;
for (v = cv_.load(std::memory_order_relaxed);;
@@ -2455,7 +2468,8 @@
std::memory_order_release);
return;
} else {
- c = Delay(c, GENTLE); // try again after a delay
+ // try again after a delay
+ c = synchronization_internal::MutexDelay(c, GENTLE);
}
}
}
@@ -2488,7 +2502,7 @@
!cv_word->compare_exchange_weak(v, v | kCvSpin,
std::memory_order_acquire,
std::memory_order_relaxed)) {
- c = Delay(c, GENTLE);
+ c = synchronization_internal::MutexDelay(c, GENTLE);
v = cv_word->load(std::memory_order_relaxed);
}
ABSL_RAW_CHECK(waitp->thread->waitp == nullptr, "waiting when shouldn't be");
@@ -2587,6 +2601,7 @@
}
void CondVar::Signal() {
+ SchedulingGuard::ScopedDisable disable_rescheduling;
ABSL_TSAN_MUTEX_PRE_SIGNAL(nullptr, 0);
intptr_t v;
int c = 0;
@@ -2619,7 +2634,7 @@
ABSL_TSAN_MUTEX_POST_SIGNAL(nullptr, 0);
return;
} else {
- c = Delay(c, GENTLE);
+ c = synchronization_internal::MutexDelay(c, GENTLE);
}
}
ABSL_TSAN_MUTEX_POST_SIGNAL(nullptr, 0);
@@ -2656,7 +2671,8 @@
ABSL_TSAN_MUTEX_POST_SIGNAL(nullptr, 0);
return;
} else {
- c = Delay(c, GENTLE); // try again after a delay
+ // try again after a delay
+ c = synchronization_internal::MutexDelay(c, GENTLE);
}
}
ABSL_TSAN_MUTEX_POST_SIGNAL(nullptr, 0);
@@ -2669,7 +2685,7 @@
this->mu_ = nullptr;
}
-#ifdef THREAD_SANITIZER
+#ifdef ABSL_HAVE_THREAD_SANITIZER
extern "C" void __tsan_read1(void *addr);
#else
#define __tsan_read1(addr) // do nothing if TSan not enabled
@@ -2720,4 +2736,5 @@
a->arg_ == b->arg_ && a->method_ == b->method_;
}
+ABSL_NAMESPACE_END
} // namespace absl
diff --git a/absl/synchronization/mutex.h b/absl/synchronization/mutex.h
index d33318d..598d1e0 100644
--- a/absl/synchronization/mutex.h
+++ b/absl/synchronization/mutex.h
@@ -31,22 +31,23 @@
//
// MutexLock - An RAII wrapper to acquire and release a `Mutex` for exclusive/
// write access within the current scope.
+//
// ReaderMutexLock
// - An RAII wrapper to acquire and release a `Mutex` for shared/read
// access within the current scope.
//
// WriterMutexLock
-// - Alias for `MutexLock` above, designed for use in distinguishing
-// reader and writer locks within code.
+// - Effectively an alias for `MutexLock` above, designed for use in
+// distinguishing reader and writer locks within code.
//
// In addition to simple mutex locks, this file also defines ways to perform
// locking under certain conditions.
//
-// Condition - (Preferred) Used to wait for a particular predicate that
-// depends on state protected by the `Mutex` to become true.
-// CondVar - A lower-level variant of `Condition` that relies on
-// application code to explicitly signal the `CondVar` when
-// a condition has been met.
+// Condition - (Preferred) Used to wait for a particular predicate that
+// depends on state protected by the `Mutex` to become true.
+// CondVar - A lower-level variant of `Condition` that relies on
+// application code to explicitly signal the `CondVar` when
+// a condition has been met.
//
// See below for more information on using `Condition` or `CondVar`.
//
@@ -72,16 +73,8 @@
#include "absl/synchronization/internal/per_thread_sem.h"
#include "absl/time/time.h"
-// Decide if we should use the non-production implementation because
-// the production implementation hasn't been fully ported yet.
-#ifdef ABSL_INTERNAL_USE_NONPROD_MUTEX
-#error ABSL_INTERNAL_USE_NONPROD_MUTEX cannot be directly set
-#elif defined(ABSL_LOW_LEVEL_ALLOC_MISSING)
-#define ABSL_INTERNAL_USE_NONPROD_MUTEX 1
-#include "absl/synchronization/internal/mutex_nonprod.inc"
-#endif
-
namespace absl {
+ABSL_NAMESPACE_BEGIN
class Condition;
struct SynchWaitParams;
@@ -330,17 +323,16 @@
// Mutex::AwaitWithTimeout()
// Mutex::AwaitWithDeadline()
//
- // If `cond` is initially true, do nothing, or act as though `cond` is
- // initially false.
- //
- // If `cond` is initially false, unlock this `Mutex` and block until
- // simultaneously:
+ // Unlocks this `Mutex` and blocks until simultaneously:
// - either `cond` is true or the {timeout has expired, deadline has passed}
// and
// - this `Mutex` can be reacquired,
// then reacquire this `Mutex` in the same mode in which it was previously
// held, returning `true` iff `cond` is `true` on return.
//
+ // If the condition is initially `true`, the implementation *may* skip the
+ // release/re-acquire step and return immediately.
+ //
// Deadlines in the past are equivalent to an immediate deadline.
// Negative timeouts are equivalent to a zero timeout.
//
@@ -461,15 +453,6 @@
static void InternalAttemptToUseMutexInFatalSignalHandler();
private:
-#ifdef ABSL_INTERNAL_USE_NONPROD_MUTEX
- friend class CondVar;
-
- synchronization_internal::MutexImpl *impl() { return impl_.get(); }
-
- synchronization_internal::SynchronizationStorage<
- synchronization_internal::MutexImpl>
- impl_;
-#else
std::atomic<intptr_t> mu_; // The Mutex state.
// Post()/Wait() versus associated PerThreadSem; in class for required
@@ -504,7 +487,6 @@
void Trans(MuHow how); // used for CondVar->Mutex transfer
void Fer(
base_internal::PerThreadSynch *w); // used for CondVar->Mutex transfer
-#endif
// Catch the error of writing Mutex when intending MutexLock.
Mutex(const volatile Mutex * /*ignored*/) {} // NOLINT(runtime/explicit)
@@ -525,22 +507,36 @@
// Example:
//
// Class Foo {
-//
+// public:
// Foo::Bar* Baz() {
-// MutexLock l(&lock_);
+// MutexLock lock(&mu_);
// ...
// return bar;
// }
//
// private:
-// Mutex lock_;
+// Mutex mu_;
// };
class ABSL_SCOPED_LOCKABLE MutexLock {
public:
+ // Constructors
+
+ // Calls `mu->Lock()` and returns when that call returns. That is, `*mu` is
+ // guaranteed to be locked when this object is constructed. Requires that
+ // `mu` be dereferenceable.
explicit MutexLock(Mutex *mu) ABSL_EXCLUSIVE_LOCK_FUNCTION(mu) : mu_(mu) {
this->mu_->Lock();
}
+ // Like above, but calls `mu->LockWhen(cond)` instead. That is, in addition to
+ // the above, the condition given by `cond` is also guaranteed to hold when
+ // this object is constructed.
+ explicit MutexLock(Mutex *mu, const Condition &cond)
+ ABSL_EXCLUSIVE_LOCK_FUNCTION(mu)
+ : mu_(mu) {
+ this->mu_->LockWhen(cond);
+ }
+
MutexLock(const MutexLock &) = delete; // NOLINT(runtime/mutex)
MutexLock(MutexLock&&) = delete; // NOLINT(runtime/mutex)
MutexLock& operator=(const MutexLock&) = delete;
@@ -562,6 +558,12 @@
mu->ReaderLock();
}
+ explicit ReaderMutexLock(Mutex *mu, const Condition &cond)
+ ABSL_SHARED_LOCK_FUNCTION(mu)
+ : mu_(mu) {
+ mu->ReaderLockWhen(cond);
+ }
+
ReaderMutexLock(const ReaderMutexLock&) = delete;
ReaderMutexLock(ReaderMutexLock&&) = delete;
ReaderMutexLock& operator=(const ReaderMutexLock&) = delete;
@@ -584,6 +586,12 @@
mu->WriterLock();
}
+ explicit WriterMutexLock(Mutex *mu, const Condition &cond)
+ ABSL_EXCLUSIVE_LOCK_FUNCTION(mu)
+ : mu_(mu) {
+ mu->WriterLockWhen(cond);
+ }
+
WriterMutexLock(const WriterMutexLock&) = delete;
WriterMutexLock(WriterMutexLock&&) = delete;
WriterMutexLock& operator=(const WriterMutexLock&) = delete;
@@ -622,16 +630,26 @@
// `noexcept`; until then this requirement cannot be enforced in the
// type system.)
//
-// Note: to use a `Condition`, you need only construct it and pass it within the
-// appropriate `Mutex' member function, such as `Mutex::Await()`.
+// Note: to use a `Condition`, you need only construct it and pass it to a
+// suitable `Mutex' member function, such as `Mutex::Await()`, or to the
+// constructor of one of the scope guard classes.
//
-// Example:
+// Example using LockWhen/Unlock:
//
// // assume count_ is not internal reference count
-// int count_ GUARDED_BY(mu_);
+// int count_ ABSL_GUARDED_BY(mu_);
+// Condition count_is_zero(+[](int *count) { return *count == 0; }, &count_);
//
-// mu_.LockWhen(Condition(+[](int* count) { return *count == 0; },
-// &count_));
+// mu_.LockWhen(count_is_zero);
+// // ...
+// mu_.Unlock();
+//
+// Example using a scope guard:
+//
+// {
+// MutexLock lock(&mu_, count_is_zero);
+// // ...
+// }
//
// When multiple threads are waiting on exactly the same condition, make sure
// that they are constructed with the same parameters (same pointer to function
@@ -685,6 +703,11 @@
// return processed_ >= current;
// };
// mu_.Await(Condition(&reached));
+ //
+ // NOTE: never use "mu_.AssertHeld()" instead of "mu_.AssertReaderHeld()" in
+ // the lambda as it may be called when the mutex is being unlocked from a
+ // scope holding only a reader lock, which will make the assertion not
+ // fulfilled and crash the binary.
// See class comment for performance advice. In particular, if there
// might be more than one waiter for the same condition, make sure
@@ -769,6 +792,8 @@
//
class CondVar {
public:
+ // A `CondVar` allocated on the heap or on the stack can use the this
+ // constructor.
CondVar();
~CondVar();
@@ -831,17 +856,10 @@
void EnableDebugLog(const char *name);
private:
-#ifdef ABSL_INTERNAL_USE_NONPROD_MUTEX
- synchronization_internal::CondVarImpl *impl() { return impl_.get(); }
- synchronization_internal::SynchronizationStorage<
- synchronization_internal::CondVarImpl>
- impl_;
-#else
bool WaitCommon(Mutex *mutex, synchronization_internal::KernelTimeout t);
void Remove(base_internal::PerThreadSynch *s);
void Wakeup(base_internal::PerThreadSynch *w);
std::atomic<intptr_t> cv_; // Condition variable state.
-#endif
CondVar(const CondVar&) = delete;
CondVar& operator=(const CondVar&) = delete;
};
@@ -863,6 +881,15 @@
this->mu_->Lock();
}
}
+
+ explicit MutexLockMaybe(Mutex *mu, const Condition &cond)
+ ABSL_EXCLUSIVE_LOCK_FUNCTION(mu)
+ : mu_(mu) {
+ if (this->mu_ != nullptr) {
+ this->mu_->LockWhen(cond);
+ }
+ }
+
~MutexLockMaybe() ABSL_UNLOCK_FUNCTION() {
if (this->mu_ != nullptr) { this->mu_->Unlock(); }
}
@@ -885,6 +912,13 @@
: mu_(mu) {
this->mu_->Lock();
}
+
+ explicit ReleasableMutexLock(Mutex *mu, const Condition &cond)
+ ABSL_EXCLUSIVE_LOCK_FUNCTION(mu)
+ : mu_(mu) {
+ this->mu_->LockWhen(cond);
+ }
+
~ReleasableMutexLock() ABSL_UNLOCK_FUNCTION() {
if (this->mu_ != nullptr) { this->mu_->Unlock(); }
}
@@ -899,10 +933,6 @@
ReleasableMutexLock& operator=(ReleasableMutexLock&&) = delete;
};
-#ifdef ABSL_INTERNAL_USE_NONPROD_MUTEX
-inline constexpr Mutex::Mutex(absl::ConstInitType) : impl_(absl::kConstInit) {}
-
-#else
inline Mutex::Mutex() : mu_(0) {
ABSL_TSAN_MUTEX_CREATE(this, __tsan_mutex_not_static);
}
@@ -910,7 +940,6 @@
inline constexpr Mutex::Mutex(absl::ConstInitType) : mu_(0) {}
inline CondVar::CondVar() : cv_(0) {}
-#endif
// static
template <typename T>
@@ -978,7 +1007,7 @@
//
// This has the same memory ordering concerns as RegisterMutexProfiler() above.
void RegisterMutexTracer(void (*fn)(const char *msg, const void *obj,
- int64_t wait_cycles));
+ int64_t wait_cycles));
// TODO(gfalcon): Combine RegisterMutexProfiler() and RegisterMutexTracer()
// into a single interface, since they are only ever called in pairs.
@@ -1000,7 +1029,7 @@
//
// 'pc' is the program counter being symbolized, 'out' is the buffer to write
// into, and 'out_size' is the size of the buffer. This function can return
-// false if symbolizing failed, or true if a null-terminated symbol was written
+// false if symbolizing failed, or true if a NUL-terminated symbol was written
// to 'out.'
//
// This has the same memory ordering concerns as RegisterMutexProfiler() above.
@@ -1039,6 +1068,7 @@
// the manner chosen here.
void SetMutexDeadlockDetectionMode(OnDeadlockCycle mode);
+ABSL_NAMESPACE_END
} // namespace absl
// In some build configurations we pass --detect-odr-violations to the
diff --git a/absl/synchronization/mutex_benchmark.cc b/absl/synchronization/mutex_benchmark.cc
index ab18800..933ea14 100644
--- a/absl/synchronization/mutex_benchmark.cc
+++ b/absl/synchronization/mutex_benchmark.cc
@@ -16,6 +16,7 @@
#include <mutex> // NOLINT(build/c++11)
#include <vector>
+#include "absl/base/config.h"
#include "absl/base/internal/cycleclock.h"
#include "absl/base/internal/spinlock.h"
#include "absl/synchronization/blocking_counter.h"
@@ -213,7 +214,7 @@
}
// Some configurations have higher thread limits than others.
-#if defined(__linux__) && !defined(THREAD_SANITIZER)
+#if defined(__linux__) && !defined(ABSL_HAVE_THREAD_SANITIZER)
constexpr int kMaxConditionWaiters = 8192;
#else
constexpr int kMaxConditionWaiters = 1024;
diff --git a/absl/synchronization/mutex_test.cc b/absl/synchronization/mutex_test.cc
index afb363a..058f757 100644
--- a/absl/synchronization/mutex_test.cc
+++ b/absl/synchronization/mutex_test.cc
@@ -30,6 +30,7 @@
#include "gtest/gtest.h"
#include "absl/base/attributes.h"
+#include "absl/base/config.h"
#include "absl/base/internal/raw_logging.h"
#include "absl/base/internal/sysinfo.h"
#include "absl/memory/memory.h"
@@ -706,6 +707,40 @@
t.join();
}
+TEST(Mutex, LockWhenGuard) {
+ absl::Mutex mu;
+ int n = 30;
+ bool done = false;
+
+ // We don't inline the lambda because the conversion is ambiguous in MSVC.
+ bool (*cond_eq_10)(int *) = [](int *p) { return *p == 10; };
+ bool (*cond_lt_10)(int *) = [](int *p) { return *p < 10; };
+
+ std::thread t1([&mu, &n, &done, cond_eq_10]() {
+ absl::ReaderMutexLock lock(&mu, absl::Condition(cond_eq_10, &n));
+ done = true;
+ });
+
+ std::thread t2[10];
+ for (std::thread &t : t2) {
+ t = std::thread([&mu, &n, cond_lt_10]() {
+ absl::WriterMutexLock lock(&mu, absl::Condition(cond_lt_10, &n));
+ ++n;
+ });
+ }
+
+ {
+ absl::MutexLock lock(&mu);
+ n = 0;
+ }
+
+ for (std::thread &t : t2) t.join();
+ t1.join();
+
+ EXPECT_TRUE(done);
+ EXPECT_EQ(n, 10);
+}
+
// --------------------------------------------------------
// The following test requires Mutex::ReaderLock to be a real shared
// lock, which is not the case in all builds.
@@ -815,7 +850,7 @@
// Test that we correctly handle the situation when a lock is
// held and then destroyed (w/o unlocking).
-#ifdef THREAD_SANITIZER
+#ifdef ABSL_HAVE_THREAD_SANITIZER
// TSAN reports errors when locked Mutexes are destroyed.
TEST(Mutex, DISABLED_LockedMutexDestructionBug) NO_THREAD_SAFETY_ANALYSIS {
#else
@@ -1001,9 +1036,6 @@
x.mu0.Unlock();
}
-// The deadlock detector is not part of non-prod builds, so do not test it.
-#if !defined(ABSL_INTERNAL_USE_NONPROD_MUTEX)
-
TEST(Mutex, DeadlockDetector) {
absl::SetMutexDeadlockDetectionMode(absl::OnDeadlockCycle::kAbort);
@@ -1067,7 +1099,7 @@
const char ScopedDisableBazelTestWarnings::kVarName[] =
"TEST_WARNINGS_OUTPUT_FILE";
-#ifdef THREAD_SANITIZER
+#ifdef ABSL_HAVE_THREAD_SANITIZER
// This test intentionally creates deadlocks to test the deadlock detector.
TEST(Mutex, DISABLED_DeadlockDetectorBazelWarning) {
#else
@@ -1101,7 +1133,7 @@
// annotation-based static thread-safety analysis is not currently
// predicate-aware and cannot tell if the two for-loops that acquire and
// release the locks have the same predicates.
-TEST(Mutex, DeadlockDetectorStessTest) ABSL_NO_THREAD_SAFETY_ANALYSIS {
+TEST(Mutex, DeadlockDetectorStressTest) ABSL_NO_THREAD_SAFETY_ANALYSIS {
// Stress test: Here we create a large number of locks and use all of them.
// If a deadlock detector keeps a full graph of lock acquisition order,
// it will likely be too slow for this test to pass.
@@ -1119,7 +1151,7 @@
}
}
-#ifdef THREAD_SANITIZER
+#ifdef ABSL_HAVE_THREAD_SANITIZER
// TSAN reports errors when locked Mutexes are destroyed.
TEST(Mutex, DISABLED_DeadlockIdBug) NO_THREAD_SAFETY_ANALYSIS {
#else
@@ -1157,7 +1189,6 @@
c.Lock();
c.Unlock();
}
-#endif // !defined(ABSL_INTERNAL_USE_NONPROD_MUTEX)
// --------------------------------------------------------
// Test for timeouts/deadlines on condition waits that are specified using
diff --git a/absl/synchronization/notification.cc b/absl/synchronization/notification.cc
index 53ace00..e91b903 100644
--- a/absl/synchronization/notification.cc
+++ b/absl/synchronization/notification.cc
@@ -22,6 +22,7 @@
#include "absl/time/time.h"
namespace absl {
+ABSL_NAMESPACE_BEGIN
void Notification::Notify() {
MutexLock l(&this->mutex_);
@@ -44,15 +45,6 @@
MutexLock l(&this->mutex_);
}
-static inline bool HasBeenNotifiedInternal(
- const std::atomic<bool> *notified_yet) {
- return notified_yet->load(std::memory_order_acquire);
-}
-
-bool Notification::HasBeenNotified() const {
- return HasBeenNotifiedInternal(&this->notified_yet_);
-}
-
void Notification::WaitForNotification() const {
if (!HasBeenNotifiedInternal(&this->notified_yet_)) {
this->mutex_.LockWhen(Condition(&HasBeenNotifiedInternal,
@@ -82,4 +74,5 @@
return notified;
}
+ABSL_NAMESPACE_END
} // namespace absl
diff --git a/absl/synchronization/notification.h b/absl/synchronization/notification.h
index 82d111a..9a354ca 100644
--- a/absl/synchronization/notification.h
+++ b/absl/synchronization/notification.h
@@ -57,6 +57,7 @@
#include "absl/time/time.h"
namespace absl {
+ABSL_NAMESPACE_BEGIN
// -----------------------------------------------------------------------------
// Notification
@@ -73,7 +74,9 @@
// Notification::HasBeenNotified()
//
// Returns the value of the notification's internal "notified" state.
- bool HasBeenNotified() const;
+ bool HasBeenNotified() const {
+ return HasBeenNotifiedInternal(&this->notified_yet_);
+ }
// Notification::WaitForNotification()
//
@@ -105,10 +108,16 @@
void Notify();
private:
+ static inline bool HasBeenNotifiedInternal(
+ const std::atomic<bool>* notified_yet) {
+ return notified_yet->load(std::memory_order_acquire);
+ }
+
mutable Mutex mutex_;
std::atomic<bool> notified_yet_; // written under mutex_
};
+ABSL_NAMESPACE_END
} // namespace absl
#endif // ABSL_SYNCHRONIZATION_NOTIFICATION_H_
diff --git a/absl/synchronization/notification_test.cc b/absl/synchronization/notification_test.cc
index 059d4cd..100ea76 100644
--- a/absl/synchronization/notification_test.cc
+++ b/absl/synchronization/notification_test.cc
@@ -21,6 +21,7 @@
#include "absl/synchronization/mutex.h"
namespace absl {
+ABSL_NAMESPACE_BEGIN
// A thread-safe class that holds a counter.
class ThreadSafeCounter {
@@ -128,4 +129,5 @@
BasicTests(true, &local_notification2);
}
+ABSL_NAMESPACE_END
} // namespace absl