Squashed 'third_party/gperftools/' changes from 54505f1d50..c25941200e
c25941200e fix cmake gperftools_enable_libunwind invalid
43504ab709 tcmalloc page fences: add TCMALLOC_PAGE_FENCE_READABLE option
d57a9ea8bc README: replace "golang" moniker with "Go"
f7c6fb6c8e bump version to 2.9.1
c2f60400a8 prefer backtrace() on OSX
a015377a54 Set tcmalloc heap limit prior to testing oom
c939dd5531 correctly check sized delete hint when asserts are on
47b5b59ca9 bump version to 2.9
d7cbc8c2ff unbreak cmake build
be0bbdb340 amputate various unused bits from elfcore.h
42bab59f25 liberate profile handler from linux_syscall_support
4629511e99 liberate spinlock futex waits from linux_syscall_support includes
2e7094a862 liberate malloc_hook_mmap_linux.h from linux_syscall_support
35301e2e59 add missing noopt wrappings around more operator new calls
fa412adfe3 Fix thread-safety (annotalysis) annotations
cc496aecb8 tcmalloc: Switch thread-safety annotations to support clang
96ba58e19b bump version to 2.9rc
9ce32aefa9 upgrade test bot to xenial (ubuntu 16.04 LTS)
91ff311449 don't default to generic_fp without frame pointers
4cf7dd0a75 enable emergency_malloc on all architectures with mmap
37087ec536 prefer libunwind on x86-64 even with -fno-omit-frame-pointer
f4aa2a435e implement generic frame pointer backtracer
17bab484ae always respect --enable-frame-pointers
22c0eceddc add emacs mode line annotations to remaining files
b12139ddba delete-trailing-whitespace on all files
419c85814d amputate unused dynamic annotations support
73a72cdb61 don't check for snprintf
95b52b0504 don't check for unused uname symbol
01c2697fac amputate unused SleepForMilliseconds from sysinfo.{h,cc}
ac68c97187 don't check for useless __builtin_stack_pointer
7271bb72be unbreak cmake check for TLS support
7c106ca241 don't bother checking for stl namespace and use std
0d6f32b9ce use standard way to print size_t-sized ints
0c11d35f4a amputate checking for __int64
92718aaaeb amputate checking for conflict-signal.h
9bb2937261 amputate checking for inline keyword support
d9c4c3b481 profile-handler: use documented sigev_notify_thread_id in sigevent
43459feb33 configure.ac: check for features.h once
290b123c01 atomicops: Remove Acquire_Store / Release_Load
3b1c60cc4e Add support for Elbrus 2000 (e2k)
c5747615da syscall: Mark x8 as clobbered
d8eb315fb1 bump version to 2.8.1
6ed61f8e91 add note that cmake build is preliminary
6bbf2ed150 Update cmake
913d3eb7d7 Fix a few macros for Apple
64a73b1cb8 Work on fixing linking errors in stacktrace
b788d51eb4 Fix conditional definitions
495229b625 Make internal tcmalloc libs
cca7f6f669 More unit tests and libraries
11dc65c3c4 Fix config headers, add more unit tests
6078fe40d9 Finish configure.ac conversion to CMake, start on Makefile.am
515fb22196 Generate config header
4adb5ff74d Add architecture checks
fa9bedc82c Add most of CMake build
9e4f72fd1f Define options, start system checks
a6ce98174b Add CMakeLists.txt
3134955875 Additional porting for riscv64.
f0e289bdbb Enable build on riscv64.
6c715b4fa1 docs: fix simple typo, defininitions -> definitions
02d5264018 Revert "drop page heap lock when returning memory back to kernel"
151cbf5146 Add OS X arm64 program counter
140e3481d0 Merge pull request #1231 from PatriosTheGreat/master
0fc5cabdfc Fix implicit-int-float-conversion warning.
bda3c82e11 Increase kMaxStackDepth to 254
1d9b8bb59d don't test sbrk hook if we're on linux and don't have __sbrk
180bfa10d7 bumped version to 2.8
c1bcc412ba Don't try to mark esp as clobbered in linux syscall support.
50f89afaed liberate gperftools tests from relying on -fno-builtin-XXX flags
98ccd0f102 prevent inlining in heap-checker unittest
e521472f1a fix linking of page_heap_test on windows
e5f77d6485 chmod -x Makefile.am gperftools.sln
6b92e00cec don't assume HAVE_MMAP on mingw builds
4cddede399 New ProfilerGetStackTrace()
db7aa547ab bumped version to 2.8rc
be3da70298 drop page heap lock when returning memory back to kernel
87acc2782f amputate span events history
e40c7f231a Fix mmap syscall on s390
b7607ee7d4 tcmalloc: ability to disable fallback allocator in memfs
1bfcb5bc3a tcmalloc: fragmentation overhead instrumentation
36bf1309de Fix a clang-tidy readability warning for static member access
2b2a962c2b Remove executable flag for c++ files
8f308afbfe Increase kClassSizesMax to 128 to allow for page size of 4K
d3fefdb694 Allow configuring page size to 4K, 8K, 16K, 32K, 64K, 128K and 256K
cf2df3b000 Fix the removed std::allocator::pointer member type removed in C++20
31024506c5 Add mips64* support
fe62a0baab Update config.h in Windows
8272631b5a Fix a long time typo
c1d546d7b2 never test and always default HAVE_MMAP to on
fba6ce0e7a Fix build on FreeBSD
98ac4ee9bc Fix typos
9e5b162873 don't try to mark rsp as clobbered in linux syscall support
1e36ed7055 Use initial exec TLS model for all thread local variables from thread_cache.cc
8f9a873fce Fix accessing PC on FreeBSD/powerpc and powerpc64
fc00474ddc Include asm/ptrace.h when accessing ucontext_t
5574c87e39 Compile time aggressive decommit option
e9ab4c5304 undef mmap64 function
5eec9d0ae3 Drop not very portable and not very useful unwind benchmark.
1561f0946f check for __sbrk
1de76671d4 Fix mmap region iteration while no regions are recorded.
acdcacc28f Use off64_t instead of __off64_t
0177a2420a Return early in WriteProfile to reduce indentation
b85652bf26 Add generic.total_physical_bytes property to MallocExtension
90df23c81f Make some tcmalloc constants truly const
49dbe4362b Add comment about gperftools 2.8 not deduplicating heapz samples.
63a12a5ed3 Drop de-duplication of heap sample (aka heapz) entries.
954f9dc0e3 Add flag to disable installing unmaintained & deprecated pprof.
893bff51bc Avoid static initialization of pprof path for symbolization.
69867c523b Clean up MSVC projects
f2bca77aed Fix page_heap_test flakiness
c41688bf20 Use standard-conforming alignof in debugallocation.cc
71c8cedaca Fix incompatible aliasing warnings
8dd3040358 Format and fix out of bound access in CpuProfilerSwitch
467502e70a provide constexpr constructor for Sampler
1fb543cc70 Patch _free_dbg to make Debug mode in MSVC works
267f431d80 Use indirect system calls in the linux spinlock implementation
73ee9b1544 Use indirect system calls in the mmap malloc hooks.
3af509d4f9 benchmark: use angle brackets to include ucontext.h
0cdda6d7cc use utf-8 for special symbols
c7a0cfda88 Fix potential missing nul character on resolved symbol names
e42bfc8c06 tcmalloc: use relative addresses with the windows addr2line wrapper
d8f8d1cced tcmalloc: add long form flag '--exe' to specify the binary
25c53aca12 tcmalloc: fixes for the windows addr2line wrapper
f02e28f348 Replace builtin_expect configure test with a direct GCC compiler check
62c4eca6e7 Under x64, the PE loader looks for callbacks in constant sections
0b588e7490 Fix uninitialized memory use in sampler_test
51a5613f21 Upgrade MSVC projects to MSVC2015
44da4ce539 build with c++11 or later
f47a52ce85 Make _recalloc adhere to MS's definition
fe87ffb7ea Disable large allocation report by default
9608fa3bcf bumped version to 2.7
db890ccfad Clean up src/windows/config.h
497ea33165 Fix WIN32_OVERRIDE_ALLOCATORS for VS2017
ebc85cca90 Enable aligned new/delete declarations on Windows when applicable
a3badd6d21 Really fix CheckAddressBits compilation warning
7c718fe176 Add tests for sized deallocation
30e5e614a8 Fix build without static libraries
836c4f29a5 Update documentation for heap_checker.html
e47d0d1c51 powerpc: Re-enable VDSO support
0a66dd3a6a linux: add aarch64_ilp32 support.
05dff09663 Fix signature of sbrk.
33ae0ed2ae unbreak compilation on GNU/Linux i386
977e0d4500 Remove not needed header in vdso_support.cc.
36bfa9a404 Enable tcmalloc VDSO support only on x86 to reduce static initializers
1cb5de6db9 Explicitly prevent int overflow
8f63f2bb98 Correctly detect presence of various functions in tcmalloc.h
736648887b Don't test OOM handling of debugallocator
c4a8e00da4 Fix warning about one of CheckAddressBits functions unused
47c99cf492 unbreak printing large span stats
34f78a2dcd bumped version to 2.7rc
db98aac55a Add a central free list for kMaxPages-sized spans
d7be938560 implement more robust detection of sized delete support
f1d3fe4a21 refactored handling of reverse span set iterator for correctness
59c77be0fa Update docs for central page heap to reflect tree
06c9414ec4 Implemented O(log n) searching among large spans
a42e44738a typo in docs/tcmalloc.html
71bf09aabe bumped version to 2.6.3
0bccb5e658 fix malloc fast path for patched windows functions
8b1d13c631 configure.ac: use link check for std::align_val_t
36ab068baa configure.ac: better test for -faligned-new
6a4b079997 bumped version to 2.6.2
2291714518 implement fast-path for memalign/aligned_alloc/tc_new_aligned
8b9728b023 add memalign benchmark to malloc_bench
79c91a9810 always define empty PERFTOOLS_NOTHROW
03da6afff5 unbreak throw declarations on operators new/delete
89fe59c831 Fix OOM handling in fast-path
a29a0cf348 delete-trailing-whitespace on thread_cache.*
e6cd69bdec reintroduce aliasing for aligned delete
fb30c3d435 fully disable aligned new on windows for now
7efb3ecf37 Add support for C++17 operator new/delete for overaligned types.
7a6e25f3b1 Add new statistics for the PageHeap
6e3a702fb9 Fix data race setting size_left_ in ThreadCache::SetMaxSize
235471f965 fix memory leak in Symbolize function
47efdd60f5 Added mising va_end() in TracePrintf function
497b60ef0f Implemented GetProgramInvocationName on FreeBSD
ac072a3fc7 Revert "Ignore current_instance heap allocation when leak sanitizer is enabled"
fb5987d579 Revert "Ensure that lsan flags are appended on all necessary targets"
5815f02105 Use safe getenv for setting up backtrace capturing method
aab4277311 Fixed LTO warning about the mismatch between return values for ProfilingIsEnabledForAllThreads()
d406f22853 implement support for C11 aligned_alloc
92a27e41a1 Fix build on macOS.
e033431e5a include fcntl.h for loff_t definition
e41bc41404 Use ucontext_t instead of struct ucontext
bf840dec04 bumped version to 2.6.1
2d220c7e26 Replace "throw()" by "PERFTOOLS_NOTHROW"
c4de73c0e6 Add PERFTOOLS_THROW where necessary (as detected by GCC).
e5fbd0e24e Rename PERFTOOLS_THROW into PERFTOOLS_NOTHROW.
eeb7b84c20 Register tcmalloc atfork handler as early as possible
208c26caef Add initial syscall support for mips64 32-bit ABI
a3bf61ca81 Ensure that lsan flags are appended on all necessary targets
97646a1932 Add missing NEWS entry for recent 2.6 release
4be05e43a1 bumped version up to 2.6
70a35422b5 Ignore current_instance heap allocation when leak sanitizer is enabled
6eca6c64fa Revert "issue-654: [pprof] handle split text segments"
a495969cb6 update the prev_class_size in each loop, or the min_object_size of tcmalloc.thread will always be 1 when calling GetFreeListSizes
163224d8af Document HEAPPROFILESIGNAL environment variable
5ac82ec5b9 added stacktrace capturing benchmark
c571ae2fc9 2.6rc4
f2bae51e7e Revert "Revert "disable dynamic sized delete support by default""
6426c0cc80 2.6rc3
0c0e2fe43b enable 48-bit page map on msvc as well
83d6818295 speed up 3-level page map access
f7ff175b92 add configure-time warning on unsupported backtrace capturing
cef582350c align fast-path functions only if compiler supports that
bddf862b18 actually support very early freeing of NULL
07a124d8c1 don't use arg-ful constructor attribute for early nallocx test
5346b8a4de don't depend on SIZE_MAX definition in sampler.cc
50125d8f70 2.6rc2
a5e8e42a47 don't link-in libunwind if libunwind.h is missing
e92acdf98d Fix compilation error for powerpc32
b48403a4b0 2.6rc
53f15325d9 fix compilation of tcmalloc_unittest.cc on older llvm-gcc
b1d88662cb change size class to be represented by 32 bit int
991f47a159 change default transfer batch back to 32
7bc34ad1f6 support different number of size classes at runtime
4585b78c8d massage allocation and deallocation fast-path for performance
5964a1d9c9 always inline a number of hot functions
e419b7b9a6 introduce ATTRIBUTE_ALWAYS_INLINE
7d588da7ec synchronized Sampler implementation with Google-internal version
27da4ade70 reduce size of class_to_size_ array
335f09d4e4 use static location for pageheap
6ff332fb51 move size classes map earlier in SizeMap
121b1cb32e slightly faster size class cache
b57c0bad41 init tcmalloc prior to replacing system alloc
71fa9f8730 use 2-level page map for 48-bit addresses
bad70249dd use 48-bit addresses on 64-bit arms too
5f12147c6d use hidden visibility for some key global variables
dfd53da578 set ENOMEM in handle_oom
14fd551072 avoid O(N²) in thread cache creation code
507a105e84 pass original size to DoSampledAllocation
bb77979dea don't declare throw() on malloc funtions since it is faster
89c74cb79c handle duplicate google_malloc frames in malloc hook stack trace
0feb1109ac fix stack trace capturing in debug malloc
0506e965ee replace LIKELY/UNLIKELY with PREDICT_{TRUE,FALSE}
59a4987054 prevent inlining ATTRIBUTE_SECTION functions
ebb575b8a0 Revert "enabled aggressive decommit by default"
b82d89cb7c Revert "disable dynamic sized delete support by default"
fac0bb44d5 Do not depend on memchr in commandlineflags::StringToBool
7d49f015a0 Make GetenvBeforeMain work inside ifunc handler
a2550b6309 turn bench_fastpath_throughput into actual throughput benchmark
b762b1a492 added sized free benchmarks to malloc_bench
71ffc1cd6b added free lists randomization step to malloc_bench
732dfeb83d Run StartStopNoOptionsEmpty profiledata unittest
cbb312fbe8 aggressive decommit: only free necessary regions and fix O(N²)
6d98223a90 don't build with -fno-exceptions
d6a1931cce fixed warning in casting heap of checker's main_thread_counter
5c778701d9 added tcmalloc minimal unittest with ASSERTs checked
a9167617ab drop unused g_load_map variable in patch_functionc.cc
d52e56dcb5 don't compare integer to NULL
bae00c0341 add fake_stacktrace_scope to few msvc projects
79aab4fed4 correctly dllexport nallocx on windows
b010895a08 don't undef PERFTOOLS_DLL_DECL
491b1aca7e don't try to use pthread_atfork on windows
691045b957 suppress warnings from legacy headers while building legacy headers test
22f7ceb97a use unsigned for few flags in mini_disassembler_types.h
9b17a8a5ba remove superfluous size_t value >= 0 check
86ce69d77f Update binary_trees.cc
cd8586ed6c Fix path names in README
98753aa737 test that sized deallocation really works before enabling it
5618ef7850 Don't assume memalign exists in memalign vs nallocx test
bf640cd740 rename sys allocator's sys_alloc symbol to tcmalloc_sys_alloc
069e3b1655 build malloc_bench_shared_full only when full tcmalloc is built
b8f9d0d44f ported nallocx support from Google-internal tcmalloc
b0abefd938 Fix a typo in the page fence flag declaration
855b380006 replace docs by doc
664210ead8 doc -> docs, with symlink
75dc9a6e14 Fix Post(s)cript tyos
dde32f8bbc Fix unaligned memory accesses in debug allocator
02eeed29df Fix redefinition of mmap on aarch64.
c07a15cff4 [windows] patch _free_base as well
acac6af26b Fix finding default zone on macOS sierra
7822b5b0b9 Stop using glibc malloc hooks
c92f0ed089 Remove references to __malloc_initialize_hook
9709eef361 Merge pull request #821 from jtmcdole/patch-1
44f276e132 Rename TCMALLOC_DEBUG to PERFTOOLS_VERBOSE
eb474c995e Summary: support gcc atomic ops on clang too
7f86eab1f3 Recognize .node files as shared libraries
bf8eacce69 Add support for 31-bit s390; merge linux_syscall_support.h changes from upstream.
c54218069b Update README
06f4ce65c2 Small performance tweak: avoid calling time() if we don't need it
db8d483609 Autogenerate ChangeLog from git on make dist
4a13598319 renamed ChangeLog to ChangeLog.old
7852eeb75b Use initial-exec tls for libunwind's recursion flag
a07f9fe75a gerftools -> gperftools in readme
9fd6d26879 added define to enable MADV_FREE usage on Linux
6f7a14f45e Don't use MADV_FREE on Linux
55cf6e6281 Fix symbol resolution on OSX
8e85843622 added simple .travis.yml config
05e40d29c0 Recognize modern Linux ARM
632de2975e bumped version up to 2.5
6682016092 Unbreak profiling with CPUPROFILE_FREQUENCY=1
6ff86ff6a7 bumped version to 2.4.91 for 2.5rc2
782165fa7f build sized delete aliases even when sized-delete is disabled
06811b3ae4 disable dynamic sized delete support by default
d4d99eb608 unbreak compilation with visual studio
126d4582c1 Call function pointers with the right type
e0fa28ef7d Don't shift a type by more than its width
a1c764d263 Initialize counters in test
22123a37c2 Don't overflow a signed integer
66e1e94f38 added minimal "header section" to README
2804b7cfee bumped version to 2.5rc
f47fefbfc1 updated NEWS for 2.5rc
cef6036174 alias same malloc/free variants to their canonical versions
ea8d242061 Re-enable MultipleIdleNonIdlePhases test
c9962f698b added maybe_emergency_malloc.h to Makefile.am
7dd4af6536 don't round up sizes for large allocation when sampling
4f3410e759 enable emergency malloc by default on arm when using libunwind
7f12051dbe implemented emergency malloc
3ee2360250 replaced invalid uses of __THROW
013b82abcf unbreak <malloc.h> inclusion in gperftools/tcmalloc.h
19903e6f15 drop detection of sys/malloc.h and malloc/malloc.h
cdff090ebd Fix several harmless clang warnings
9095ed0840 implemented stacktrace capturing via libgcc's C++ ABI function
728cbe1021 force profiler_unittest to do 'real' work
fff6b4fb88 Extend low-level allocator to support custom pages allocator
32d9926795 added malloc_bench_shared_full
00d8fa1ef8 always use real throw() on operators new/delete
08e034ad59 Detect working ifunc before enabling dynamic sized delete support
a788f354a0 include unistd.h for getpid in thread_lister.c
644a6bdbdb Add support for Linux s390x
bab7753aad Fix typo in heap-checker-death_unittest.sh
17182e1d3c Fix include of malloc_hook_c.h in malloc_hook.h
c69721b2b2 Add support for obtaining cache size of the current thread and softer idling
5ce42e535d Don't always arm the profiling timer.
7f801ea091 Make sure the alias is not removed by link-time optimization when it can prove that it isn't used by the program, as it might still be needed to override the corresponding symbol in shared libraries (or inline assembler for that matter). For example, suppose the program uses malloc and free but not calloc and is statically linked against tcmalloc (built with -flto) and LTO is done. Then before this patch the calloc alias would be deleted by LTO due to not being used, but the malloc/free aliases would be kept because they are used by the program. Suppose the program is dynamically linked with a shared library that allocates memory using calloc and later frees it by calling free. Then calloc will use the libc memory allocator, because the calloc alias was deleted, but free will call into tcmalloc, resulting in a crash.
6b3e6ef5e0 don't retain compatibility with old docdir behavior
ccffcbd9e9 support use of configure --docdir argument
050f2d28be use alias attribute only for elf platforms
07b0b21ddd fix compilation error in spinlock
e14450366a Added better description for GetStats API
64892ae730 lower default transfer batch size down to 512
6fdfc5a7f4 implemented enabling sized-delete support at runtime
c2a79d063c use x86 pause in spin loop
0fb6dd8aa3 added binary_trees benchmark
a8852489e5 drop unsupported allocation sampling code in tcmalloc_minimal
a9db0ae516 implemented (disabled by default) sized delete support
88686972b9 pass -fsized-deallocation to gcc 5
0a18fab3af implemented sized free support via tc_free_sized
464688ab6d speedup free code path by dropping "fast path allowed check"
10f7e20716 added SizeMap::MaybeSizeClass
436e1dea43 slightly faster GetCacheIfPresent
04df911915 tell compiler that non-empty hooks are unlikely
8cc75acd1f correctly test for -Wno-unused-result support
7753d8239b fixed clang warning about shifting negative values
ae09ebb383 Fix tmpdir usage in heap-profiler_unittest.sh
df34e71b57 use $0 when referring to pprof
7773ea64ee Alignment fix to static variables for system allocators
c46eb1f3d2 Fixed printf misuse in pprof - printed string was passed as format. Better use print instead
9bbed8b1a8 Fixed assembler argument passing inside _syscall6 on MIPS - it was causing 'Expression too complex' compilation errors in spinlock
962aa53c55 added more fastpath microbenchmarks
347a830689 Ensure that PPROF_PATH is set for debugallocation_test
a9059b7c30 prevent clang from inlining Mallocer in heap checker unittest
6627f9217d drop cycleclock
f985abc296 amputate unportable and unused stuff from sysinfo
16408eb4d7 amputated wait_cycles accounting in spinlocks
fedceef40c drop cycleclock reference in ThreadCache
d7fdc3fc9d dropped unused and unsupported synchronization profiling facility
3a054d37c1 dropped unused SpinLockWait function
5b62d38329 avoid checking for dup. entries on empty backtrace
7b9ded722e fixed compiler warning in memory_region_map.cc
4194e485cb Don't link libtcmalloc_minimal.so to libpthread.so
121038308d Check if _MSC_VER is defined to avoid warnings
7367322995 Make default config.h work with VS2015
ae0a444db0 Ensure ThreadCache objects are CACHELINE_ALIGNED.
ea0b1d3154 unbreak TestErrno again
e53aef24ad don't try to test memalign on windows
7707582448 Merge pull request #717 from myrsloik/master
9eb63bddfb Use correct mangled new and delete symbols on windows x64
5078abdb33 Don't discard curl options if timeout is not defined.
Change-Id: If80121a97ff4c18289c6ebff7ccea1d1b355ec89
git-subtree-dir: third_party/gperftools
git-subtree-split: c25941200ef4ce39d0774c1332ff7abfbeab7035
Signed-off-by: Brian Silverman <bsilver16834@gmail.com>
diff --git a/src/tcmalloc.cc b/src/tcmalloc.cc
index b7d1913..9ec663e 100644
--- a/src/tcmalloc.cc
+++ b/src/tcmalloc.cc
@@ -34,7 +34,7 @@
// A malloc that uses a per-thread cache to satisfy small malloc requests.
// (The time for malloc/free of a small object drops from 300 ns to 50 ns.)
//
-// See doc/tcmalloc.html for a high-level
+// See docs/tcmalloc.html for a high-level
// description of how this malloc works.
//
// SYNCHRONIZATION
@@ -88,12 +88,12 @@
// goes from about 1100 ns to about 300 ns.
#include "config.h"
+// At least for gcc on Linux/i386 and Linux/amd64 not adding throw()
+// to tc_xxx functions actually ends up generating better code.
+#define PERFTOOLS_NOTHROW
#include <gperftools/tcmalloc.h>
#include <errno.h> // for ENOMEM, EINVAL, errno
-#ifdef HAVE_SYS_CDEFS_H
-#include <sys/cdefs.h> // for __THROW
-#endif
#if defined HAVE_STDINT_H
#include <stdint.h>
#elif defined HAVE_INTTYPES_H
@@ -114,6 +114,7 @@
#include <gperftools/malloc_extension.h>
#include <gperftools/malloc_hook.h> // for MallocHook
+#include <gperftools/nallocx.h>
#include "base/basictypes.h" // for int64
#include "base/commandlineflags.h" // for RegisterFlagValidator, etc
#include "base/dynamic_annotations.h" // for RunningOnValgrind
@@ -132,16 +133,7 @@
#include "tcmalloc_guard.h" // for TCMallocGuard
#include "thread_cache.h" // for ThreadCache
-#ifdef __clang__
-// clang's apparent focus on code size somehow causes it to ignore
-// normal inline directives even for few functions which inlining is
-// key for performance. In order to get performance of clang's
-// generated code closer to normal, we're forcing inlining via
-// attribute.
-#define ALWAYS_INLINE inline __attribute__((always_inline))
-#else
-#define ALWAYS_INLINE inline
-#endif
+#include "maybe_emergency_malloc.h"
#if (defined(_WIN32) && !defined(__CYGWIN__) && !defined(__CYGWIN32__)) && !defined(WIN32_OVERRIDE_ALLOCATORS)
# define WIN32_DO_PATCHING 1
@@ -150,20 +142,13 @@
// Some windows file somewhere (at least on cygwin) #define's small (!)
#undef small
-using STL_NAMESPACE::max;
-using STL_NAMESPACE::numeric_limits;
-using STL_NAMESPACE::vector;
+using std::max;
+using std::min;
+using std::numeric_limits;
+using std::vector;
#include "libc_override.h"
-// __THROW is defined in glibc (via <sys/cdefs.h>). It means,
-// counter-intuitively, "This function will never throw an exception."
-// It's an optional optimization tool, but we may need to use it to
-// match glibc prototypes.
-#ifndef __THROW // I guess we're not on a glibc system
-# define __THROW // __THROW is just an optimization, so ok to make it ""
-#endif
-
using tcmalloc::AlignmentForSize;
using tcmalloc::kLog;
using tcmalloc::kCrash;
@@ -177,8 +162,30 @@
using tcmalloc::Static;
using tcmalloc::ThreadCache;
-DECLARE_int64(tcmalloc_sample_parameter);
DECLARE_double(tcmalloc_release_rate);
+DECLARE_int64(tcmalloc_heap_limit_mb);
+
+// Those common architectures are known to be safe w.r.t. aliasing function
+// with "extra" unused args to function with fewer arguments (e.g.
+// tc_delete_nothrow being aliased to tc_delete).
+//
+// Benefit of aliasing is relatively moderate. It reduces instruction
+// cache pressure a bit (not relevant for largely unused
+// tc_delete_nothrow, but is potentially relevant for
+// tc_delete_aligned (or sized)). It also used to be the case that gcc
+// 5+ optimization for merging identical functions kicked in and
+// "screwed" one of the otherwise identical functions with extra
+// jump. I am not able to reproduce that anymore.
+#if !defined(__i386__) && !defined(__x86_64__) && \
+ !defined(__ppc__) && !defined(__PPC__) && \
+ !defined(__aarch64__) && !defined(__mips__) && !defined(__arm__)
+#undef TCMALLOC_NO_ALIASES
+#define TCMALLOC_NO_ALIASES
+#endif
+
+#if defined(__GNUC__) && defined(__ELF__) && !defined(TCMALLOC_NO_ALIASES)
+#define TC_ALIAS(name) __attribute__((alias(#name)))
+#endif
// For windows, the printf we use to report large allocs is
// potentially dangerous: it could cause a malloc that would cause an
@@ -213,64 +220,97 @@
// MallocHook::GetCallerStackTrace can function accurately.
#ifndef _WIN32 // windows doesn't have attribute_section, so don't bother
extern "C" {
- void* tc_malloc(size_t size) __THROW
+ void* tc_malloc(size_t size) PERFTOOLS_NOTHROW
ATTRIBUTE_SECTION(google_malloc);
- void tc_free(void* ptr) __THROW
+ void tc_free(void* ptr) PERFTOOLS_NOTHROW
ATTRIBUTE_SECTION(google_malloc);
- void* tc_realloc(void* ptr, size_t size) __THROW
+ void tc_free_sized(void* ptr, size_t size) PERFTOOLS_NOTHROW
ATTRIBUTE_SECTION(google_malloc);
- void* tc_calloc(size_t nmemb, size_t size) __THROW
+ void* tc_realloc(void* ptr, size_t size) PERFTOOLS_NOTHROW
ATTRIBUTE_SECTION(google_malloc);
- void tc_cfree(void* ptr) __THROW
+ void* tc_calloc(size_t nmemb, size_t size) PERFTOOLS_NOTHROW
+ ATTRIBUTE_SECTION(google_malloc);
+ void tc_cfree(void* ptr) PERFTOOLS_NOTHROW
ATTRIBUTE_SECTION(google_malloc);
- void* tc_memalign(size_t __alignment, size_t __size) __THROW
+ void* tc_memalign(size_t __alignment, size_t __size) PERFTOOLS_NOTHROW
ATTRIBUTE_SECTION(google_malloc);
- int tc_posix_memalign(void** ptr, size_t align, size_t size) __THROW
+ int tc_posix_memalign(void** ptr, size_t align, size_t size) PERFTOOLS_NOTHROW
ATTRIBUTE_SECTION(google_malloc);
- void* tc_valloc(size_t __size) __THROW
+ void* tc_valloc(size_t __size) PERFTOOLS_NOTHROW
ATTRIBUTE_SECTION(google_malloc);
- void* tc_pvalloc(size_t __size) __THROW
+ void* tc_pvalloc(size_t __size) PERFTOOLS_NOTHROW
ATTRIBUTE_SECTION(google_malloc);
- void tc_malloc_stats(void) __THROW
+ void tc_malloc_stats(void) PERFTOOLS_NOTHROW
ATTRIBUTE_SECTION(google_malloc);
- int tc_mallopt(int cmd, int value) __THROW
+ int tc_mallopt(int cmd, int value) PERFTOOLS_NOTHROW
ATTRIBUTE_SECTION(google_malloc);
#ifdef HAVE_STRUCT_MALLINFO
- struct mallinfo tc_mallinfo(void) __THROW
+ struct mallinfo tc_mallinfo(void) PERFTOOLS_NOTHROW
ATTRIBUTE_SECTION(google_malloc);
#endif
void* tc_new(size_t size)
ATTRIBUTE_SECTION(google_malloc);
- void tc_delete(void* p) __THROW
+ void tc_delete(void* p) PERFTOOLS_NOTHROW
+ ATTRIBUTE_SECTION(google_malloc);
+ void tc_delete_sized(void* p, size_t size) PERFTOOLS_NOTHROW
ATTRIBUTE_SECTION(google_malloc);
void* tc_newarray(size_t size)
ATTRIBUTE_SECTION(google_malloc);
- void tc_deletearray(void* p) __THROW
+ void tc_deletearray(void* p) PERFTOOLS_NOTHROW
+ ATTRIBUTE_SECTION(google_malloc);
+ void tc_deletearray_sized(void* p, size_t size) PERFTOOLS_NOTHROW
ATTRIBUTE_SECTION(google_malloc);
// And the nothrow variants of these:
- void* tc_new_nothrow(size_t size, const std::nothrow_t&) __THROW
+ void* tc_new_nothrow(size_t size, const std::nothrow_t&) PERFTOOLS_NOTHROW
ATTRIBUTE_SECTION(google_malloc);
- void* tc_newarray_nothrow(size_t size, const std::nothrow_t&) __THROW
+ void* tc_newarray_nothrow(size_t size, const std::nothrow_t&) PERFTOOLS_NOTHROW
ATTRIBUTE_SECTION(google_malloc);
// Surprisingly, standard C++ library implementations use a
// nothrow-delete internally. See, eg:
// http://www.dinkumware.com/manuals/?manual=compleat&page=new.html
- void tc_delete_nothrow(void* ptr, const std::nothrow_t&) __THROW
+ void tc_delete_nothrow(void* ptr, const std::nothrow_t&) PERFTOOLS_NOTHROW
ATTRIBUTE_SECTION(google_malloc);
- void tc_deletearray_nothrow(void* ptr, const std::nothrow_t&) __THROW
+ void tc_deletearray_nothrow(void* ptr, const std::nothrow_t&) PERFTOOLS_NOTHROW
ATTRIBUTE_SECTION(google_malloc);
+#if defined(ENABLE_ALIGNED_NEW_DELETE)
+
+ void* tc_new_aligned(size_t size, std::align_val_t al)
+ ATTRIBUTE_SECTION(google_malloc);
+ void tc_delete_aligned(void* p, std::align_val_t al) PERFTOOLS_NOTHROW
+ ATTRIBUTE_SECTION(google_malloc);
+ void tc_delete_sized_aligned(void* p, size_t size, std::align_val_t al) PERFTOOLS_NOTHROW
+ ATTRIBUTE_SECTION(google_malloc);
+ void* tc_newarray_aligned(size_t size, std::align_val_t al)
+ ATTRIBUTE_SECTION(google_malloc);
+ void tc_deletearray_aligned(void* p, std::align_val_t al) PERFTOOLS_NOTHROW
+ ATTRIBUTE_SECTION(google_malloc);
+ void tc_deletearray_sized_aligned(void* p, size_t size, std::align_val_t al) PERFTOOLS_NOTHROW
+ ATTRIBUTE_SECTION(google_malloc);
+
+ // And the nothrow variants of these:
+ void* tc_new_aligned_nothrow(size_t size, std::align_val_t al, const std::nothrow_t&) PERFTOOLS_NOTHROW
+ ATTRIBUTE_SECTION(google_malloc);
+ void* tc_newarray_aligned_nothrow(size_t size, std::align_val_t al, const std::nothrow_t&) PERFTOOLS_NOTHROW
+ ATTRIBUTE_SECTION(google_malloc);
+ void tc_delete_aligned_nothrow(void* ptr, std::align_val_t al, const std::nothrow_t&) PERFTOOLS_NOTHROW
+ ATTRIBUTE_SECTION(google_malloc);
+ void tc_deletearray_aligned_nothrow(void* ptr, std::align_val_t al, const std::nothrow_t&) PERFTOOLS_NOTHROW
+ ATTRIBUTE_SECTION(google_malloc);
+
+#endif // defined(ENABLE_ALIGNED_NEW_DELETE)
+
// Some non-standard extensions that we support.
// This is equivalent to
// OS X: malloc_size()
// glibc: malloc_usable_size()
// Windows: _msize()
- size_t tc_malloc_size(void* p) __THROW
+ size_t tc_malloc_size(void* p) PERFTOOLS_NOTHROW
ATTRIBUTE_SECTION(google_malloc);
} // extern "C"
#endif // #ifndef _WIN32
@@ -285,7 +325,11 @@
// the pagemap cache has a non-zero sizeclass.) This is a cheap (source-editing
// required) kind of exception handling for these routines.
namespace {
-void InvalidFree(void* ptr) {
+ATTRIBUTE_NOINLINE void InvalidFree(void* ptr) {
+ if (tcmalloc::IsEmergencyPtr(ptr)) {
+ tcmalloc::EmergencyFree(ptr);
+ return;
+ }
Log(kCrash, __FILE__, __LINE__, "Attempt to free invalid pointer", ptr);
}
@@ -320,7 +364,7 @@
PageHeap::LargeSpanStats* large_spans) {
r->central_bytes = 0;
r->transfer_bytes = 0;
- for (int cl = 0; cl < kNumClasses; ++cl) {
+ for (int cl = 0; cl < Static::num_size_classes(); ++cl) {
const int length = Static::central_cache()[cl].length();
const int tc_length = Static::central_cache()[cl].tc_length();
const size_t cache_overhead = Static::central_cache()[cl].OverheadBytes();
@@ -359,7 +403,7 @@
// WRITE stats to "out"
static void DumpStats(TCMalloc_Printer* out, int level) {
TCMallocStats stats;
- uint64_t class_count[kNumClasses];
+ uint64_t class_count[kClassSizesMax];
PageHeap::SmallSpanStats small;
PageHeap::LargeSpanStats large;
if (level >= 2) {
@@ -425,18 +469,25 @@
out->printf("Total size of freelists for per-thread caches,\n");
out->printf("transfer cache, and central cache, by size class\n");
out->printf("------------------------------------------------\n");
- uint64_t cumulative = 0;
- for (int cl = 0; cl < kNumClasses; ++cl) {
+ uint64_t cumulative_bytes = 0;
+ uint64_t cumulative_overhead = 0;
+ for (uint32 cl = 0; cl < Static::num_size_classes(); ++cl) {
if (class_count[cl] > 0) {
- uint64_t class_bytes =
- class_count[cl] * Static::sizemap()->ByteSizeForClass(cl);
- cumulative += class_bytes;
- out->printf("class %3d [ %8" PRIuS " bytes ] : "
- "%8" PRIu64 " objs; %5.1f MiB; %5.1f cum MiB\n",
- cl, Static::sizemap()->ByteSizeForClass(cl),
+ size_t cl_size = Static::sizemap()->ByteSizeForClass(cl);
+ const uint64_t class_bytes = class_count[cl] * cl_size;
+ cumulative_bytes += class_bytes;
+ const uint64_t class_overhead =
+ Static::central_cache()[cl].OverheadBytes();
+ cumulative_overhead += class_overhead;
+ out->printf("class %3d [ %8zu bytes ] : "
+ "%8" PRIu64 " objs; %5.1f MiB; %5.1f cum MiB; "
+ "%8.3f overhead MiB; %8.3f cum overhead MiB\n",
+ cl, cl_size,
class_count[cl],
class_bytes / MiB,
- cumulative / MiB);
+ cumulative_bytes / MiB,
+ class_overhead / MiB,
+ cumulative_overhead / MiB);
}
}
@@ -454,9 +505,9 @@
out->printf("------------------------------------------------\n");
uint64_t total_normal = 0;
uint64_t total_returned = 0;
- for (int s = 0; s < kMaxPages; s++) {
- const int n_length = small.normal_length[s];
- const int r_length = small.returned_length[s];
+ for (int s = 1; s <= kMaxPages; s++) {
+ const int n_length = small.normal_length[s - 1];
+ const int r_length = small.returned_length[s - 1];
if (n_length + r_length > 0) {
uint64_t n_pages = s * n_length;
uint64_t r_pages = s * r_length;
@@ -475,8 +526,9 @@
total_normal += large.normal_pages;
total_returned += large.returned_pages;
- out->printf(">255 large * %6u spans ~ %6.1f MiB; %6.1f MiB cum"
+ out->printf(">%-5u large * %6u spans ~ %6.1f MiB; %6.1f MiB cum"
"; unmapped: %6.1f MiB; %6.1f MiB cum\n",
+ static_cast<unsigned int>(kMaxPages),
static_cast<unsigned int>(large.spans),
PagesToMiB(large.normal_pages + large.returned_pages),
PagesToMiB(total_normal + total_returned),
@@ -630,6 +682,17 @@
return DumpHeapGrowthStackTraces();
}
+ virtual size_t GetThreadCacheSize() {
+ ThreadCache* tc = ThreadCache::GetCacheIfPresent();
+ if (!tc)
+ return 0;
+ return tc->Size();
+ }
+
+ virtual void MarkThreadTemporarilyIdle() {
+ ThreadCache::BecomeTemporarilyIdle();
+ }
+
virtual void Ranges(void* arg, RangeFunction func) {
IterateOverRanges(arg, func);
}
@@ -656,6 +719,14 @@
return true;
}
+ if (strcmp(name, "generic.total_physical_bytes") == 0) {
+ TCMallocStats stats;
+ ExtractStats(&stats, NULL, NULL, NULL);
+ *value = stats.pageheap.system_bytes + stats.metadata_bytes -
+ stats.pageheap.unmapped_bytes;
+ return true;
+ }
+
if (strcmp(name, "tcmalloc.slack_bytes") == 0) {
// Kept for backwards compatibility. Now defined externally as:
// pageheap_free_bytes + pageheap_unmapped_bytes.
@@ -698,6 +769,54 @@
return true;
}
+ if (strcmp(name, "tcmalloc.pageheap_committed_bytes") == 0) {
+ SpinLockHolder l(Static::pageheap_lock());
+ *value = Static::pageheap()->stats().committed_bytes;
+ return true;
+ }
+
+ if (strcmp(name, "tcmalloc.pageheap_scavenge_count") == 0) {
+ SpinLockHolder l(Static::pageheap_lock());
+ *value = Static::pageheap()->stats().scavenge_count;
+ return true;
+ }
+
+ if (strcmp(name, "tcmalloc.pageheap_commit_count") == 0) {
+ SpinLockHolder l(Static::pageheap_lock());
+ *value = Static::pageheap()->stats().commit_count;
+ return true;
+ }
+
+ if (strcmp(name, "tcmalloc.pageheap_total_commit_bytes") == 0) {
+ SpinLockHolder l(Static::pageheap_lock());
+ *value = Static::pageheap()->stats().total_commit_bytes;
+ return true;
+ }
+
+ if (strcmp(name, "tcmalloc.pageheap_decommit_count") == 0) {
+ SpinLockHolder l(Static::pageheap_lock());
+ *value = Static::pageheap()->stats().decommit_count;
+ return true;
+ }
+
+ if (strcmp(name, "tcmalloc.pageheap_total_decommit_bytes") == 0) {
+ SpinLockHolder l(Static::pageheap_lock());
+ *value = Static::pageheap()->stats().total_decommit_bytes;
+ return true;
+ }
+
+ if (strcmp(name, "tcmalloc.pageheap_reserve_count") == 0) {
+ SpinLockHolder l(Static::pageheap_lock());
+ *value = Static::pageheap()->stats().reserve_count;
+ return true;
+ }
+
+ if (strcmp(name, "tcmalloc.pageheap_total_reserve_bytes") == 0) {
+ SpinLockHolder l(Static::pageheap_lock());
+ *value = Static::pageheap()->stats().total_reserve_bytes;
+ return true;
+ }
+
if (strcmp(name, "tcmalloc.max_total_thread_cache_bytes") == 0) {
SpinLockHolder l(Static::pageheap_lock());
*value = ThreadCache::overall_thread_cache_size();
@@ -712,10 +831,17 @@
}
if (strcmp(name, "tcmalloc.aggressive_memory_decommit") == 0) {
+ SpinLockHolder l(Static::pageheap_lock());
*value = size_t(Static::pageheap()->GetAggressiveDecommit());
return true;
}
+ if (strcmp(name, "tcmalloc.heap_limit_mb") == 0) {
+ SpinLockHolder l(Static::pageheap_lock());
+ *value = FLAGS_tcmalloc_heap_limit_mb;
+ return true;
+ }
+
return false;
}
@@ -729,10 +855,17 @@
}
if (strcmp(name, "tcmalloc.aggressive_memory_decommit") == 0) {
+ SpinLockHolder l(Static::pageheap_lock());
Static::pageheap()->SetAggressiveDecommit(value != 0);
return true;
}
+ if (strcmp(name, "tcmalloc.heap_limit_mb") == 0) {
+ SpinLockHolder l(Static::pageheap_lock());
+ FLAGS_tcmalloc_heap_limit_mb = value;
+ return true;
+ }
+
return false;
}
@@ -744,12 +877,12 @@
virtual SysAllocator* GetSystemAllocator() {
SpinLockHolder h(Static::pageheap_lock());
- return sys_alloc;
+ return tcmalloc_sys_alloc;
}
virtual void SetSystemAllocator(SysAllocator* alloc) {
SpinLockHolder h(Static::pageheap_lock());
- sys_alloc = alloc;
+ tcmalloc_sys_alloc = alloc;
}
virtual void ReleaseToSystem(size_t num_bytes) {
@@ -784,15 +917,7 @@
virtual double GetMemoryReleaseRate() {
return FLAGS_tcmalloc_release_rate;
}
- virtual size_t GetEstimatedAllocatedSize(size_t size) {
- if (size <= kMaxSize) {
- const size_t cl = Static::sizemap()->SizeClass(size);
- const size_t alloc_size = Static::sizemap()->ByteSizeForClass(cl);
- return alloc_size;
- } else {
- return tcmalloc::pages(size) << kPageShift;
- }
- }
+ virtual size_t GetEstimatedAllocatedSize(size_t size);
// This just calls GetSizeWithCallback, but because that's in an
// unnamed namespace, we need to move the definition below it in the
@@ -810,8 +935,8 @@
if ((p >> (kAddressBits - kPageShift)) > 0) {
return kNotOwned;
}
- size_t cl = Static::pageheap()->GetSizeClassIfCached(p);
- if (cl != 0) {
+ uint32 cl;
+ if (Static::pageheap()->TryGetSizeClass(p, &cl)) {
return kOwned;
}
const Span *span = Static::pageheap()->GetDescriptor(p);
@@ -819,19 +944,19 @@
}
virtual void GetFreeListSizes(vector<MallocExtension::FreeListInfo>* v) {
- static const char* kCentralCacheType = "tcmalloc.central";
- static const char* kTransferCacheType = "tcmalloc.transfer";
- static const char* kThreadCacheType = "tcmalloc.thread";
- static const char* kPageHeapType = "tcmalloc.page";
- static const char* kPageHeapUnmappedType = "tcmalloc.page_unmapped";
- static const char* kLargeSpanType = "tcmalloc.large";
- static const char* kLargeUnmappedSpanType = "tcmalloc.large_unmapped";
+ static const char kCentralCacheType[] = "tcmalloc.central";
+ static const char kTransferCacheType[] = "tcmalloc.transfer";
+ static const char kThreadCacheType[] = "tcmalloc.thread";
+ static const char kPageHeapType[] = "tcmalloc.page";
+ static const char kPageHeapUnmappedType[] = "tcmalloc.page_unmapped";
+ static const char kLargeSpanType[] = "tcmalloc.large";
+ static const char kLargeUnmappedSpanType[] = "tcmalloc.large_unmapped";
v->clear();
// central class information
int64 prev_class_size = 0;
- for (int cl = 1; cl < kNumClasses; ++cl) {
+ for (int cl = 1; cl < Static::num_size_classes(); ++cl) {
size_t class_size = Static::sizemap()->ByteSizeForClass(cl);
MallocExtension::FreeListInfo i;
i.min_object_size = prev_class_size + 1;
@@ -851,7 +976,7 @@
}
// Add stats from per-thread heaps
- uint64_t class_count[kNumClasses];
+ uint64_t class_count[kClassSizesMax];
memset(class_count, 0, sizeof(class_count));
{
SpinLockHolder h(Static::pageheap_lock());
@@ -860,7 +985,7 @@
}
prev_class_size = 0;
- for (int cl = 1; cl < kNumClasses; ++cl) {
+ for (int cl = 1; cl < Static::num_size_classes(); ++cl) {
MallocExtension::FreeListInfo i;
i.min_object_size = prev_class_size + 1;
i.max_object_size = Static::sizemap()->ByteSizeForClass(cl);
@@ -868,6 +993,8 @@
class_count[cl] * Static::sizemap()->ByteSizeForClass(cl);
i.type = kThreadCacheType;
v->push_back(i);
+
+ prev_class_size = Static::sizemap()->ByteSizeForClass(cl);
}
// append page heap info
@@ -893,22 +1020,102 @@
v->push_back(span_info);
// small spans
- for (int s = 1; s < kMaxPages; s++) {
+ for (int s = 1; s <= kMaxPages; s++) {
MallocExtension::FreeListInfo i;
i.max_object_size = (s << kPageShift);
i.min_object_size = ((s - 1) << kPageShift);
i.type = kPageHeapType;
- i.total_bytes_free = (s << kPageShift) * small.normal_length[s];
+ i.total_bytes_free = (s << kPageShift) * small.normal_length[s - 1];
v->push_back(i);
i.type = kPageHeapUnmappedType;
- i.total_bytes_free = (s << kPageShift) * small.returned_length[s];
+ i.total_bytes_free = (s << kPageShift) * small.returned_length[s - 1];
v->push_back(i);
}
}
};
+static inline ATTRIBUTE_ALWAYS_INLINE
+size_t align_size_up(size_t size, size_t align) {
+ ASSERT(align <= kPageSize);
+ size_t new_size = (size + align - 1) & ~(align - 1);
+ if (PREDICT_FALSE(new_size == 0)) {
+ // Note, new_size == 0 catches both integer overflow and size
+ // being 0.
+ if (size == 0) {
+ new_size = align;
+ } else {
+ new_size = size;
+ }
+ }
+ return new_size;
+}
+
+// Puts in *cl size class that is suitable for allocation of size bytes with
+// align alignment. Returns true if such size class exists and false otherwise.
+static bool size_class_with_alignment(size_t size, size_t align, uint32_t* cl) {
+ if (PREDICT_FALSE(align > kPageSize)) {
+ return false;
+ }
+ size = align_size_up(size, align);
+ if (PREDICT_FALSE(!Static::sizemap()->GetSizeClass(size, cl))) {
+ return false;
+ }
+ ASSERT((Static::sizemap()->class_to_size(*cl) & (align - 1)) == 0);
+ return true;
+}
+
+// nallocx slow path. Moved to a separate function because
+// ThreadCache::InitModule is not inlined which would cause nallocx to
+// become non-leaf function with stack frame and stack spills.
+static ATTRIBUTE_NOINLINE size_t nallocx_slow(size_t size, int flags) {
+ if (PREDICT_FALSE(!Static::IsInited())) ThreadCache::InitModule();
+
+ size_t align = static_cast<size_t>(1ull << (flags & 0x3f));
+ uint32 cl;
+ bool ok = size_class_with_alignment(size, align, &cl);
+ if (ok) {
+ return Static::sizemap()->ByteSizeForClass(cl);
+ } else {
+ return tcmalloc::pages(size) << kPageShift;
+ }
+}
+
+// The nallocx function allocates no memory, but it performs the same size
+// computation as the malloc function, and returns the real size of the
+// allocation that would result from the equivalent malloc function call.
+// nallocx is a malloc extension originally implemented by jemalloc:
+// http://www.unix.com/man-page/freebsd/3/nallocx/
+extern "C" PERFTOOLS_DLL_DECL
+size_t tc_nallocx(size_t size, int flags) {
+ if (PREDICT_FALSE(flags != 0)) {
+ return nallocx_slow(size, flags);
+ }
+ uint32 cl;
+ // size class 0 is only possible if malloc is not yet initialized
+ if (Static::sizemap()->GetSizeClass(size, &cl) && cl != 0) {
+ return Static::sizemap()->ByteSizeForClass(cl);
+ } else {
+ return nallocx_slow(size, 0);
+ }
+}
+
+extern "C" PERFTOOLS_DLL_DECL
+size_t nallocx(size_t size, int flags)
+#ifdef TC_ALIAS
+ TC_ALIAS(tc_nallocx);
+#else
+{
+ return nallocx_slow(size, flags);
+}
+#endif
+
+
+size_t TCMallocImplementation::GetEstimatedAllocatedSize(size_t size) {
+ return tc_nallocx(size, 0);
+}
+
// The constructor allocates an object to ensure that initialization
// runs before main(), and therefore we do not have a chance to become
// multi-threaded before initialization. We also create the TSD key
@@ -966,23 +1173,26 @@
static inline bool CheckCachedSizeClass(void *ptr) {
PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift;
- size_t cached_value = Static::pageheap()->GetSizeClassIfCached(p);
- return cached_value == 0 ||
- cached_value == Static::pageheap()->GetDescriptor(p)->sizeclass;
+ uint32 cached_value;
+ if (!Static::pageheap()->TryGetSizeClass(p, &cached_value)) {
+ return true;
+ }
+ return cached_value == Static::pageheap()->GetDescriptor(p)->sizeclass;
}
-static inline void* CheckedMallocResult(void *result) {
+static inline ATTRIBUTE_ALWAYS_INLINE void* CheckedMallocResult(void *result) {
ASSERT(result == NULL || CheckCachedSizeClass(result));
return result;
}
-static inline void* SpanToMallocResult(Span *span) {
- Static::pageheap()->CacheSizeClass(span->start, 0);
+static inline ATTRIBUTE_ALWAYS_INLINE void* SpanToMallocResult(Span *span) {
+ Static::pageheap()->InvalidateCachedSizeClass(span->start);
return
CheckedMallocResult(reinterpret_cast<void*>(span->start << kPageShift));
}
static void* DoSampledAllocation(size_t size) {
+#ifndef NO_TCMALLOC_SAMPLES
// Grab the stack trace outside the heap lock
StackTrace tmp;
tmp.depth = GetStackTrace(tmp.stack, tcmalloc::kMaxStackDepth, 1);
@@ -991,13 +1201,13 @@
SpinLockHolder h(Static::pageheap_lock());
// Allocate span
Span *span = Static::pageheap()->New(tcmalloc::pages(size == 0 ? 1 : size));
- if (UNLIKELY(span == NULL)) {
+ if (PREDICT_FALSE(span == NULL)) {
return NULL;
}
// Allocate stack trace
StackTrace *stack = Static::stacktrace_allocator()->New();
- if (UNLIKELY(stack == NULL)) {
+ if (PREDICT_FALSE(stack == NULL)) {
// Sampling failed because of lack of memory
return span;
}
@@ -1007,6 +1217,9 @@
tcmalloc::DLL_Prepend(Static::sampled_objects(), span);
return SpanToMallocResult(span);
+#else
+ abort();
+#endif
}
namespace {
@@ -1019,6 +1232,16 @@
void* retry_arg,
bool from_operator,
bool nothrow) {
+ // we hit out of memory condition, usually if it happens we've
+ // called sbrk or mmap and failed, and thus errno is set. But there
+ // is support for setting up custom system allocator or setting up
+ // page heap size limit, in which cases errno may remain
+ // untouched.
+ //
+ // So we set errno here. C++ operator new doesn't require ENOMEM to
+ // be set, but doesn't forbid it too (and often C++ oom does happen
+ // with ENOMEM set).
+ errno = ENOMEM;
if (!from_operator && !tc_new_mode) {
// we're out of memory in C library function (malloc etc) and no
// "new mode" forced on us. Just return NULL
@@ -1077,9 +1300,11 @@
// Copy of FLAGS_tcmalloc_large_alloc_report_threshold with
// automatic increases factored in.
+#ifdef ENABLE_LARGE_ALLOC_REPORT
static int64_t large_alloc_threshold =
(kPageSize > FLAGS_tcmalloc_large_alloc_report_threshold
? kPageSize : FLAGS_tcmalloc_large_alloc_report_threshold);
+#endif
static void ReportLargeAlloc(Length num_pages, void* result) {
StackTrace stack;
@@ -1098,36 +1323,9 @@
write(STDERR_FILENO, buffer, strlen(buffer));
}
-void* do_memalign(size_t align, size_t size);
-
-struct retry_memaligh_data {
- size_t align;
- size_t size;
-};
-
-static void *retry_do_memalign(void *arg) {
- retry_memaligh_data *data = static_cast<retry_memaligh_data *>(arg);
- return do_memalign(data->align, data->size);
-}
-
-static void *maybe_do_cpp_memalign_slow(size_t align, size_t size) {
- retry_memaligh_data data;
- data.align = align;
- data.size = size;
- return handle_oom(retry_do_memalign, &data,
- false, true);
-}
-
-inline void* do_memalign_or_cpp_memalign(size_t align, size_t size) {
- void *rv = do_memalign(align, size);
- if (LIKELY(rv != NULL)) {
- return rv;
- }
- return maybe_do_cpp_memalign_slow(align, size);
-}
-
// Must be called with the page lock held.
inline bool should_report_large(Length num_pages) {
+#ifdef ENABLE_LARGE_ALLOC_REPORT
const int64 threshold = large_alloc_threshold;
if (threshold > 0 && num_pages >= (threshold >> kPageShift)) {
// Increase the threshold by 1/8 every time we generate a report.
@@ -1136,18 +1334,24 @@
? threshold + threshold/8 : 8ll<<30);
return true;
}
+#endif
return false;
}
// Helper for do_malloc().
-inline void* do_malloc_pages(ThreadCache* heap, size_t size) {
+static void* do_malloc_pages(ThreadCache* heap, size_t size) {
void* result;
bool report_large;
Length num_pages = tcmalloc::pages(size);
- size = num_pages << kPageShift;
- if ((FLAGS_tcmalloc_sample_parameter > 0) && heap->SampleAllocation(size)) {
+ // NOTE: we're passing original size here as opposed to rounded-up
+ // size as we do in do_malloc_small. The difference is small here
+ // (at most 4k out of at least 256k). And not rounding up saves us
+ // from possibility of overflow, which rounding up could produce.
+ //
+ // See https://github.com/gperftools/gperftools/issues/723
+ if (heap->SampleAllocation(size)) {
result = DoSampledAllocation(size);
SpinLockHolder h(Static::pageheap_lock());
@@ -1155,7 +1359,7 @@
} else {
SpinLockHolder h(Static::pageheap_lock());
Span* span = Static::pageheap()->New(num_pages);
- result = (UNLIKELY(span == NULL) ? NULL : SpanToMallocResult(span));
+ result = (PREDICT_FALSE(span == NULL) ? NULL : SpanToMallocResult(span));
report_large = should_report_large(num_pages);
}
@@ -1165,53 +1369,57 @@
return result;
}
-ALWAYS_INLINE void* do_malloc_small(ThreadCache* heap, size_t size) {
- ASSERT(Static::IsInited());
- ASSERT(heap != NULL);
- size_t cl = Static::sizemap()->SizeClass(size);
- size = Static::sizemap()->class_to_size(cl);
-
- if (UNLIKELY(FLAGS_tcmalloc_sample_parameter > 0) && heap->SampleAllocation(size)) {
- return DoSampledAllocation(size);
- } else {
- // The common case, and also the simplest. This just pops the
- // size-appropriate freelist, after replenishing it if it's empty.
- return CheckedMallocResult(heap->Allocate(size, cl));
- }
+static void *nop_oom_handler(size_t size) {
+ return NULL;
}
-ALWAYS_INLINE void* do_malloc(size_t size) {
- if (ThreadCache::have_tls &&
- LIKELY(size < ThreadCache::MinSizeForSlowPath())) {
- return do_malloc_small(ThreadCache::GetCacheWhichMustBePresent(), size);
- } else if (size <= kMaxSize) {
- return do_malloc_small(ThreadCache::GetCache(), size);
- } else {
- return do_malloc_pages(ThreadCache::GetCache(), size);
+ATTRIBUTE_ALWAYS_INLINE inline void* do_malloc(size_t size) {
+ if (PREDICT_FALSE(ThreadCache::IsUseEmergencyMalloc())) {
+ return tcmalloc::EmergencyMalloc(size);
}
+
+ // note: it will force initialization of malloc if necessary
+ ThreadCache* cache = ThreadCache::GetCache();
+ uint32 cl;
+
+ ASSERT(Static::IsInited());
+ ASSERT(cache != NULL);
+
+ if (PREDICT_FALSE(!Static::sizemap()->GetSizeClass(size, &cl))) {
+ return do_malloc_pages(cache, size);
+ }
+
+ size_t allocated_size = Static::sizemap()->class_to_size(cl);
+ if (PREDICT_FALSE(cache->SampleAllocation(allocated_size))) {
+ return DoSampledAllocation(size);
+ }
+
+ // The common case, and also the simplest. This just pops the
+ // size-appropriate freelist, after replenishing it if it's empty.
+ return CheckedMallocResult(cache->Allocate(allocated_size, cl, nop_oom_handler));
}
static void *retry_malloc(void* size) {
return do_malloc(reinterpret_cast<size_t>(size));
}
-ALWAYS_INLINE void* do_malloc_or_cpp_alloc(size_t size) {
+ATTRIBUTE_ALWAYS_INLINE inline void* do_malloc_or_cpp_alloc(size_t size) {
void *rv = do_malloc(size);
- if (LIKELY(rv != NULL)) {
+ if (PREDICT_TRUE(rv != NULL)) {
return rv;
}
return handle_oom(retry_malloc, reinterpret_cast<void *>(size),
false, true);
}
-ALWAYS_INLINE void* do_calloc(size_t n, size_t elem_size) {
+ATTRIBUTE_ALWAYS_INLINE inline void* do_calloc(size_t n, size_t elem_size) {
// Overflow check
const size_t size = n * elem_size;
if (elem_size != 0 && size / elem_size != n) return NULL;
void* result = do_malloc_or_cpp_alloc(size);
if (result != NULL) {
- memset(result, 0, size);
+ memset(result, 0, tc_nallocx(size, 0));
}
return result;
}
@@ -1223,97 +1431,105 @@
}
}
-// Helper for do_free_with_callback(), below. Inputs:
-// ptr is object to be freed
-// invalid_free_fn is a function that gets invoked on certain "bad frees"
-// heap is the ThreadCache for this thread, or NULL if it isn't known
-// heap_must_be_valid is whether heap is known to be non-NULL
-//
-// This function may only be used after Static::IsInited() is true.
-//
-// We can usually detect the case where ptr is not pointing to a page that
-// tcmalloc is using, and in those cases we invoke invalid_free_fn.
-//
-// To maximize speed in the common case, we usually get here with
-// heap_must_be_valid being a manifest constant equal to true.
-ALWAYS_INLINE void do_free_helper(void* ptr,
- void (*invalid_free_fn)(void*),
- ThreadCache* heap,
- bool heap_must_be_valid) {
- ASSERT((Static::IsInited() && heap != NULL) || !heap_must_be_valid);
- if (!heap_must_be_valid && !Static::IsInited()) {
- // We called free() before malloc(). This can occur if the
- // (system) malloc() is called before tcmalloc is loaded, and then
- // free() is called after tcmalloc is loaded (and tc_free has
- // replaced free), but before the global constructor has run that
- // sets up the tcmalloc data structures.
- free_null_or_invalid(ptr, invalid_free_fn);
- return;
+static ATTRIBUTE_NOINLINE void do_free_pages(Span* span, void* ptr) {
+ SpinLockHolder h(Static::pageheap_lock());
+ if (span->sample) {
+ StackTrace* st = reinterpret_cast<StackTrace*>(span->objects);
+ tcmalloc::DLL_Remove(span);
+ Static::stacktrace_allocator()->Delete(st);
+ span->objects = NULL;
}
- Span* span = NULL;
- const PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift;
- size_t cl = Static::pageheap()->GetSizeClassIfCached(p);
- if (UNLIKELY(cl == 0)) {
- span = Static::pageheap()->GetDescriptor(p);
- if (UNLIKELY(!span)) {
- // span can be NULL because the pointer passed in is NULL or invalid
- // (not something returned by malloc or friends), or because the
- // pointer was allocated with some other allocator besides
- // tcmalloc. The latter can happen if tcmalloc is linked in via
- // a dynamic library, but is not listed last on the link line.
- // In that case, libraries after it on the link line will
- // allocate with libc malloc, but free with tcmalloc's free.
- free_null_or_invalid(ptr, invalid_free_fn);
- return;
- }
- cl = span->sizeclass;
- Static::pageheap()->CacheSizeClass(p, cl);
- }
- ASSERT(ptr != NULL);
- if (LIKELY(cl != 0)) {
- ASSERT(!Static::pageheap()->GetDescriptor(p)->sample);
- if (heap_must_be_valid || heap != NULL) {
- heap->Deallocate(ptr, cl);
- } else {
- // Delete directly into central cache
- tcmalloc::SLL_SetNext(ptr, NULL);
- Static::central_cache()[cl].InsertRange(ptr, ptr, 1);
- }
- } else {
- SpinLockHolder h(Static::pageheap_lock());
- ASSERT(reinterpret_cast<uintptr_t>(ptr) % kPageSize == 0);
- ASSERT(span != NULL && span->start == p);
- if (span->sample) {
- StackTrace* st = reinterpret_cast<StackTrace*>(span->objects);
- tcmalloc::DLL_Remove(span);
- Static::stacktrace_allocator()->Delete(st);
- span->objects = NULL;
- }
- Static::pageheap()->Delete(span);
- }
+ Static::pageheap()->Delete(span);
}
+#ifndef NDEBUG
+// note, with sized deletions we have no means to support win32
+// behavior where we detect "not ours" points and delegate them native
+// memory management. This is because nature of sized deletes
+// bypassing addr -> size class checks. So in this validation code we
+// also assume that sized delete is always used with "our" pointers.
+bool ValidateSizeHint(void* ptr, size_t size_hint) {
+ const PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift;
+ Span* span = Static::pageheap()->GetDescriptor(p);
+ uint32 cl = 0;
+ Static::sizemap()->GetSizeClass(size_hint, &cl);
+ return (span->sizeclass == cl);
+}
+#endif
+
// Helper for the object deletion (free, delete, etc.). Inputs:
// ptr is object to be freed
// invalid_free_fn is a function that gets invoked on certain "bad frees"
//
// We can usually detect the case where ptr is not pointing to a page that
// tcmalloc is using, and in those cases we invoke invalid_free_fn.
-ALWAYS_INLINE void do_free_with_callback(void* ptr,
- void (*invalid_free_fn)(void*)) {
- ThreadCache* heap = NULL;
- if (LIKELY(ThreadCache::IsFastPathAllowed())) {
- heap = ThreadCache::GetCacheWhichMustBePresent();
- do_free_helper(ptr, invalid_free_fn, heap, true);
- } else {
- heap = ThreadCache::GetCacheIfPresent();
- do_free_helper(ptr, invalid_free_fn, heap, false);
+ATTRIBUTE_ALWAYS_INLINE inline
+void do_free_with_callback(void* ptr,
+ void (*invalid_free_fn)(void*),
+ bool use_hint, size_t size_hint) {
+ ThreadCache* heap = ThreadCache::GetCacheIfPresent();
+
+ const PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift;
+ uint32 cl;
+
+ ASSERT(!use_hint || ValidateSizeHint(ptr, size_hint));
+
+ if (!use_hint || PREDICT_FALSE(!Static::sizemap()->GetSizeClass(size_hint, &cl))) {
+ // if we're in sized delete, but size is too large, no need to
+ // probe size cache
+ bool cache_hit = !use_hint && Static::pageheap()->TryGetSizeClass(p, &cl);
+ if (PREDICT_FALSE(!cache_hit)) {
+ Span* span = Static::pageheap()->GetDescriptor(p);
+ if (PREDICT_FALSE(!span)) {
+ // span can be NULL because the pointer passed in is NULL or invalid
+ // (not something returned by malloc or friends), or because the
+ // pointer was allocated with some other allocator besides
+ // tcmalloc. The latter can happen if tcmalloc is linked in via
+ // a dynamic library, but is not listed last on the link line.
+ // In that case, libraries after it on the link line will
+ // allocate with libc malloc, but free with tcmalloc's free.
+ free_null_or_invalid(ptr, invalid_free_fn);
+ return;
+ }
+ cl = span->sizeclass;
+ if (PREDICT_FALSE(cl == 0)) {
+ ASSERT(reinterpret_cast<uintptr_t>(ptr) % kPageSize == 0);
+ ASSERT(span != NULL && span->start == p);
+ do_free_pages(span, ptr);
+ return;
+ }
+ if (!use_hint) {
+ Static::pageheap()->SetCachedSizeClass(p, cl);
+ }
+ }
}
+
+ if (PREDICT_TRUE(heap != NULL)) {
+ ASSERT(Static::IsInited());
+ // If we've hit initialized thread cache, so we're done.
+ heap->Deallocate(ptr, cl);
+ return;
+ }
+
+ if (PREDICT_FALSE(!Static::IsInited())) {
+ // if free was called very early we've could have missed the case
+ // of invalid or nullptr free. I.e. because probing size classes
+ // cache could return bogus result (cl = 0 as of this
+ // writing). But since there is no way we could be dealing with
+ // ptr we've allocated, since successfull malloc implies IsInited,
+ // we can just call "invalid free" handling code.
+ free_null_or_invalid(ptr, invalid_free_fn);
+ return;
+ }
+
+ // Otherwise, delete directly into central cache
+ tcmalloc::SLL_SetNext(ptr, NULL);
+ Static::central_cache()[cl].InsertRange(ptr, ptr, 1);
}
// The default "do_free" that uses the default callback.
-ALWAYS_INLINE void do_free(void* ptr) {
- return do_free_with_callback(ptr, &InvalidFree);
+ATTRIBUTE_ALWAYS_INLINE inline void do_free(void* ptr) {
+ return do_free_with_callback(ptr, &InvalidFree, false, 0);
}
// NOTE: some logic here is duplicated in GetOwnership (above), for
@@ -1323,25 +1539,31 @@
if (ptr == NULL)
return 0;
const PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift;
- size_t cl = Static::pageheap()->GetSizeClassIfCached(p);
- if (cl != 0) {
+ uint32 cl;
+ if (Static::pageheap()->TryGetSizeClass(p, &cl)) {
return Static::sizemap()->ByteSizeForClass(cl);
- } else {
- const Span *span = Static::pageheap()->GetDescriptor(p);
- if (UNLIKELY(span == NULL)) { // means we do not own this memory
- return (*invalid_getsize_fn)(ptr);
- } else if (span->sizeclass != 0) {
- Static::pageheap()->CacheSizeClass(p, span->sizeclass);
- return Static::sizemap()->ByteSizeForClass(span->sizeclass);
- } else {
- return span->length << kPageShift;
- }
}
+
+ const Span *span = Static::pageheap()->GetDescriptor(p);
+ if (PREDICT_FALSE(span == NULL)) { // means we do not own this memory
+ return (*invalid_getsize_fn)(ptr);
+ }
+
+ if (span->sizeclass != 0) {
+ return Static::sizemap()->ByteSizeForClass(span->sizeclass);
+ }
+
+ if (span->sample) {
+ size_t orig_size = reinterpret_cast<StackTrace*>(span->objects)->size;
+ return tc_nallocx(orig_size, 0);
+ }
+
+ return span->length << kPageShift;
}
// This lets you call back to a given function pointer if ptr is invalid.
// It is used primarily by windows code which wants a specialized callback.
-ALWAYS_INLINE void* do_realloc_with_callback(
+ATTRIBUTE_ALWAYS_INLINE inline void* do_realloc_with_callback(
void* old_ptr, size_t new_size,
void (*invalid_free_fn)(void*),
size_t (*invalid_get_size_fn)(const void*)) {
@@ -1354,7 +1576,9 @@
// . If we need to grow, grow to max(new_size, old_size * 1.X)
// . Don't shrink unless new_size < old_size * 0.Y
// X and Y trade-off time for wasted space. For now we do 1.25 and 0.5.
- const size_t lower_bound_to_grow = old_size + old_size / 4ul;
+ const size_t min_growth = min(old_size / 4,
+ (std::numeric_limits<size_t>::max)() - old_size); // Avoid overflow.
+ const size_t lower_bound_to_grow = old_size + min_growth;
const size_t upper_bound_to_shrink = old_size / 2ul;
if ((new_size > old_size) || (new_size < upper_bound_to_shrink)) {
// Need to reallocate.
@@ -1367,7 +1591,7 @@
// Either new_size is not a tiny increment, or last do_malloc failed.
new_ptr = do_malloc_or_cpp_alloc(new_size);
}
- if (UNLIKELY(new_ptr == NULL)) {
+ if (PREDICT_FALSE(new_ptr == NULL)) {
return NULL;
}
MallocHook::InvokeNewHook(new_ptr, new_size);
@@ -1376,7 +1600,7 @@
// We could use a variant of do_free() that leverages the fact
// that we already know the sizeclass of old_ptr. The benefit
// would be small, so don't bother.
- do_free_with_callback(old_ptr, invalid_free_fn);
+ do_free_with_callback(old_ptr, invalid_free_fn, false, 0);
return new_ptr;
} else {
// We still need to call hooks to report the updated size:
@@ -1386,69 +1610,29 @@
}
}
-ALWAYS_INLINE void* do_realloc(void* old_ptr, size_t new_size) {
+ATTRIBUTE_ALWAYS_INLINE inline void* do_realloc(void* old_ptr, size_t new_size) {
return do_realloc_with_callback(old_ptr, new_size,
&InvalidFree, &InvalidGetSizeForRealloc);
}
-// For use by exported routines below that want specific alignments
-//
-// Note: this code can be slow for alignments > 16, and can
-// significantly fragment memory. The expectation is that
-// memalign/posix_memalign/valloc/pvalloc will not be invoked very
-// often. This requirement simplifies our implementation and allows
-// us to tune for expected allocation patterns.
-void* do_memalign(size_t align, size_t size) {
+static ATTRIBUTE_ALWAYS_INLINE inline
+void* do_memalign_pages(size_t align, size_t size) {
ASSERT((align & (align - 1)) == 0);
- ASSERT(align > 0);
+ ASSERT(align > kPageSize);
if (size + align < size) return NULL; // Overflow
- // Fall back to malloc if we would already align this memory access properly.
- if (align <= AlignmentForSize(size)) {
- void* p = do_malloc(size);
- ASSERT((reinterpret_cast<uintptr_t>(p) % align) == 0);
- return p;
- }
-
- if (UNLIKELY(Static::pageheap() == NULL)) ThreadCache::InitModule();
+ if (PREDICT_FALSE(Static::pageheap() == NULL)) ThreadCache::InitModule();
// Allocate at least one byte to avoid boundary conditions below
if (size == 0) size = 1;
- if (size <= kMaxSize && align < kPageSize) {
- // Search through acceptable size classes looking for one with
- // enough alignment. This depends on the fact that
- // InitSizeClasses() currently produces several size classes that
- // are aligned at powers of two. We will waste time and space if
- // we miss in the size class array, but that is deemed acceptable
- // since memalign() should be used rarely.
- int cl = Static::sizemap()->SizeClass(size);
- while (cl < kNumClasses &&
- ((Static::sizemap()->class_to_size(cl) & (align - 1)) != 0)) {
- cl++;
- }
- if (cl < kNumClasses) {
- ThreadCache* heap = ThreadCache::GetCache();
- size = Static::sizemap()->class_to_size(cl);
- return CheckedMallocResult(heap->Allocate(size, cl));
- }
- }
-
// We will allocate directly from the page heap
SpinLockHolder h(Static::pageheap_lock());
- if (align <= kPageSize) {
- // Any page-level allocation will be fine
- // TODO: We could put the rest of this page in the appropriate
- // TODO: cache but it does not seem worth it.
- Span* span = Static::pageheap()->New(tcmalloc::pages(size));
- return UNLIKELY(span == NULL) ? NULL : SpanToMallocResult(span);
- }
-
// Allocate extra pages and carve off an aligned portion
const Length alloc = tcmalloc::pages(size + align);
Span* span = Static::pageheap()->New(alloc);
- if (UNLIKELY(span == NULL)) return NULL;
+ if (PREDICT_FALSE(span == NULL)) return NULL;
// Skip starting portion so that we end up aligned
Length skip = 0;
@@ -1510,15 +1694,6 @@
}
#endif // HAVE_STRUCT_MALLINFO
-inline void* cpp_alloc(size_t size, bool nothrow) {
- void* p = do_malloc(size);
- if (LIKELY(p)) {
- return p;
- }
- return handle_oom(retry_malloc, reinterpret_cast<void *>(size),
- true, nothrow);
-}
-
} // end unnamed namespace
// As promised, the definition of this function, declared above.
@@ -1541,7 +1716,7 @@
//-------------------------------------------------------------------
extern "C" PERFTOOLS_DLL_DECL const char* tc_version(
- int* major, int* minor, const char** patch) __THROW {
+ int* major, int* minor, const char** patch) PERFTOOLS_NOTHROW {
if (major) *major = TC_VERSION_MAJOR;
if (minor) *minor = TC_VERSION_MINOR;
if (patch) *patch = TC_VERSION_PATCH;
@@ -1553,12 +1728,16 @@
// If flag is 1, calls to malloc will behave like calls to new,
// and the std_new_handler will be invoked on failure.
// Returns the previous mode.
-extern "C" PERFTOOLS_DLL_DECL int tc_set_new_mode(int flag) __THROW {
+extern "C" PERFTOOLS_DLL_DECL int tc_set_new_mode(int flag) PERFTOOLS_NOTHROW {
int old_mode = tc_new_mode;
tc_new_mode = flag;
return old_mode;
}
+extern "C" PERFTOOLS_DLL_DECL int tc_query_new_mode() PERFTOOLS_NOTHROW {
+ return tc_new_mode;
+}
+
#ifndef TCMALLOC_USING_DEBUGALLOCATION // debugallocation.cc defines its own
// CAVEAT: The code structure below ensures that MallocHook methods are always
@@ -1566,31 +1745,259 @@
// heap-checker.cc depends on this to start a stack trace from
// the call to the (de)allocation function.
-extern "C" PERFTOOLS_DLL_DECL void* tc_malloc(size_t size) __THROW {
- void* result = do_malloc_or_cpp_alloc(size);
- MallocHook::InvokeNewHook(result, size);
- return result;
-}
+namespace tcmalloc {
-extern "C" PERFTOOLS_DLL_DECL void tc_free(void* ptr) __THROW {
+
+static ATTRIBUTE_SECTION(google_malloc)
+void invoke_hooks_and_free(void *ptr) {
MallocHook::InvokeDeleteHook(ptr);
do_free(ptr);
}
+ATTRIBUTE_SECTION(google_malloc)
+void* cpp_throw_oom(size_t size) {
+ return handle_oom(retry_malloc, reinterpret_cast<void *>(size),
+ true, false);
+}
+
+ATTRIBUTE_SECTION(google_malloc)
+void* cpp_nothrow_oom(size_t size) {
+ return handle_oom(retry_malloc, reinterpret_cast<void *>(size),
+ true, true);
+}
+
+ATTRIBUTE_SECTION(google_malloc)
+void* malloc_oom(size_t size) {
+ return handle_oom(retry_malloc, reinterpret_cast<void *>(size),
+ false, true);
+}
+
+// tcmalloc::allocate_full_XXX is called by fast-path malloc when some
+// complex handling is needed (such as fetching object from central
+// freelist or malloc sampling). It contains all 'operator new' logic,
+// as opposed to malloc_fast_path which only deals with important
+// subset of cases.
+//
+// Note that this is under tcmalloc namespace so that pprof
+// can automatically filter it out of growthz/heapz profiles.
+//
+// We have slightly fancy setup because we need to call hooks from
+// function in 'google_malloc' section and we cannot place template
+// into this section. Thus 3 separate functions 'built' by macros.
+//
+// Also note that we're carefully orchestrating for
+// MallocHook::GetCallerStackTrace to work even if compiler isn't
+// optimizing tail calls (e.g. -O0 is given). We still require
+// ATTRIBUTE_ALWAYS_INLINE to work for that case, but it was seen to
+// work for -O0 -fno-inline across both GCC and clang. I.e. in this
+// case we'll get stack frame for tc_new, followed by stack frame for
+// allocate_full_cpp_throw_oom, followed by hooks machinery and user
+// code's stack frames. So GetCallerStackTrace will find 2
+// subsequent stack frames in google_malloc section and correctly
+// 'cut' stack trace just before tc_new.
+template <void* OOMHandler(size_t)>
+ATTRIBUTE_ALWAYS_INLINE inline
+static void* do_allocate_full(size_t size) {
+ void* p = do_malloc(size);
+ if (PREDICT_FALSE(p == NULL)) {
+ p = OOMHandler(size);
+ }
+ MallocHook::InvokeNewHook(p, size);
+ return CheckedMallocResult(p);
+}
+
+#define AF(oom) \
+ ATTRIBUTE_SECTION(google_malloc) \
+ void* allocate_full_##oom(size_t size) { \
+ return do_allocate_full<oom>(size); \
+ }
+
+AF(cpp_throw_oom)
+AF(cpp_nothrow_oom)
+AF(malloc_oom)
+
+#undef AF
+
+template <void* OOMHandler(size_t)>
+static ATTRIBUTE_ALWAYS_INLINE inline void* dispatch_allocate_full(size_t size) {
+ if (OOMHandler == cpp_throw_oom) {
+ return allocate_full_cpp_throw_oom(size);
+ }
+ if (OOMHandler == cpp_nothrow_oom) {
+ return allocate_full_cpp_nothrow_oom(size);
+ }
+ ASSERT(OOMHandler == malloc_oom);
+ return allocate_full_malloc_oom(size);
+}
+
+struct retry_memalign_data {
+ size_t align;
+ size_t size;
+};
+
+static void *retry_do_memalign(void *arg) {
+ retry_memalign_data *data = static_cast<retry_memalign_data *>(arg);
+ return do_memalign_pages(data->align, data->size);
+}
+
+static ATTRIBUTE_SECTION(google_malloc)
+void* memalign_pages(size_t align, size_t size,
+ bool from_operator, bool nothrow) {
+ void *rv = do_memalign_pages(align, size);
+ if (PREDICT_FALSE(rv == NULL)) {
+ retry_memalign_data data;
+ data.align = align;
+ data.size = size;
+ rv = handle_oom(retry_do_memalign, &data,
+ from_operator, nothrow);
+ }
+ MallocHook::InvokeNewHook(rv, size);
+ return CheckedMallocResult(rv);
+}
+
+} // namespace tcmalloc
+
+// This is quick, fast-path-only implementation of malloc/new. It is
+// designed to only have support for fast-path. It checks if more
+// complex handling is needed (such as a pageheap allocation or
+// sampling) and only performs allocation if none of those uncommon
+// conditions hold. When we have one of those odd cases it simply
+// tail-calls to one of tcmalloc::allocate_full_XXX defined above.
+//
+// Such approach was found to be quite effective. Generated code for
+// tc_{new,malloc} either succeeds quickly or tail-calls to
+// allocate_full. Terseness of the source and lack of
+// non-tail calls enables compiler to produce better code. Also
+// produced code is short enough to enable effort-less human
+// comprehension. Which itself led to elimination of various checks
+// that were not necessary for fast-path.
+template <void* OOMHandler(size_t)>
+ATTRIBUTE_ALWAYS_INLINE inline
+static void * malloc_fast_path(size_t size) {
+ if (PREDICT_FALSE(!base::internal::new_hooks_.empty())) {
+ return tcmalloc::dispatch_allocate_full<OOMHandler>(size);
+ }
+
+ ThreadCache *cache = ThreadCache::GetFastPathCache();
+
+ if (PREDICT_FALSE(cache == NULL)) {
+ return tcmalloc::dispatch_allocate_full<OOMHandler>(size);
+ }
+
+ uint32 cl;
+ if (PREDICT_FALSE(!Static::sizemap()->GetSizeClass(size, &cl))) {
+ return tcmalloc::dispatch_allocate_full<OOMHandler>(size);
+ }
+
+ size_t allocated_size = Static::sizemap()->ByteSizeForClass(cl);
+
+ if (PREDICT_FALSE(!cache->TryRecordAllocationFast(allocated_size))) {
+ return tcmalloc::dispatch_allocate_full<OOMHandler>(size);
+ }
+
+ return CheckedMallocResult(cache->Allocate(allocated_size, cl, OOMHandler));
+}
+
+template <void* OOMHandler(size_t)>
+ATTRIBUTE_ALWAYS_INLINE inline
+static void* memalign_fast_path(size_t align, size_t size) {
+ if (PREDICT_FALSE(align > kPageSize)) {
+ if (OOMHandler == tcmalloc::cpp_throw_oom) {
+ return tcmalloc::memalign_pages(align, size, true, false);
+ } else if (OOMHandler == tcmalloc::cpp_nothrow_oom) {
+ return tcmalloc::memalign_pages(align, size, true, true);
+ } else {
+ ASSERT(OOMHandler == tcmalloc::malloc_oom);
+ return tcmalloc::memalign_pages(align, size, false, true);
+ }
+ }
+
+ // Everything with alignment <= kPageSize we can easily delegate to
+ // regular malloc
+
+ return malloc_fast_path<OOMHandler>(align_size_up(size, align));
+}
+
+extern "C" PERFTOOLS_DLL_DECL CACHELINE_ALIGNED_FN
+void* tc_malloc(size_t size) PERFTOOLS_NOTHROW {
+ return malloc_fast_path<tcmalloc::malloc_oom>(size);
+}
+
+static ATTRIBUTE_ALWAYS_INLINE inline
+void free_fast_path(void *ptr) {
+ if (PREDICT_FALSE(!base::internal::delete_hooks_.empty())) {
+ tcmalloc::invoke_hooks_and_free(ptr);
+ return;
+ }
+ do_free(ptr);
+}
+
+extern "C" PERFTOOLS_DLL_DECL CACHELINE_ALIGNED_FN
+void tc_free(void* ptr) PERFTOOLS_NOTHROW {
+ free_fast_path(ptr);
+}
+
+extern "C" PERFTOOLS_DLL_DECL CACHELINE_ALIGNED_FN
+void tc_free_sized(void *ptr, size_t size) PERFTOOLS_NOTHROW {
+ if (PREDICT_FALSE(!base::internal::delete_hooks_.empty())) {
+ tcmalloc::invoke_hooks_and_free(ptr);
+ return;
+ }
+#ifndef NO_TCMALLOC_SAMPLES
+ // if ptr is kPageSize-aligned, then it could be sampled allocation,
+ // thus we don't trust hint and just do plain free. It also handles
+ // nullptr for us.
+ if (PREDICT_FALSE((reinterpret_cast<uintptr_t>(ptr) & (kPageSize-1)) == 0)) {
+ tc_free(ptr);
+ return;
+ }
+#else
+ if (!ptr) {
+ return;
+ }
+#endif
+ do_free_with_callback(ptr, &InvalidFree, true, size);
+}
+
+#ifdef TC_ALIAS
+
+extern "C" PERFTOOLS_DLL_DECL void tc_delete_sized(void *p, size_t size) PERFTOOLS_NOTHROW
+ TC_ALIAS(tc_free_sized);
+extern "C" PERFTOOLS_DLL_DECL void tc_deletearray_sized(void *p, size_t size) PERFTOOLS_NOTHROW
+ TC_ALIAS(tc_free_sized);
+
+#else
+
+extern "C" PERFTOOLS_DLL_DECL void tc_delete_sized(void *p, size_t size) PERFTOOLS_NOTHROW {
+ tc_free_sized(p, size);
+}
+extern "C" PERFTOOLS_DLL_DECL void tc_deletearray_sized(void *p, size_t size) PERFTOOLS_NOTHROW {
+ tc_free_sized(p, size);
+}
+
+#endif
+
extern "C" PERFTOOLS_DLL_DECL void* tc_calloc(size_t n,
- size_t elem_size) __THROW {
+ size_t elem_size) PERFTOOLS_NOTHROW {
+ if (ThreadCache::IsUseEmergencyMalloc()) {
+ return tcmalloc::EmergencyCalloc(n, elem_size);
+ }
void* result = do_calloc(n, elem_size);
MallocHook::InvokeNewHook(result, n * elem_size);
return result;
}
-extern "C" PERFTOOLS_DLL_DECL void tc_cfree(void* ptr) __THROW {
- MallocHook::InvokeDeleteHook(ptr);
- do_free(ptr);
+extern "C" PERFTOOLS_DLL_DECL void tc_cfree(void* ptr) PERFTOOLS_NOTHROW
+#ifdef TC_ALIAS
+TC_ALIAS(tc_free);
+#else
+{
+ free_fast_path(ptr);
}
+#endif
extern "C" PERFTOOLS_DLL_DECL void* tc_realloc(void* old_ptr,
- size_t new_size) __THROW {
+ size_t new_size) PERFTOOLS_NOTHROW {
if (old_ptr == NULL) {
void* result = do_malloc_or_cpp_alloc(new_size);
MallocHook::InvokeNewHook(result, new_size);
@@ -1601,85 +2008,95 @@
do_free(old_ptr);
return NULL;
}
+ if (PREDICT_FALSE(tcmalloc::IsEmergencyPtr(old_ptr))) {
+ return tcmalloc::EmergencyRealloc(old_ptr, new_size);
+ }
return do_realloc(old_ptr, new_size);
}
-extern "C" PERFTOOLS_DLL_DECL void* tc_new(size_t size) {
- void* p = cpp_alloc(size, false);
- // We keep this next instruction out of cpp_alloc for a reason: when
- // it's in, and new just calls cpp_alloc, the optimizer may fold the
- // new call into cpp_alloc, which messes up our whole section-based
- // stacktracing (see ATTRIBUTE_SECTION, above). This ensures cpp_alloc
- // isn't the last thing this fn calls, and prevents the folding.
- MallocHook::InvokeNewHook(p, size);
- return p;
+extern "C" PERFTOOLS_DLL_DECL CACHELINE_ALIGNED_FN
+void* tc_new(size_t size) {
+ return malloc_fast_path<tcmalloc::cpp_throw_oom>(size);
}
-extern "C" PERFTOOLS_DLL_DECL void* tc_new_nothrow(size_t size, const std::nothrow_t&) __THROW {
- void* p = cpp_alloc(size, true);
- MallocHook::InvokeNewHook(p, size);
- return p;
+extern "C" PERFTOOLS_DLL_DECL CACHELINE_ALIGNED_FN
+void* tc_new_nothrow(size_t size, const std::nothrow_t&) PERFTOOLS_NOTHROW {
+ return malloc_fast_path<tcmalloc::cpp_nothrow_oom>(size);
}
-extern "C" PERFTOOLS_DLL_DECL void tc_delete(void* p) __THROW {
- MallocHook::InvokeDeleteHook(p);
- do_free(p);
+extern "C" PERFTOOLS_DLL_DECL void tc_delete(void* p) PERFTOOLS_NOTHROW
+#ifdef TC_ALIAS
+TC_ALIAS(tc_free);
+#else
+{
+ free_fast_path(p);
}
+#endif
// Standard C++ library implementations define and use this
// (via ::operator delete(ptr, nothrow)).
// But it's really the same as normal delete, so we just do the same thing.
-extern "C" PERFTOOLS_DLL_DECL void tc_delete_nothrow(void* p, const std::nothrow_t&) __THROW {
- MallocHook::InvokeDeleteHook(p);
+extern "C" PERFTOOLS_DLL_DECL void tc_delete_nothrow(void* p, const std::nothrow_t&) PERFTOOLS_NOTHROW
+{
+ if (PREDICT_FALSE(!base::internal::delete_hooks_.empty())) {
+ tcmalloc::invoke_hooks_and_free(p);
+ return;
+ }
do_free(p);
}
-extern "C" PERFTOOLS_DLL_DECL void* tc_newarray(size_t size) {
- void* p = cpp_alloc(size, false);
- // We keep this next instruction out of cpp_alloc for a reason: when
- // it's in, and new just calls cpp_alloc, the optimizer may fold the
- // new call into cpp_alloc, which messes up our whole section-based
- // stacktracing (see ATTRIBUTE_SECTION, above). This ensures cpp_alloc
- // isn't the last thing this fn calls, and prevents the folding.
- MallocHook::InvokeNewHook(p, size);
- return p;
+extern "C" PERFTOOLS_DLL_DECL void* tc_newarray(size_t size)
+#ifdef TC_ALIAS
+TC_ALIAS(tc_new);
+#else
+{
+ return malloc_fast_path<tcmalloc::cpp_throw_oom>(size);
}
+#endif
extern "C" PERFTOOLS_DLL_DECL void* tc_newarray_nothrow(size_t size, const std::nothrow_t&)
- __THROW {
- void* p = cpp_alloc(size, true);
- MallocHook::InvokeNewHook(p, size);
- return p;
+ PERFTOOLS_NOTHROW
+#ifdef TC_ALIAS
+TC_ALIAS(tc_new_nothrow);
+#else
+{
+ return malloc_fast_path<tcmalloc::cpp_nothrow_oom>(size);
}
+#endif
-extern "C" PERFTOOLS_DLL_DECL void tc_deletearray(void* p) __THROW {
- MallocHook::InvokeDeleteHook(p);
- do_free(p);
+extern "C" PERFTOOLS_DLL_DECL void tc_deletearray(void* p) PERFTOOLS_NOTHROW
+#ifdef TC_ALIAS
+TC_ALIAS(tc_free);
+#else
+{
+ free_fast_path(p);
}
+#endif
-extern "C" PERFTOOLS_DLL_DECL void tc_deletearray_nothrow(void* p, const std::nothrow_t&) __THROW {
- MallocHook::InvokeDeleteHook(p);
- do_free(p);
+extern "C" PERFTOOLS_DLL_DECL void tc_deletearray_nothrow(void* p, const std::nothrow_t&) PERFTOOLS_NOTHROW
+#ifdef TC_ALIAS
+TC_ALIAS(tc_delete_nothrow);
+#else
+{
+ free_fast_path(p);
}
+#endif
-extern "C" PERFTOOLS_DLL_DECL void* tc_memalign(size_t align,
- size_t size) __THROW {
- void* result = do_memalign_or_cpp_memalign(align, size);
- MallocHook::InvokeNewHook(result, size);
- return result;
+extern "C" PERFTOOLS_DLL_DECL CACHELINE_ALIGNED_FN
+void* tc_memalign(size_t align, size_t size) PERFTOOLS_NOTHROW {
+ return memalign_fast_path<tcmalloc::malloc_oom>(align, size);
}
extern "C" PERFTOOLS_DLL_DECL int tc_posix_memalign(
- void** result_ptr, size_t align, size_t size) __THROW {
+ void** result_ptr, size_t align, size_t size) PERFTOOLS_NOTHROW {
if (((align % sizeof(void*)) != 0) ||
((align & (align - 1)) != 0) ||
(align == 0)) {
return EINVAL;
}
- void* result = do_memalign_or_cpp_memalign(align, size);
- MallocHook::InvokeNewHook(result, size);
- if (UNLIKELY(result == NULL)) {
+ void* result = tc_memalign(align, size);
+ if (PREDICT_FALSE(result == NULL)) {
return ENOMEM;
} else {
*result_ptr = result;
@@ -1687,47 +2104,119 @@
}
}
-static size_t pagesize = 0;
+#if defined(ENABLE_ALIGNED_NEW_DELETE)
-extern "C" PERFTOOLS_DLL_DECL void* tc_valloc(size_t size) __THROW {
- // Allocate page-aligned object of length >= size bytes
- if (pagesize == 0) pagesize = getpagesize();
- void* result = do_memalign_or_cpp_memalign(pagesize, size);
- MallocHook::InvokeNewHook(result, size);
- return result;
+extern "C" PERFTOOLS_DLL_DECL void* tc_new_aligned(size_t size, std::align_val_t align) {
+ return memalign_fast_path<tcmalloc::cpp_throw_oom>(static_cast<size_t>(align), size);
}
-extern "C" PERFTOOLS_DLL_DECL void* tc_pvalloc(size_t size) __THROW {
+extern "C" PERFTOOLS_DLL_DECL void* tc_new_aligned_nothrow(size_t size, std::align_val_t align, const std::nothrow_t&) PERFTOOLS_NOTHROW {
+ return memalign_fast_path<tcmalloc::cpp_nothrow_oom>(static_cast<size_t>(align), size);
+}
+
+extern "C" PERFTOOLS_DLL_DECL void tc_delete_aligned(void* p, std::align_val_t) PERFTOOLS_NOTHROW
+{
+ free_fast_path(p);
+}
+
+// There is no easy way to obtain the actual size used by do_memalign to allocate aligned storage, so for now
+// just ignore the size. It might get useful in the future.
+extern "C" PERFTOOLS_DLL_DECL void tc_delete_sized_aligned(void* p, size_t size, std::align_val_t align) PERFTOOLS_NOTHROW
+{
+ free_fast_path(p);
+}
+
+extern "C" PERFTOOLS_DLL_DECL void tc_delete_aligned_nothrow(void* p, std::align_val_t, const std::nothrow_t&) PERFTOOLS_NOTHROW
+{
+ free_fast_path(p);
+}
+
+extern "C" PERFTOOLS_DLL_DECL void* tc_newarray_aligned(size_t size, std::align_val_t align)
+#ifdef TC_ALIAS
+TC_ALIAS(tc_new_aligned);
+#else
+{
+ return memalign_fast_path<tcmalloc::cpp_throw_oom>(static_cast<size_t>(align), size);
+}
+#endif
+
+extern "C" PERFTOOLS_DLL_DECL void* tc_newarray_aligned_nothrow(size_t size, std::align_val_t align, const std::nothrow_t& nt) PERFTOOLS_NOTHROW
+#ifdef TC_ALIAS
+TC_ALIAS(tc_new_aligned_nothrow);
+#else
+{
+ return memalign_fast_path<tcmalloc::cpp_nothrow_oom>(static_cast<size_t>(align), size);
+}
+#endif
+
+extern "C" PERFTOOLS_DLL_DECL void tc_deletearray_aligned(void* p, std::align_val_t) PERFTOOLS_NOTHROW
+#ifdef TC_ALIAS
+TC_ALIAS(tc_delete_aligned);
+#else
+{
+ free_fast_path(p);
+}
+#endif
+
+// There is no easy way to obtain the actual size used by do_memalign to allocate aligned storage, so for now
+// just ignore the size. It might get useful in the future.
+extern "C" PERFTOOLS_DLL_DECL void tc_deletearray_sized_aligned(void* p, size_t size, std::align_val_t align) PERFTOOLS_NOTHROW
+#ifdef TC_ALIAS
+TC_ALIAS(tc_delete_sized_aligned);
+#else
+{
+ free_fast_path(p);
+}
+#endif
+
+extern "C" PERFTOOLS_DLL_DECL void tc_deletearray_aligned_nothrow(void* p, std::align_val_t, const std::nothrow_t&) PERFTOOLS_NOTHROW
+#ifdef TC_ALIAS
+TC_ALIAS(tc_delete_aligned_nothrow);
+#else
+{
+ free_fast_path(p);
+}
+#endif
+
+#endif // defined(ENABLE_ALIGNED_NEW_DELETE)
+
+static size_t pagesize = 0;
+
+extern "C" PERFTOOLS_DLL_DECL void* tc_valloc(size_t size) PERFTOOLS_NOTHROW {
+ // Allocate page-aligned object of length >= size bytes
+ if (pagesize == 0) pagesize = getpagesize();
+ return tc_memalign(pagesize, size);
+}
+
+extern "C" PERFTOOLS_DLL_DECL void* tc_pvalloc(size_t size) PERFTOOLS_NOTHROW {
// Round up size to a multiple of pagesize
if (pagesize == 0) pagesize = getpagesize();
if (size == 0) { // pvalloc(0) should allocate one page, according to
size = pagesize; // http://man.free4web.biz/man3/libmpatrol.3.html
}
size = (size + pagesize - 1) & ~(pagesize - 1);
- void* result = do_memalign_or_cpp_memalign(pagesize, size);
- MallocHook::InvokeNewHook(result, size);
- return result;
+ return tc_memalign(pagesize, size);
}
-extern "C" PERFTOOLS_DLL_DECL void tc_malloc_stats(void) __THROW {
+extern "C" PERFTOOLS_DLL_DECL void tc_malloc_stats(void) PERFTOOLS_NOTHROW {
do_malloc_stats();
}
-extern "C" PERFTOOLS_DLL_DECL int tc_mallopt(int cmd, int value) __THROW {
+extern "C" PERFTOOLS_DLL_DECL int tc_mallopt(int cmd, int value) PERFTOOLS_NOTHROW {
return do_mallopt(cmd, value);
}
#ifdef HAVE_STRUCT_MALLINFO
-extern "C" PERFTOOLS_DLL_DECL struct mallinfo tc_mallinfo(void) __THROW {
+extern "C" PERFTOOLS_DLL_DECL struct mallinfo tc_mallinfo(void) PERFTOOLS_NOTHROW {
return do_mallinfo();
}
#endif
-extern "C" PERFTOOLS_DLL_DECL size_t tc_malloc_size(void* ptr) __THROW {
+extern "C" PERFTOOLS_DLL_DECL size_t tc_malloc_size(void* ptr) PERFTOOLS_NOTHROW {
return MallocExtension::instance()->GetAllocatedSize(ptr);
}
-extern "C" PERFTOOLS_DLL_DECL void* tc_malloc_skip_new_handler(size_t size) __THROW {
+extern "C" PERFTOOLS_DLL_DECL void* tc_malloc_skip_new_handler(size_t size) PERFTOOLS_NOTHROW {
void* result = do_malloc(size);
MallocHook::InvokeNewHook(result, size);
return result;