Squashed 'third_party/gperftools/' content from commit 54505f1
Change-Id: Id02e833828732b0efe7dac722b8485279e67c5fa
git-subtree-dir: third_party/gperftools
git-subtree-split: 54505f1d50c2d1f4676f5e87090b64a117fd980e
diff --git a/src/memfs_malloc.cc b/src/memfs_malloc.cc
new file mode 100644
index 0000000..ce20891
--- /dev/null
+++ b/src/memfs_malloc.cc
@@ -0,0 +1,268 @@
+// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*-
+// Copyright (c) 2007, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+// * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// ---
+// Author: Arun Sharma
+//
+// A tcmalloc system allocator that uses a memory based filesystem such as
+// tmpfs or hugetlbfs
+//
+// Since these only exist on linux, we only register this allocator there.
+
+#ifdef __linux
+
+#include <config.h>
+#include <errno.h> // for errno, EINVAL
+#include <inttypes.h> // for PRId64
+#include <limits.h> // for PATH_MAX
+#include <stddef.h> // for size_t, NULL
+#ifdef HAVE_STDINT_H
+#include <stdint.h> // for int64_t, uintptr_t
+#endif
+#include <stdio.h> // for snprintf
+#include <stdlib.h> // for mkstemp
+#include <string.h> // for strerror
+#include <sys/mman.h> // for mmap, MAP_FAILED, etc
+#include <sys/statfs.h> // for fstatfs, statfs
+#include <unistd.h> // for ftruncate, off_t, unlink
+#include <new> // for operator new
+#include <string>
+
+#include <gperftools/malloc_extension.h>
+#include "base/basictypes.h"
+#include "base/googleinit.h"
+#include "base/sysinfo.h"
+#include "internal_logging.h"
+
+// TODO(sanjay): Move the code below into the tcmalloc namespace
+using tcmalloc::kLog;
+using tcmalloc::kCrash;
+using tcmalloc::Log;
+using std::string;
+
+DEFINE_string(memfs_malloc_path, EnvToString("TCMALLOC_MEMFS_MALLOC_PATH", ""),
+ "Path where hugetlbfs or tmpfs is mounted. The caller is "
+ "responsible for ensuring that the path is unique and does "
+ "not conflict with another process");
+DEFINE_int64(memfs_malloc_limit_mb,
+ EnvToInt("TCMALLOC_MEMFS_LIMIT_MB", 0),
+ "Limit total allocation size to the "
+ "specified number of MiB. 0 == no limit.");
+DEFINE_bool(memfs_malloc_abort_on_fail,
+ EnvToBool("TCMALLOC_MEMFS_ABORT_ON_FAIL", false),
+ "abort() whenever memfs_malloc fails to satisfy an allocation "
+ "for any reason.");
+DEFINE_bool(memfs_malloc_ignore_mmap_fail,
+ EnvToBool("TCMALLOC_MEMFS_IGNORE_MMAP_FAIL", false),
+ "Ignore failures from mmap");
+DEFINE_bool(memfs_malloc_map_private,
+ EnvToBool("TCMALLOC_MEMFS_MAP_PRIVATE", false),
+ "Use MAP_PRIVATE with mmap");
+
+// Hugetlbfs based allocator for tcmalloc
+class HugetlbSysAllocator: public SysAllocator {
+public:
+ explicit HugetlbSysAllocator(SysAllocator* fallback)
+ : failed_(true), // To disable allocator until Initialize() is called.
+ big_page_size_(0),
+ hugetlb_fd_(-1),
+ hugetlb_base_(0),
+ fallback_(fallback) {
+ }
+
+ void* Alloc(size_t size, size_t *actual_size, size_t alignment);
+ bool Initialize();
+
+ bool failed_; // Whether failed to allocate memory.
+
+private:
+ void* AllocInternal(size_t size, size_t *actual_size, size_t alignment);
+
+ int64 big_page_size_;
+ int hugetlb_fd_; // file descriptor for hugetlb
+ off_t hugetlb_base_;
+
+ SysAllocator* fallback_; // Default system allocator to fall back to.
+};
+static char hugetlb_space[sizeof(HugetlbSysAllocator)];
+
+// No locking needed here since we assume that tcmalloc calls
+// us with an internal lock held (see tcmalloc/system-alloc.cc).
+void* HugetlbSysAllocator::Alloc(size_t size, size_t *actual_size,
+ size_t alignment) {
+ if (failed_) {
+ return fallback_->Alloc(size, actual_size, alignment);
+ }
+
+ // We don't respond to allocation requests smaller than big_page_size_ unless
+ // the caller is ok to take more than they asked for. Used by MetaDataAlloc.
+ if (actual_size == NULL && size < big_page_size_) {
+ return fallback_->Alloc(size, actual_size, alignment);
+ }
+
+ // Enforce huge page alignment. Be careful to deal with overflow.
+ size_t new_alignment = alignment;
+ if (new_alignment < big_page_size_) new_alignment = big_page_size_;
+ size_t aligned_size = ((size + new_alignment - 1) /
+ new_alignment) * new_alignment;
+ if (aligned_size < size) {
+ return fallback_->Alloc(size, actual_size, alignment);
+ }
+
+ void* result = AllocInternal(aligned_size, actual_size, new_alignment);
+ if (result != NULL) {
+ return result;
+ }
+ Log(kLog, __FILE__, __LINE__,
+ "HugetlbSysAllocator: (failed, allocated)", failed_, hugetlb_base_);
+ if (FLAGS_memfs_malloc_abort_on_fail) {
+ Log(kCrash, __FILE__, __LINE__,
+ "memfs_malloc_abort_on_fail is set");
+ }
+ return fallback_->Alloc(size, actual_size, alignment);
+}
+
+void* HugetlbSysAllocator::AllocInternal(size_t size, size_t* actual_size,
+ size_t alignment) {
+ // Ask for extra memory if alignment > pagesize
+ size_t extra = 0;
+ if (alignment > big_page_size_) {
+ extra = alignment - big_page_size_;
+ }
+
+ // Test if this allocation would put us over the limit.
+ off_t limit = FLAGS_memfs_malloc_limit_mb*1024*1024;
+ if (limit > 0 && hugetlb_base_ + size + extra > limit) {
+ // Disable the allocator when there's less than one page left.
+ if (limit - hugetlb_base_ < big_page_size_) {
+ Log(kLog, __FILE__, __LINE__, "reached memfs_malloc_limit_mb");
+ failed_ = true;
+ }
+ else {
+ Log(kLog, __FILE__, __LINE__,
+ "alloc too large (size, bytes left)", size, limit-hugetlb_base_);
+ }
+ return NULL;
+ }
+
+ // This is not needed for hugetlbfs, but needed for tmpfs. Annoyingly
+ // hugetlbfs returns EINVAL for ftruncate.
+ int ret = ftruncate(hugetlb_fd_, hugetlb_base_ + size + extra);
+ if (ret != 0 && errno != EINVAL) {
+ Log(kLog, __FILE__, __LINE__,
+ "ftruncate failed", strerror(errno));
+ failed_ = true;
+ return NULL;
+ }
+
+ // Note: size + extra does not overflow since:
+ // size + alignment < (1<<NBITS).
+ // and extra <= alignment
+ // therefore size + extra < (1<<NBITS)
+ void *result;
+ result = mmap(0, size + extra, PROT_WRITE|PROT_READ,
+ FLAGS_memfs_malloc_map_private ? MAP_PRIVATE : MAP_SHARED,
+ hugetlb_fd_, hugetlb_base_);
+ if (result == reinterpret_cast<void*>(MAP_FAILED)) {
+ if (!FLAGS_memfs_malloc_ignore_mmap_fail) {
+ Log(kLog, __FILE__, __LINE__,
+ "mmap failed (size, error)", size + extra, strerror(errno));
+ failed_ = true;
+ }
+ return NULL;
+ }
+ uintptr_t ptr = reinterpret_cast<uintptr_t>(result);
+
+ // Adjust the return memory so it is aligned
+ size_t adjust = 0;
+ if ((ptr & (alignment - 1)) != 0) {
+ adjust = alignment - (ptr & (alignment - 1));
+ }
+ ptr += adjust;
+ hugetlb_base_ += (size + extra);
+
+ if (actual_size) {
+ *actual_size = size + extra - adjust;
+ }
+
+ return reinterpret_cast<void*>(ptr);
+}
+
+bool HugetlbSysAllocator::Initialize() {
+ char path[PATH_MAX];
+ const int pathlen = FLAGS_memfs_malloc_path.size();
+ if (pathlen + 8 > sizeof(path)) {
+ Log(kCrash, __FILE__, __LINE__, "XX fatal: memfs_malloc_path too long");
+ return false;
+ }
+ memcpy(path, FLAGS_memfs_malloc_path.data(), pathlen);
+ memcpy(path + pathlen, ".XXXXXX", 8); // Also copies terminating \0
+
+ int hugetlb_fd = mkstemp(path);
+ if (hugetlb_fd == -1) {
+ Log(kLog, __FILE__, __LINE__,
+ "warning: unable to create memfs_malloc_path",
+ path, strerror(errno));
+ return false;
+ }
+
+ // Cleanup memory on process exit
+ if (unlink(path) == -1) {
+ Log(kCrash, __FILE__, __LINE__,
+ "fatal: error unlinking memfs_malloc_path", path, strerror(errno));
+ return false;
+ }
+
+ // Use fstatfs to figure out the default page size for memfs
+ struct statfs sfs;
+ if (fstatfs(hugetlb_fd, &sfs) == -1) {
+ Log(kCrash, __FILE__, __LINE__,
+ "fatal: error fstatfs of memfs_malloc_path", strerror(errno));
+ return false;
+ }
+ int64 page_size = sfs.f_bsize;
+
+ hugetlb_fd_ = hugetlb_fd;
+ big_page_size_ = page_size;
+ failed_ = false;
+ return true;
+}
+
+REGISTER_MODULE_INITIALIZER(memfs_malloc, {
+ if (FLAGS_memfs_malloc_path.length()) {
+ SysAllocator* alloc = MallocExtension::instance()->GetSystemAllocator();
+ HugetlbSysAllocator* hp = new (hugetlb_space) HugetlbSysAllocator(alloc);
+ if (hp->Initialize()) {
+ MallocExtension::instance()->SetSystemAllocator(hp);
+ }
+ }
+});
+
+#endif /* ifdef __linux */