X-Git-Url: http://plrg.eecs.uci.edu/git/?p=folly.git;a=blobdiff_plain;f=folly%2FMalloc.h;h=ca8c7c7aa9801e66563926b7b06a98c04343d10e;hp=e43c2afef0ba4cfd537a92407adecb440ced994d;hb=275ca94d04e44f28cfa411668eb1c1dd8db90b80;hpb=f2f512b45cf346e3d505c22d98d381436933c441 diff --git a/folly/Malloc.h b/folly/Malloc.h index e43c2afe..ca8c7c7a 100644 --- a/folly/Malloc.h +++ b/folly/Malloc.h @@ -1,5 +1,5 @@ /* - * Copyright 2012 Facebook, Inc. + * Copyright 2015 Facebook, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,13 +20,28 @@ #ifndef FOLLY_MALLOC_H_ #define FOLLY_MALLOC_H_ +/** + * Define various MALLOCX_* macros normally provided by jemalloc. We define + * them so that we don't have to include jemalloc.h, in case the program is + * built without jemalloc support. + */ +#ifndef MALLOCX_LG_ALIGN +#define MALLOCX_LG_ALIGN(la) (la) +#endif +#ifndef MALLOCX_ZERO +#define MALLOCX_ZERO (static_cast(0x40)) +#endif + // If using fbstring from libstdc++, then just define stub code // here to typedef the fbstring type into the folly namespace. // This provides backwards compatibility for code that explicitly // includes and uses fbstring. #if defined(_GLIBCXX_USE_FB) && !defined(_LIBSTDCXX_FBSTRING) +#include + #include + namespace folly { using std::goodMallocSize; using std::jemallocMinInPlaceExpandable; @@ -41,15 +56,38 @@ namespace folly { #ifdef _LIBSTDCXX_FBSTRING #pragma GCC system_header + +/** + * Declare *allocx() and mallctl() as weak symbols. These will be provided by + * jemalloc if we are using jemalloc, or will be NULL if we are using another + * malloc implementation. + */ +extern "C" void* mallocx(size_t, int) +__attribute__((__weak__)); +extern "C" void* rallocx(void*, size_t, int) +__attribute__((__weak__)); +extern "C" size_t xallocx(void*, size_t, size_t, int) +__attribute__((__weak__)); +extern "C" size_t sallocx(const void*, int) +__attribute__((__weak__)); +extern "C" void dallocx(void*, int) +__attribute__((__weak__)); +extern "C" size_t nallocx(size_t, int) +__attribute__((__weak__)); +extern "C" int mallctl(const char*, void*, size_t*, void*, size_t) +__attribute__((__weak__)); + +#include #define FOLLY_HAVE_MALLOC_H 1 #else -#include "folly-config.h" +#include /* nolint */ +#include #endif // for malloc_usable_size // NOTE: FreeBSD 9 doesn't have malloc.h. It's defitions // are found in stdlib.h. -#ifdef FOLLY_HAVE_MALLOC_H +#if FOLLY_HAVE_MALLOC_H #include #else #include @@ -62,30 +100,6 @@ namespace folly { #include -#include - -/** - * Declare rallocm() and malloc_usable_size() as weak symbols. It - * will be provided by jemalloc if we are using jemalloc, or it will - * be NULL if we are using another malloc implementation. - */ -extern "C" int rallocm(void**, size_t*, size_t, size_t, int) -__attribute__((weak)); - -/** - * Define the ALLOCM_SUCCESS, ALLOCM_ZERO, and ALLOCM_NO_MOVE constants - * normally provided by jemalloc. We define them so that we don't have to - * include jemalloc.h, in case the program is built without jemalloc support. - */ -#ifndef ALLOCM_SUCCESS -#define ALLOCM_SUCCESS 0 -#define ALLOCM_ERR_OOM 1 -#define ALLOCM_ERR_NOT_MOVED 2 - -#define ALLOCM_ZERO 64 -#define ALLOCM_NO_MOVE 128 -#endif - #ifdef _LIBSTDCXX_FBSTRING namespace std _GLIBCXX_VISIBILITY(default) { _GLIBCXX_BEGIN_NAMESPACE_VERSION @@ -93,45 +107,47 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION namespace folly { #endif +bool usingJEMallocSlow(); /** * Determine if we are using jemalloc or not. */ inline bool usingJEMalloc() { - return rallocm != NULL; + // Checking for rallocx != NULL is not sufficient; we may be in a dlopen()ed + // module that depends on libjemalloc, so rallocx is resolved, but the main + // program might be using a different memory allocator. Look at the + // implementation of usingJEMallocSlow() for the (hacky) details. + static const bool result = usingJEMallocSlow(); + return result; } /** * For jemalloc's size classes, see * http://www.canonware.com/download/jemalloc/jemalloc-latest/doc/jemalloc.html */ -inline size_t goodMallocSize(size_t minSize) { +inline size_t goodMallocSize(size_t minSize) noexcept { if (!usingJEMalloc()) { // Not using jemalloc - no smarts return minSize; } + size_t goodSize; if (minSize <= 64) { // Choose smallest allocation to be 64 bytes - no tripping over // cache line boundaries, and small string optimization takes care // of short strings anyway. - return 64; - } - if (minSize <= 512) { + goodSize = 64; + } else if (minSize <= 512) { // Round up to the next multiple of 64; we don't want to trip over // cache line boundaries. - return (minSize + 63) & ~size_t(63); - } - if (minSize <= 3840) { - // Round up to the next multiple of 256 - return (minSize + 255) & ~size_t(255); - } - if (minSize <= 4072 * 1024) { - // Round up to the next multiple of 4KB - return (minSize + 4095) & ~size_t(4095); + goodSize = (minSize + 63) & ~size_t(63); + } else { + // Boundaries between size classes depend on numerious factors, some of + // which can even be modified at run-time. Determine the good allocation + // size by calling nallocx() directly. + goodSize = nallocx(minSize, 0); } - // Holy Moly - // Round up to the next multiple of 4MB - return (minSize + 4194303) & ~size_t(4194303); + assert(nallocx(goodSize, 0) == goodSize); + return goodSize; } // We always request "good" sizes for allocation, so jemalloc can @@ -185,8 +201,14 @@ inline void* smartRealloc(void* p, if (usingJEMalloc()) { // using jemalloc's API. Don't forget that jemalloc can never grow // in place blocks smaller than 4096 bytes. + // + // NB: newCapacity may not be precisely equal to a jemalloc size class, + // i.e. newCapacity is not guaranteed to be the result of a + // goodMallocSize() call, therefore xallocx() may return more than + // newCapacity bytes of space. Use >= rather than == to check whether + // xallocx() successfully expanded in place. if (currentCapacity >= jemallocMinInPlaceExpandable && - rallocm(&p, NULL, newCapacity, 0, ALLOCM_NO_MOVE) == ALLOCM_SUCCESS) { + xallocx(p, newCapacity, 0, 0) >= newCapacity) { // Managed to expand in place return p; }