Skip to content

Make MetadataObjectAllocator bss-suitable #980

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
copybara-service merged 1 commit into from
Sep 18, 2026
Merged

Make MetadataObjectAllocator bss-suitable #980

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 10 additions & 4 deletions tcmalloc/metadata_object_allocator.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,9 +50,15 @@ struct AllocatorStats {
template <class T, ArenaAlloc AllocType>
class MetadataObjectAllocator {
public:
constexpr MetadataObjectAllocator() = default;
constexpr explicit MetadataObjectAllocator(
Arena& arena ABSL_ATTRIBUTE_LIFETIME_BOUND)
: arena_(&arena), free_list_(nullptr), stats_{0, 0} {}
: arena_(&arena) {}

void Init(Arena& arena) {
TC_CHECK(!arena_);
arena_ = &arena;
}

// Allocates storage for a T.
//
Expand Down Expand Up @@ -125,15 +131,15 @@ class MetadataObjectAllocator {
}

// Arena from which to allocate memory
Arena* arena_;
Arena* arena_ = nullptr;

mutable absl::base_internal::SpinLock metadata_lock_{
absl::base_internal::SCHEDULE_KERNEL_ONLY};

// Free list of already carved objects
T* free_list_ ABSL_GUARDED_BY(metadata_lock_);
T* free_list_ ABSL_GUARDED_BY(metadata_lock_) = nullptr;

AllocatorStats stats_ ABSL_GUARDED_BY(metadata_lock_);
AllocatorStats stats_ ABSL_GUARDED_BY(metadata_lock_) = {};
};

} // namespace tcmalloc_internal
Expand Down
89 changes: 48 additions & 41 deletions tcmalloc/static_vars.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,22 +73,22 @@ namespace tcmalloc_internal {
//
// IF YOU ADD TO THIS LIST, ADD TO STATIC_VAR_SIZE TOO!
// LINT.IfChange(static_vars)
ABSL_CONST_INIT absl::base_internal::SpinLock pageheap_lock(
TC_ENSURE_BSS absl::base_internal::SpinLock pageheap_lock(
absl::base_internal::SCHEDULE_KERNEL_ONLY);
ABSL_CONST_INIT Arena Static::arena_;
ABSL_CONST_INIT SizeMap ABSL_CACHELINE_ALIGNED Static::sizemap_;
TC_ENSURE_BSS Arena Static::arena_;
TC_ENSURE_BSS ABSL_CACHELINE_ALIGNED SizeMap Static::sizemap_;
TCMALLOC_ATTRIBUTE_NO_DESTROY ABSL_CONST_INIT TransferCacheManager
Static::transfer_cache_;
ABSL_CONST_INIT ShardedTransferCacheManager
TC_ENSURE_BSS ShardedTransferCacheManager
Static::sharded_transfer_cache_(nullptr, nullptr);
TC_ENSURE_BSS ABSL_CACHELINE_ALIGNED Static::CpuCacheType Static::cpu_cache_;
ABSL_CONST_INIT
TC_ENSURE_BSS
MetadataObjectAllocator<SampledAllocation, ArenaAlloc::kSampledAllocation>
Static::sampledallocation_allocator_{arena_};
ABSL_CONST_INIT MetadataObjectAllocator<Span, ArenaAlloc::kSpan>
Static::span_allocator_{arena_};
ABSL_CONST_INIT MetadataObjectAllocator<ThreadCache, ArenaAlloc::kThreadCache>
Static::threadcache_allocator_{arena_};
Static::sampledallocation_allocator_;
TC_ENSURE_BSS MetadataObjectAllocator<Span, ArenaAlloc::kSpan>
Static::span_allocator_;
TC_ENSURE_BSS MetadataObjectAllocator<ThreadCache, ArenaAlloc::kThreadCache>
Static::threadcache_allocator_;
TCMALLOC_ATTRIBUTE_NO_DESTROY ABSL_CONST_INIT
Static::NoDestructorStorage<SampledAllocationRecorder>
Static::sampled_allocation_recorder_{sampledallocation_allocator_};
Expand All @@ -103,9 +103,9 @@ ABSL_CONST_INIT std::atomic<int64_t> Static::sampled_alloc_handle_generator{0};
TCMALLOC_ATTRIBUTE_NO_DESTROY ABSL_CONST_INIT
Static::NoDestructorStorage<PeakHeapTracker>
Static::peak_heap_tracker_{sampledallocation_allocator_};
ABSL_CONST_INIT MetadataObjectAllocator<StackTraceTable::LinkedSample,
ArenaAlloc::kStackTraceTable>
Static::linked_sample_allocator_{arena_};
TC_ENSURE_BSS MetadataObjectAllocator<StackTraceTable::LinkedSample,
ArenaAlloc::kStackTraceTable>
Static::linked_sample_allocator_;
ABSL_CONST_INIT std::atomic<bool> Static::inited_{false};
ABSL_CONST_INIT std::atomic<bool> Static::cpu_cache_active_{false};
ABSL_CONST_INIT Static::PageAllocatorStorage Static::page_allocator_;
Expand Down Expand Up @@ -208,36 +208,43 @@ ABSL_ATTRIBUTE_COLD ABSL_ATTRIBUTE_NOINLINE void Static::SlowInitIfNecessary() {
PageHeapSpinLockHolder l;

// double-checked locking
if (!inited_.load(std::memory_order_acquire)) {
TC_CHECK(sizemap_.Init(SizeMap::CurrentClasses().classes));
// Verify we can determine the number of CPUs now, since we will need it
// later for per-CPU caches and initializing the cache topology.
if (ABSL_PREDICT_FALSE(!NumCPUsMaybe().has_value())) {
TCMalloc_Internal_SetPerCpuCachesEnabledNoBuildRequirement(false);
}
(void)subtle::percpu::IsFast();
PerCpuState::state().Init();
numa_topology_.Init();
CacheTopology::Instance().Init();
cpu_cache_.Init();

if (IsExperimentActive(Experiment::TCMALLOC_PGHO_EXPERIMENT)
) {
TCMalloc_Internal_SetMinHotAccessHint(/*v=*/2);
}
if (inited_.load(std::memory_order_acquire)) {
return;
}

// Do a bit of sanitizing: make sure central_cache is aligned properly
TC_CHECK_EQ((sizeof(transfer_cache_) % ABSL_CACHELINE_SIZE), 0);
transfer_cache_.Init();
// The constructor of the sharded transfer cache leaves it in a disabled
// state.
sharded_transfer_cache_.Init();
new (page_allocator_.memory) PageAllocator;
guardedpage_allocator_.Init(/*max_allocated_pages=*/64,
/*total_pages=*/128);

inited_.store(true, std::memory_order_release);
TC_CHECK(sizemap_.Init(SizeMap::CurrentClasses().classes));
sampledallocation_allocator_.Init(arena_);
span_allocator_.Init(arena_);
threadcache_allocator_.Init(arena_);
linked_sample_allocator_.Init(arena_);

// Verify we can determine the number of CPUs now, since we will need it
// later for per-CPU caches and initializing the cache topology.
if (ABSL_PREDICT_FALSE(!NumCPUsMaybe().has_value())) {
TCMalloc_Internal_SetPerCpuCachesEnabledNoBuildRequirement(false);
}
(void)subtle::percpu::IsFast();
PerCpuState::state().Init();
numa_topology_.Init();
CacheTopology::Instance().Init();
cpu_cache_.Init();

if (IsExperimentActive(Experiment::TCMALLOC_PGHO_EXPERIMENT)
) {
TCMalloc_Internal_SetMinHotAccessHint(/*v=*/2);
}

// Do a bit of sanitizing: make sure central_cache is aligned properly
TC_CHECK_EQ((sizeof(transfer_cache_) % ABSL_CACHELINE_SIZE), 0);
transfer_cache_.Init();
// The constructor of the sharded transfer cache leaves it in a disabled
// state.
sharded_transfer_cache_.Init();
new (page_allocator_.memory) PageAllocator;
guardedpage_allocator_.Init(/*max_allocated_pages=*/64,
/*total_pages=*/128);

inited_.store(true, std::memory_order_release);
}

} // namespace tcmalloc_internal
Expand Down
Loading