Release pageheap_lock while HugePageFiller unbacks memory.

tcmalloc/BUILD

@@ -1057,6 +1057,7 @@ cc_test(
         "//tcmalloc/internal:pageflags",
         "//tcmalloc/internal:range_tracker",
         "//tcmalloc/internal:residency",
+        "//tcmalloc/internal:scoped_allow_allocation",
         "//tcmalloc/internal:system_allocator",
         "//tcmalloc/testing:testutil",
         "@com_github_google_benchmark//:benchmark",
@@ -1066,11 +1067,11 @@ cc_test(
         "@com_google_absl//absl/container:flat_hash_map",
         "@com_google_absl//absl/container:flat_hash_set",
         "@com_google_absl//absl/flags:flag",
+        "@com_google_absl//absl/functional:any_invocable",
         "@com_google_absl//absl/random",
         "@com_google_absl//absl/status",
         "@com_google_absl//absl/strings",
         "@com_google_absl//absl/strings:str_format",
-        "@com_google_absl//absl/synchronization",
         "@com_google_absl//absl/time",
         "@com_google_googletest//:gtest_main",
     ],

tcmalloc/CMakeLists.txt

@@ -1229,6 +1229,7 @@ tcmalloc_cc_test(
     "GTest::gtest_main"
     "GTest::gmock_main"
     "GTest::gmock"
+    "absl::any_invocable"
     "absl::base"
     "absl::core_headers"
     "absl::flags"
@@ -1241,7 +1242,6 @@ tcmalloc_cc_test(
     "absl::status"
     "absl::str_format"
     "absl::strings"
-    "absl::synchronization"
     "absl::time"
     "benchmark::benchmark"
     "tcmalloc::common_8k_pages"
@@ -1252,6 +1252,7 @@ tcmalloc_cc_test(
     "tcmalloc::internal_pageflags"
     "tcmalloc::internal_range_tracker"
     "tcmalloc::internal_residency"
+    "tcmalloc::internal_scoped_allow_allocation"
     "tcmalloc::internal_system_allocator"
     "tcmalloc::tcmalloc"
     "tcmalloc::testing_testutil"

tcmalloc/huge_page_aware_allocator.h

@@ -461,8 +461,8 @@ class HugePageAwareAllocator final : public PageAllocatorInterface {
 
   void ReleaseHugepage(FillerType::Tracker* pt)
       ABSL_EXCLUSIVE_LOCKS_REQUIRED(pageheap_lock);
-  // Returns hugepages that the filler emptied while it did not hold
-  // pageheap_lock (during TreatHugepageTrackers) to the cache.
+  // Returns hugepages that the filler emptied while a release or treatment
+  // had pageheap_lock dropped (see HugePageFiller::FetchFullyFreedTracker).
   void DrainFreedTrackers() ABSL_EXCLUSIVE_LOCKS_REQUIRED(pageheap_lock);
   // Return an allocation from a single hugepage.
   void DeleteFromHugepage(FillerType::Tracker* pt, Range r, bool might_abandon,
@@ -490,8 +490,8 @@ inline HugePageAwareAllocator<Forwarder>::HugePageAwareAllocator(
       unback_without_lock_(*this),
       collapse_(*this),
       set_anon_vma_name_(*this),
-      filler_(tag_, unback_, unback_without_lock_, collapse_,
-              set_anon_vma_name_, forwarder_.subrelease_unbacked_hugepages()),
+      filler_(tag_, unback_without_lock_, collapse_, set_anon_vma_name_,
+              forwarder_.subrelease_unbacked_hugepages()),
       regions_(options.use_huge_region_more_often),
       tracker_allocator_(forwarder_.arena()),
       region_allocator_(forwarder_.arena()),
@@ -1050,6 +1050,7 @@ inline Length HugePageAwareAllocator<Forwarder>::ReleaseAtLeastNPages(
                   forwarder_.filler_skip_subrelease_long_interval()},
           forwarder_.release_partial_alloc_pages(),
           /*hit_limit*/ false);
+      DrainFreedTrackers();
     }
   }
 
@@ -1254,6 +1255,7 @@ HugePageAwareAllocator<Forwarder>::ReleaseAtLeastNPagesBreakingHugepages(
   released += filler_.ReleasePages(n - released, SkipSubreleaseIntervals{},
                                    /*release_partial_alloc_pages=*/false,
                                    /*hit_limit=*/true);
+  DrainFreedTrackers();
 
   info_.RecordRelease(n, released, reason);
   return released;

tcmalloc/huge_page_aware_allocator_fuzz.cc

@@ -485,8 +485,8 @@ struct State {
       if (tcmalloc::tcmalloc_internal::pageheap_lock.IsHeld()) {
         // This permits a slight degree of nondeterminism when linked against
         // TCMalloc for the real memory allocator, as a background thread could
-        // also be holding the lock.  Nevertheless, HPAA doesn't make it clear
-        // when we are releasing with/without the pageheap_lock.
+        // also be holding the lock.  HugeCache and HugePageFiller release with
+        // the lock dropped, HugeRegion does not.
         //
         // TODO(b/73749855): When all release paths unconditionally release the
         // lock, remove this check and take the lock for an instant to ensure it
@@ -713,11 +713,15 @@ void GatherAndCheckStats::Perform(State& state) const {
     PageHeapSpinLockHolder l;
     stats = state.allocator.stats();
   }
-  uint64_t used_bytes =
+  const uint64_t used_bytes =
       stats.system_bytes - stats.free_bytes - stats.unmapped_bytes;
-  TC_CHECK_EQ(used_bytes,
-              state.allocated.in_bytes() +
-                  state.allocator.forwarder().pending_release_.in_bytes());
+  // While a release has pageheap_lock dropped, HugeCache has already removed
+  // the range from its size (so it appears used) whereas HugePageFiller keeps
+  // the pages free until unback succeeds.  Outside of a release, the two agree.
+  const uint64_t pending_bytes =
+      state.allocator.forwarder().pending_release_.in_bytes();
+  TC_CHECK_GE(used_bytes, state.allocated.in_bytes());
+  TC_CHECK_LE(used_bytes, state.allocated.in_bytes() + pending_bytes);
 }
 
 void GatherSpanStats::Perform(State& state) const {