From c04d4f4f5c85ad8477bdba87f8387667a5cff2e7 Mon Sep 17 00:00:00 2001 From: Chris Kennelly CA Date: Fri, 18 Sep 2026 14:15:41 -0700 Subject: [PATCH] Track releasable hugepages in HugeRegion with a bitmap. `HugeRegion::Release` scanned all 512 `backed_`/`pages_used_` entries to find free-and-backed hugepages. Maintain a 512-bit `unreleasable_` bitmap at the transitions that already update `free_backed_count_`, and walk its free ranges with `NextFreeRange`/`PrevFreeRange` (new) instead, marking whole ranges at once. PiperOrigin-RevId: 984062570 --- tcmalloc/huge_region.h | 51 ++++++++++------- tcmalloc/huge_region_test.cc | 48 ++++++++++++++++ tcmalloc/internal/range_tracker.h | 20 +++++++ tcmalloc/internal/range_tracker_test.cc | 73 +++++++++++++++++++++++++ 4 files changed, 172 insertions(+), 20 deletions(-) diff --git a/tcmalloc/huge_region.h b/tcmalloc/huge_region.h index 4bfae15db..3136ec8ef 100644 --- a/tcmalloc/huge_region.h +++ b/tcmalloc/huge_region.h @@ -159,6 +159,11 @@ class HugeRegion : public TList::Elem { Length pages_used_[kNumHugePages]; // Is this hugepage backed? bool backed_[kNumHugePages]; + // Set for hugepages that cannot be released: unbacked or with pages in use. + // Its clear bits are the free-and-backed hugepages counted by + // free_backed_count_, so Release can find them without scanning pages_used_ + // and backed_. + Bitmap unreleasable_; HugeLength nbacked_; HugeLength total_unbacked_{NHugePages(0)}; HugeLength free_backed_count_; @@ -295,6 +300,7 @@ inline HugeRegion::HugeRegion(HugeRange r, MemoryModifyFunction& unback, pages_used_[i] = Length(0); backed_[i] = false; } + unreleasable_.SetRange(0, kNumHugePages); free_backed_count_ = NHugePages(0); char name[256]; @@ -331,26 +337,29 @@ inline void HugeRegion::Put(Range r, bool release) { // sophisticated mechanism similar to Filler/Cache, that accounts for a recent // peak while releasing pages. inline HugeLength HugeRegion::Release(Length desired, bool adaptive_release) { - if (desired == Length(0)) return NHugePages(0); - const Length free_yet_backed = free_backed_count_.in_pages(); const Length to_release = std::min(desired, free_yet_backed); - - HugeLength release_target = NHugePages(0); + if (to_release == Length(0)) return NHugePages(0); + TC_ASSERT_EQ(free_backed_count_.raw_num(), + kNumHugePages - unreleasable_.CountBits()); + + // Releasable hugepages are the free ranges of unreleasable_. Adaptive + // release takes them from the end of the region, non-adaptive release from + // the start. + const size_t needed = HLFromPages(to_release).raw_num(); + size_t released = 0; bool should_unback[kNumHugePages] = {}; - const int start = adaptive_release ? kNumHugePages - 1 : 0; - const int end = adaptive_release ? -1 : kNumHugePages; - const int step = adaptive_release ? -1 : 1; - - // TODO(b/73749855): Consider optimizing this search by consulting tracker_. - for (int i = start; i != end; i += step) { - if (backed_[i] && pages_used_[i] == Length(0)) { - should_unback[i] = true; - ++release_target; - } - - if (release_target.in_pages() >= to_release) break; + size_t index = adaptive_release ? kNumHugePages : 0, n; + while (released < needed && + (adaptive_release ? unreleasable_.PrevFreeRange(index, &index, &n) + : unreleasable_.NextFreeRange(index, &index, &n))) { + const size_t take = std::min(n, needed - released); + const size_t first = adaptive_release ? index + n - take : index; + std::fill_n(should_unback + first, take, true); + released += take; + if (!adaptive_release) index += n; } + TC_ASSERT_EQ(released, needed); return UnbackHugepages(should_unback); } @@ -466,6 +475,7 @@ inline void HugeRegion::Inc(Range r, bool* from_released) { if (backed_[i]) { TC_ASSERT_GT(free_backed_count_, NHugePages(0)); --free_backed_count_; + unreleasable_.SetBit(i); } else { backed_[i] = true; should_back = true; @@ -494,6 +504,7 @@ inline void HugeRegion::Dec(Range r, bool release) { if (pages_used_[i] == Length(0)) { should_unback[i] = true; ++free_backed_count_; + unreleasable_.ClearBit(i); } r.p += here; r.n -= here; @@ -522,10 +533,8 @@ inline HugeLength HugeRegion::UnbackHugepages( // Temporarily block allocations to these pages. // - // We both Mark and toggle pages_used_, as allocations use FindAndMark but - // Release only uses pages_used_. - // - // TODO(b/73749855): Optimize release by consulting the bitmap first. + // We both Mark and toggle pages_used_, as allocations use FindAndMark. + // Release consults unreleasable_, which is set below. tracker_.Mark(NHugePages(i).in_pages().raw_num(), hl.in_pages().raw_num()); Length used; for (size_t k = i; k != j; ++k) { @@ -537,6 +546,7 @@ inline HugeLength HugeRegion::UnbackHugepages( // middle of being unbacked and are not eligible for allocation/release. TC_ASSERT_GE(free_backed_count_, hl); free_backed_count_ -= hl; + unreleasable_.SetRange(i, j - i); if (ABSL_PREDICT_TRUE(unback_(HugeRange(p, hl)).success)) { nbacked_ -= hl; @@ -551,6 +561,7 @@ inline HugeLength HugeRegion::UnbackHugepages( } else { // Restore the count if unback failed. free_backed_count_ += hl; + unreleasable_.ClearRange(i, j - i); } used = Length(0); diff --git a/tcmalloc/huge_region_test.cc b/tcmalloc/huge_region_test.cc index 148a791c5..b7e646aa8 100644 --- a/tcmalloc/huge_region_test.cc +++ b/tcmalloc/huge_region_test.cc @@ -315,6 +315,54 @@ TEST_F(HugeRegionTest, ReleaseAdaptive) { CheckMock(); } +// Release skips in-use hugepages between free ones, in both directions. +TEST_F(HugeRegionTest, ReleaseFragmented) { + for (const bool adaptive_release : {false, true}) { + SCOPED_TRACE(adaptive_release); + const Length n = kPagesPerHugePage; + bool from_released; + std::optional allocs[8]; + for (int i = 0; i < 8; ++i) { + allocs[i] = Allocate(n, &from_released); + EXPECT_TRUE(from_released); + } + + // Delete hugepages 0, 1, 3, 4, 6, 7 while keeping 2 and 5 allocated, so + // the free, backed hugepages are [0, 1], [3, 4], [6, 7]. + for (int i : {0, 1, 3, 4, 6, 7}) { + Delete(*allocs[i]); + } + + if (adaptive_release) { + // Reverse order releases 7, 6 from the last range, then 4. + ExpectUnback({p_ + NHugePages(4), NHugePages(1)}); + ExpectUnback({p_ + NHugePages(6), NHugePages(2)}); + } else { + // Forward order releases 0, 1 from the first range, then 3. + ExpectUnback({p_, NHugePages(2)}); + ExpectUnback({p_ + NHugePages(3), NHugePages(1)}); + } + EXPECT_EQ(NHugePages(3), + region_.Release(NHugePages(3).in_pages(), adaptive_release)); + CheckMock(); + + // Release the remaining five backed hugepages so the next iteration starts + // from an empty, unbacked region. + Delete(*allocs[2]); + Delete(*allocs[5]); + if (adaptive_release) { + ExpectUnback({p_, NHugePages(4)}); + ExpectUnback({p_ + NHugePages(5), NHugePages(1)}); + } else { + ExpectUnback({p_ + NHugePages(2), NHugePages(1)}); + ExpectUnback({p_ + NHugePages(4), NHugePages(4)}); + } + EXPECT_EQ(NHugePages(5), + region_.Release(NHugePages(8).in_pages(), adaptive_release)); + CheckMock(); + } +} + TEST_F(HugeRegionTest, ReleaseFailure) { const Length n = kPagesPerHugePage; bool from_released; diff --git a/tcmalloc/internal/range_tracker.h b/tcmalloc/internal/range_tracker.h index 335ea8fca..773803a7a 100644 --- a/tcmalloc/internal/range_tracker.h +++ b/tcmalloc/internal/range_tracker.h @@ -103,6 +103,12 @@ class Bitmap { // If there is at least one free range at or after , // put it in *index, *length and return true; else return false. bool NextFreeRange(size_t start, size_t* index, size_t* length) const; + // If there is at least one free range ending at or before , + // put it in *index, *length and return true; else return false. + // To iterate backwards through all free ranges: + // size_t index = bitmap.size(), n; + // while (bitmap.PrevFreeRange(index, &index, &n)) { ... } + bool PrevFreeRange(size_t end, size_t* index, size_t* length) const; // Returns index of the first {true, false} bit >= index, or N if none. size_t FindSet(size_t index) const; @@ -517,6 +523,20 @@ inline bool Bitmap::NextFreeRange(size_t start, size_t* index, return true; } +template +inline bool Bitmap::PrevFreeRange(size_t end, size_t* index, + size_t* length) const { + if (end == 0) return false; + if (end > N) end = N; + const ssize_t last = FindClearBackwards(end - 1); + if (last < 0) return false; + // FindSetBackwards returns -1 if the free range extends to bit 0. + const size_t first = FindSetBackwards(last) + 1; + *index = first; + *length = last + 1 - first; + return true; +} + template inline size_t Bitmap::FindSet(size_t index) const { return FindValue(index); diff --git a/tcmalloc/internal/range_tracker_test.cc b/tcmalloc/internal/range_tracker_test.cc index ed2b57b21..063aadc0e 100644 --- a/tcmalloc/internal/range_tracker_test.cc +++ b/tcmalloc/internal/range_tracker_test.cc @@ -394,6 +394,79 @@ TEST_F(BitmapTest, PopBatch) { EXPECT_TRUE(map.IsZero()); } +TEST_F(BitmapTest, PrevFreeRange) { + // Empty bitmap (all free). + { + Bitmap<253> map; + size_t index = map.size(), len = 0; + EXPECT_TRUE(map.PrevFreeRange(index, &index, &len)); + EXPECT_EQ(index, 0); + EXPECT_EQ(len, 253); + EXPECT_FALSE(map.PrevFreeRange(index, &index, &len)); + } + + // Full bitmap (no free ranges). + { + Bitmap<253> map; + map.SetRange(0, 253); + size_t index = map.size(), len = 0; + EXPECT_FALSE(map.PrevFreeRange(index, &index, &len)); + } + + // Bounds and edge conditions. + { + Bitmap<64> map; + size_t index = 0, len = 0; + EXPECT_FALSE(map.PrevFreeRange(0, &index, &len)); + + // end > N clamps to N. + EXPECT_TRUE(map.PrevFreeRange(100, &index, &len)); + EXPECT_EQ(index, 0); + EXPECT_EQ(len, 64); + + // end inside a free range truncates it. + EXPECT_TRUE(map.PrevFreeRange(30, &index, &len)); + EXPECT_EQ(index, 0); + EXPECT_EQ(len, 30); + + // A free range ending at bit 0. + map.SetRange(1, 63); + EXPECT_TRUE(map.PrevFreeRange(64, &index, &len)); + EXPECT_EQ(index, 0); + EXPECT_EQ(len, 1); + EXPECT_FALSE(map.PrevFreeRange(index, &index, &len)); + } + + // Multiple ranges, including ones straddling word boundaries, matching + // forward traversal in reverse. + { + Bitmap<253> map; + map.SetRange(0, 253); + // Open free ranges: [10, 20), [50, 80), [120, 150), [200, 253) + map.ClearRange(10, 10); + map.ClearRange(50, 30); + map.ClearRange(120, 30); + map.ClearRange(200, 53); + + std::vector> forward_ranges; + size_t f_index = 0, f_len; + while (map.NextFreeRange(f_index, &f_index, &f_len)) { + forward_ranges.push_back({f_index, f_len}); + f_index += f_len; + } + EXPECT_THAT(forward_ranges, ElementsAre(Pair(10, 10), Pair(50, 30), + Pair(120, 30), Pair(200, 53))); + + std::vector> backward_ranges; + size_t b_index = map.size(), b_len; + while (map.PrevFreeRange(b_index, &b_index, &b_len)) { + backward_ranges.push_back({b_index, b_len}); + } + EXPECT_THAT(backward_ranges, ElementsAre(Pair(200, 53), Pair(120, 30), + Pair(50, 30), Pair(10, 10))); + } +} + class RangeTrackerTest : public ::testing::Test { protected: std::vector> FreeRanges() {