Skip to content

Track releasable hugepages in HugeRegion with a bitmap. #1017

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
copybara-service wants to merge 1 commit into from
+172 −20
Closed

Track releasable hugepages in HugeRegion with a bitmap. #1017

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 31 additions & 20 deletions tcmalloc/huge_region.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,11 @@ class HugeRegion : public TList<HugeRegion>::Elem {
Length pages_used_[kNumHugePages];
// Is this hugepage backed?
bool backed_[kNumHugePages];
// Set for hugepages that cannot be released: unbacked or with pages in use.
// Its clear bits are the free-and-backed hugepages counted by
// free_backed_count_, so Release can find them without scanning pages_used_
// and backed_.
Bitmap<kNumHugePages> unreleasable_;
HugeLength nbacked_;
HugeLength total_unbacked_{NHugePages(0)};
HugeLength free_backed_count_;
Expand Down Expand Up @@ -295,6 +300,7 @@ inline HugeRegion::HugeRegion(HugeRange r, MemoryModifyFunction& unback,
pages_used_[i] = Length(0);
backed_[i] = false;
}
unreleasable_.SetRange(0, kNumHugePages);
free_backed_count_ = NHugePages(0);

char name[256];
Expand Down Expand Up @@ -331,26 +337,29 @@ inline void HugeRegion::Put(Range r, bool release) {
// sophisticated mechanism similar to Filler/Cache, that accounts for a recent
// peak while releasing pages.
inline HugeLength HugeRegion::Release(Length desired, bool adaptive_release) {
if (desired == Length(0)) return NHugePages(0);

const Length free_yet_backed = free_backed_count_.in_pages();
const Length to_release = std::min(desired, free_yet_backed);

HugeLength release_target = NHugePages(0);
if (to_release == Length(0)) return NHugePages(0);
TC_ASSERT_EQ(free_backed_count_.raw_num(),
kNumHugePages - unreleasable_.CountBits());

// Releasable hugepages are the free ranges of unreleasable_. Adaptive
// release takes them from the end of the region, non-adaptive release from
// the start.
const size_t needed = HLFromPages(to_release).raw_num();
size_t released = 0;
bool should_unback[kNumHugePages] = {};
const int start = adaptive_release ? kNumHugePages - 1 : 0;
const int end = adaptive_release ? -1 : kNumHugePages;
const int step = adaptive_release ? -1 : 1;

// TODO(b/73749855): Consider optimizing this search by consulting tracker_.
for (int i = start; i != end; i += step) {
if (backed_[i] && pages_used_[i] == Length(0)) {
should_unback[i] = true;
++release_target;
}

if (release_target.in_pages() >= to_release) break;
size_t index = adaptive_release ? kNumHugePages : 0, n;
while (released < needed &&
(adaptive_release ? unreleasable_.PrevFreeRange(index, &index, &n)
: unreleasable_.NextFreeRange(index, &index, &n))) {
const size_t take = std::min(n, needed - released);
const size_t first = adaptive_release ? index + n - take : index;
std::fill_n(should_unback + first, take, true);
released += take;
if (!adaptive_release) index += n;
}
TC_ASSERT_EQ(released, needed);
return UnbackHugepages(should_unback);
}

Expand Down Expand Up @@ -466,6 +475,7 @@ inline void HugeRegion::Inc(Range r, bool* from_released) {
if (backed_[i]) {
TC_ASSERT_GT(free_backed_count_, NHugePages(0));
--free_backed_count_;
unreleasable_.SetBit(i);
} else {
backed_[i] = true;
should_back = true;
Expand Down Expand Up @@ -494,6 +504,7 @@ inline void HugeRegion::Dec(Range r, bool release) {
if (pages_used_[i] == Length(0)) {
should_unback[i] = true;
++free_backed_count_;
unreleasable_.ClearBit(i);
}
r.p += here;
r.n -= here;
Expand Down Expand Up @@ -522,10 +533,8 @@ inline HugeLength HugeRegion::UnbackHugepages(

// Temporarily block allocations to these pages.
//
// We both Mark and toggle pages_used_, as allocations use FindAndMark but
// Release only uses pages_used_.
//
// TODO(b/73749855): Optimize release by consulting the bitmap first.
// We both Mark and toggle pages_used_, as allocations use FindAndMark.
// Release consults unreleasable_, which is set below.
tracker_.Mark(NHugePages(i).in_pages().raw_num(), hl.in_pages().raw_num());
Length used;
for (size_t k = i; k != j; ++k) {
Expand All @@ -537,6 +546,7 @@ inline HugeLength HugeRegion::UnbackHugepages(
// middle of being unbacked and are not eligible for allocation/release.
TC_ASSERT_GE(free_backed_count_, hl);
free_backed_count_ -= hl;
unreleasable_.SetRange(i, j - i);

if (ABSL_PREDICT_TRUE(unback_(HugeRange(p, hl)).success)) {
nbacked_ -= hl;
Expand All @@ -551,6 +561,7 @@ inline HugeLength HugeRegion::UnbackHugepages(
} else {
// Restore the count if unback failed.
free_backed_count_ += hl;
unreleasable_.ClearRange(i, j - i);
}

used = Length(0);
Expand Down
48 changes: 48 additions & 0 deletions tcmalloc/huge_region_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -315,6 +315,54 @@ TEST_F(HugeRegionTest, ReleaseAdaptive) {
CheckMock();
}

// Release skips in-use hugepages between free ones, in both directions.
TEST_F(HugeRegionTest, ReleaseFragmented) {
for (const bool adaptive_release : {false, true}) {
SCOPED_TRACE(adaptive_release);
const Length n = kPagesPerHugePage;
bool from_released;
std::optional<Alloc> allocs[8];
for (int i = 0; i < 8; ++i) {
allocs[i] = Allocate(n, &from_released);
EXPECT_TRUE(from_released);
}

// Delete hugepages 0, 1, 3, 4, 6, 7 while keeping 2 and 5 allocated, so
// the free, backed hugepages are [0, 1], [3, 4], [6, 7].
for (int i : {0, 1, 3, 4, 6, 7}) {
Delete(*allocs[i]);
}

if (adaptive_release) {
// Reverse order releases 7, 6 from the last range, then 4.
ExpectUnback({p_ + NHugePages(4), NHugePages(1)});
ExpectUnback({p_ + NHugePages(6), NHugePages(2)});
} else {
// Forward order releases 0, 1 from the first range, then 3.
ExpectUnback({p_, NHugePages(2)});
ExpectUnback({p_ + NHugePages(3), NHugePages(1)});
}
EXPECT_EQ(NHugePages(3),
region_.Release(NHugePages(3).in_pages(), adaptive_release));
CheckMock();

// Release the remaining five backed hugepages so the next iteration starts
// from an empty, unbacked region.
Delete(*allocs[2]);
Delete(*allocs[5]);
if (adaptive_release) {
ExpectUnback({p_, NHugePages(4)});
ExpectUnback({p_ + NHugePages(5), NHugePages(1)});
} else {
ExpectUnback({p_ + NHugePages(2), NHugePages(1)});
ExpectUnback({p_ + NHugePages(4), NHugePages(4)});
}
EXPECT_EQ(NHugePages(5),
region_.Release(NHugePages(8).in_pages(), adaptive_release));
CheckMock();
}
}

TEST_F(HugeRegionTest, ReleaseFailure) {
const Length n = kPagesPerHugePage;
bool from_released;
Expand Down
20 changes: 20 additions & 0 deletions tcmalloc/internal/range_tracker.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,12 @@ class Bitmap {
// If there is at least one free range at or after <start>,
// put it in *index, *length and return true; else return false.
bool NextFreeRange(size_t start, size_t* index, size_t* length) const;
// If there is at least one free range ending at or before <end>,
// put it in *index, *length and return true; else return false.
// To iterate backwards through all free ranges:
// size_t index = bitmap.size(), n;
// while (bitmap.PrevFreeRange(index, &index, &n)) { ... }
bool PrevFreeRange(size_t end, size_t* index, size_t* length) const;

// Returns index of the first {true, false} bit >= index, or N if none.
size_t FindSet(size_t index) const;
Expand Down Expand Up @@ -517,6 +523,20 @@ inline bool Bitmap<N>::NextFreeRange(size_t start, size_t* index,
return true;
}

template <size_t N>
inline bool Bitmap<N>::PrevFreeRange(size_t end, size_t* index,
size_t* length) const {
if (end == 0) return false;
if (end > N) end = N;
const ssize_t last = FindClearBackwards(end - 1);
if (last < 0) return false;
// FindSetBackwards returns -1 if the free range extends to bit 0.
const size_t first = FindSetBackwards(last) + 1;
*index = first;
*length = last + 1 - first;
return true;
}

template <size_t N>
inline size_t Bitmap<N>::FindSet(size_t index) const {
return FindValue<true>(index);
Expand Down
73 changes: 73 additions & 0 deletions tcmalloc/internal/range_tracker_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -394,6 +394,79 @@ TEST_F(BitmapTest, PopBatch) {
EXPECT_TRUE(map.IsZero());
}

TEST_F(BitmapTest, PrevFreeRange) {
// Empty bitmap (all free).
{
Bitmap<253> map;
size_t index = map.size(), len = 0;
EXPECT_TRUE(map.PrevFreeRange(index, &index, &len));
EXPECT_EQ(index, 0);
EXPECT_EQ(len, 253);
EXPECT_FALSE(map.PrevFreeRange(index, &index, &len));
}

// Full bitmap (no free ranges).
{
Bitmap<253> map;
map.SetRange(0, 253);
size_t index = map.size(), len = 0;
EXPECT_FALSE(map.PrevFreeRange(index, &index, &len));
}

// Bounds and edge conditions.
{
Bitmap<64> map;
size_t index = 0, len = 0;
EXPECT_FALSE(map.PrevFreeRange(0, &index, &len));

// end > N clamps to N.
EXPECT_TRUE(map.PrevFreeRange(100, &index, &len));
EXPECT_EQ(index, 0);
EXPECT_EQ(len, 64);

// end inside a free range truncates it.
EXPECT_TRUE(map.PrevFreeRange(30, &index, &len));
EXPECT_EQ(index, 0);
EXPECT_EQ(len, 30);

// A free range ending at bit 0.
map.SetRange(1, 63);
EXPECT_TRUE(map.PrevFreeRange(64, &index, &len));
EXPECT_EQ(index, 0);
EXPECT_EQ(len, 1);
EXPECT_FALSE(map.PrevFreeRange(index, &index, &len));
}

// Multiple ranges, including ones straddling word boundaries, matching
// forward traversal in reverse.
{
Bitmap<253> map;
map.SetRange(0, 253);
// Open free ranges: [10, 20), [50, 80), [120, 150), [200, 253)
map.ClearRange(10, 10);
map.ClearRange(50, 30);
map.ClearRange(120, 30);
map.ClearRange(200, 53);

std::vector<std::pair<size_t, size_t>> forward_ranges;
size_t f_index = 0, f_len;
while (map.NextFreeRange(f_index, &f_index, &f_len)) {
forward_ranges.push_back({f_index, f_len});
f_index += f_len;
}
EXPECT_THAT(forward_ranges, ElementsAre(Pair(10, 10), Pair(50, 30),
Pair(120, 30), Pair(200, 53)));

std::vector<std::pair<size_t, size_t>> backward_ranges;
size_t b_index = map.size(), b_len;
while (map.PrevFreeRange(b_index, &b_index, &b_len)) {
backward_ranges.push_back({b_index, b_len});
}
EXPECT_THAT(backward_ranges, ElementsAre(Pair(200, 53), Pair(120, 30),
Pair(50, 30), Pair(10, 10)));
}
}

class RangeTrackerTest : public ::testing::Test {
protected:
std::vector<std::pair<size_t, size_t>> FreeRanges() {
Expand Down
Loading