From 2a02559cbeb728d707b97af7c28f18092375844b Mon Sep 17 00:00:00 2001
From: Chris Kennelly CA <ckennelly@google.com>
Date: Thu, 17 Sep 2026 20:09:49 -0700
Subject: [PATCH] Split FreeWithHooksOrPerThread into per-thread and hooked
 free slow paths.

FreeSmallSlow routed both "delete hooks installed" and "per-CPU caches
inactive" into one NOINLINE callee. That callee unconditionally built a
DeleteInfo (class_to_size load + 4 stack stores), tested delete_hooks_, then
re-tested CpuCacheActive before falling into the inlined ThreadCache::
Deallocate. Per-thread-mode binaries therefore paid the hook plumbing on
every free, and the hooked per-CPU path carried the ThreadCache body in its
frame (3 callee-saved pushes, 0x38 stack).

Dispatch directly from FreeSmallSlow instead:
 - HaveHooks()          -> FreeSmallHooked (hook + DeallocateSlow, or the
                           per-thread callee if per-CPU is inactive),
 - !UsePerCpuCache()    -> FreeSmallPerThread (ThreadCache::Deallocate /
                           transfer cache only, no hook plumbing),
 - otherwise            -> inlined DeallocateSlowNoHooks (unchanged).

Both are tail calls out of FreeSmallSlow; its hot per-CPU body is unchanged
(register renaming only). Routing still keys on HaveHooks() rather than
delete_hooks_.empty() because new-hook-only installs still need
DeallocateSlow -> MaybeForceSlowPath to uncache the slab. Sampled-object
handling is untouched. The FreeSmall inline body and fast_path goldens are
byte-identical.

x86-64 -c opt, per-thread free (thread cache present, no overflow), callee
after FreeSmallSlow's dispatch:
  before FreeWithHooksOrPerThread: 51 insns, 15 loads, 10 stores, 4 pushes
  after  FreeSmallPerThread:       32 insns,  9 loads,  5 stores, 1 push
Hooked per-CPU path: same instruction sequence, one fewer ptr spill/reload;
function shrinks 402 -> 249 bytes. FreeSmallSlow +16 bytes (second tail
call).

PiperOrigin-RevId: 983583456
---
 tcmalloc/tcmalloc.cc | 38 +++++++++++++++++++++++++-------------
 1 file changed, 25 insertions(+), 13 deletions(-)

diff --git a/tcmalloc/tcmalloc.cc b/tcmalloc/tcmalloc.cc
index e4a38a796..c454518cb 100644
--- a/tcmalloc/tcmalloc.cc
+++ b/tcmalloc/tcmalloc.cc
@@ -572,16 +572,12 @@ inline SizeAndSampled GetSizeAndSampled(const void* ptr) {
 
 inline size_t GetSize(const void* ptr) { return GetSizeAndSampled(ptr).size; }
 
-// This slow path also handles delete hooks and non-per-cpu mode.
-ABSL_ATTRIBUTE_NOINLINE static void FreeWithHooksOrPerThread(
-    void* ptr, std::optional<size_t> size, size_t size_class) {
-  MallocHook::InvokeDeleteHook({ptr, size,
-                                tc_globals.sizemap().class_to_size(size_class),
-                                HookMemoryMutable::kMutable});
-  if (ABSL_PREDICT_TRUE(UsePerCpuCache(tc_globals))) {
-    tc_globals.cpu_cache().DeallocateSlow(ptr, size_class);
-  } else if (ThreadCache* cache = ThreadCache::GetCacheIfPresent();
-             ABSL_PREDICT_TRUE(cache)) {
+// This slow path handles non-per-cpu mode.  It is kept out-of-line so that the
+// per-CPU slow path does not carry the ThreadCache::Deallocate body.
+ABSL_ATTRIBUTE_NOINLINE static void FreeSmallPerThread(void* ptr,
+                                                       size_t size_class) {
+  if (ThreadCache* cache = ThreadCache::GetCacheIfPresent();
+      ABSL_PREDICT_TRUE(cache)) {
     cache->Deallocate(ptr, size_class);
   } else {
     // This thread doesn't have thread-cache yet or already. Delete directly
@@ -591,6 +587,20 @@ ABSL_ATTRIBUTE_NOINLINE static void FreeWithHooksOrPerThread(
   }
 }
 
+// This slow path also handles delete hooks.
+ABSL_ATTRIBUTE_NOINLINE static void FreeSmallHooked(void* ptr,
+                                                    std::optional<size_t> size,
+                                                    size_t size_class) {
+  MallocHook::InvokeDeleteHook({ptr, size,
+                                tc_globals.sizemap().class_to_size(size_class),
+                                HookMemoryMutable::kMutable});
+  if (ABSL_PREDICT_TRUE(UsePerCpuCache(tc_globals))) {
+    tc_globals.cpu_cache().DeallocateSlow(ptr, size_class);
+  } else {
+    FreeSmallPerThread(ptr, size_class);
+  }
+}
+
 // In free fast-path we handle a number of conditions (delete hooks,
 // full cpu cache, uncached per-cpu slab pointer, etc) by delegating work to
 // slower function that handles all of these cases. This is done so that free
@@ -599,9 +609,11 @@ ABSL_ATTRIBUTE_NOINLINE static void FreeWithHooksOrPerThread(
 ABSL_ATTRIBUTE_NOINLINE static void FreeSmallSlow(void* ptr,
                                                   std::optional<size_t> size,
                                                   size_t size_class) {
-  if (ABSL_PREDICT_FALSE(Static::HaveHooks()) ||
-      ABSL_PREDICT_FALSE(!UsePerCpuCache(tc_globals))) {
-    return FreeWithHooksOrPerThread(ptr, size, size_class);
+  if (ABSL_PREDICT_FALSE(Static::HaveHooks())) {
+    return FreeSmallHooked(ptr, size, size_class);
+  }
+  if (ABSL_PREDICT_FALSE(!UsePerCpuCache(tc_globals))) {
+    return FreeSmallPerThread(ptr, size_class);
   }
   TCMALLOC_ALWAYS_INLINE_CALL tc_globals.cpu_cache().DeallocateSlowNoHooks(
       ptr, size_class);