Add defaulted-off option to gate madvise'ing sampled allocations.

tcmalloc/allocation_sampling.h

@@ -25,6 +25,8 @@
 #include "tcmalloc/internal/config.h"
 #include "tcmalloc/internal/logging.h"
 #include "tcmalloc/internal/percpu.h"
+#include "tcmalloc/internal/residency.h"
+#include "tcmalloc/internal/util.h"
 #include "tcmalloc/malloc_extension.h"
 #include "tcmalloc/malloc_hook.h"
 #include "tcmalloc/malloc_hook_invoke.h"
@@ -170,12 +172,48 @@ ABSL_ATTRIBUTE_NOINLINE sized_ptr_t SampleifyAllocation(
   // A span must be provided or created by this point.
   TC_ASSERT_NE(span, nullptr);
 
+  // Do not madvise guarded (GWP-ASan) allocations: GWP-ASan initializes magic
+  // canary bytes in the allocated page to detect buffer overflows on
+  // deallocation; releasing memory zeroes the page and corrupts the canaries.
+  if (Parameters::madvise_sampled_allocations() ==
+          MadviseSampledAllocations::kEnabled &&
+      alloc_with_status.status != Profile::Sample::GuardedStatus::Guarded) {
+    switch (GetMemoryTag(span->start_address())) {
+      case MemoryTag::kSampled:
+      case MemoryTag::kSampledP1:
+      case MemoryTag::kCold: {
+        // TODO(b/540945006): Reconsider whether to skip the first page.
+        const uintptr_t hardware_page_size = GetPageSize();
+        uintptr_t start = reinterpret_cast<uintptr_t>(span->start_address());
+        uintptr_t length = span->bytes_in_span();
+        if (length <= hardware_page_size) {
+          break;
+        }
+        start += hardware_page_size;
+        length -= hardware_page_size;
+
+        (void)state.system_allocator().Release(reinterpret_cast<void*>(start),
+                                               length);
+        break;
+      }
+      case MemoryTag::kNormal:
+      case MemoryTag::kNormalP1:
+      case MemoryTag::kMetadata:
+        break;
+    }
+  }
+
   // TODO(b/414876446): Add entropy to the handles generated.
   stack_trace.sampled_alloc_handle =
       AllocHandle(state.sampled_alloc_handle_generator.fetch_add(
                       1, std::memory_order_relaxed) +
                   1);
-  stack_trace.span_start_address = span->start_address();
+  // For guarded allocations under large page sizes, span->start_address()
+  // rounds down to a PROT_NONE guard page; record the object address instead
+  // so residency queries (e.g. mincore) inspect the accessible page.
+  stack_trace.span_start_address = (alloc_with_status.alloc != nullptr)
+                                       ? alloc_with_status.alloc
+                                       : span->start_address();
   stack_trace.allocation_time = absl::Now();
   stack_trace.guarded_status = alloc_with_status.status;
   stack_trace.allocation_type = policy.allocation_type();
@@ -195,9 +233,7 @@ ABSL_ATTRIBUTE_NOINLINE sized_ptr_t SampleifyAllocation(
       .weight = allocation_estimate,
       .stack = absl::MakeSpan(stack_trace.stack, stack_trace.depth),
       .allocation_time = stack_trace.allocation_time,
-      .ptr = (alloc_with_status.alloc != nullptr)
-                 ? alloc_with_status.alloc
-                 : stack_trace.span_start_address,
+      .ptr = stack_trace.span_start_address,
       .access_hint = stack_trace.access_hint,
       .access_allocated = stack_trace.cold_allocated ? MallocHook::Access::Cold
                                                      : MallocHook::Access::Hot,
@@ -213,6 +249,7 @@ ABSL_ATTRIBUTE_NOINLINE sized_ptr_t SampleifyAllocation(
   // heap profile, and won't need any information from Span::Sample() next.
   SampledAllocation* sampled_allocation =
       state.sampled_allocation_recorder().Register(std::move(stack_trace));
+
   // No pageheap_lock required. The span is freshly allocated and no one else
   // can access it. It is visible after we return from this allocation path.
   span->Sample(sampled_allocation);

tcmalloc/global_stats.cc

@@ -637,6 +637,9 @@ void DumpStats(Printer& out, int level) {
                        MadviseRegionsNoHugepage::kEnabled
                    ? 1
                    : 0);
+    out.printf("PARAMETER tcmalloc_madvise_sampled_allocations %d\n",
+               Parameters::madvise_sampled_allocations() ==
+                   MadviseSampledAllocations::kEnabled);
     out.printf("PARAMETER tcmalloc_use_wider_slabs %d\n",
                tc_globals.cpu_cache().UseWiderSlabs() ? 1 : 0);
     out.printf("PARAMETER heap_partitioning %d\n",
@@ -923,6 +926,9 @@ void DumpStatsInPbtxt(Printer& out, int level) {
   region.PrintBool("subrelease_unbacked_hugepages",
                    Parameters::subrelease_unbacked_hugepages() ==
                        SubreleaseUnbackedMode::kEnabled);
+  region.PrintBool("tcmalloc_madvise_sampled_allocations",
+                   Parameters::madvise_sampled_allocations() ==
+                       MadviseSampledAllocations::kEnabled);
 
   region.PrintBool("back_small_allocations",
                    Parameters::back_small_allocations());

tcmalloc/internal/parameter_accessors.h

@@ -35,6 +35,8 @@ struct TracerSizeClassInfo {
   size_t num_objects_to_move;
 };
 
+enum class MadviseSampledAllocations : bool { kDisabled, kEnabled };
+
 }  // namespace tcmalloc_internal
 }  // namespace tcmalloc
 
@@ -92,6 +94,11 @@ TCMalloc_Internal_SetHugePageFillerSkipSubreleaseLongInterval(absl::Duration v);
 ABSL_ATTRIBUTE_WEAK bool TCMalloc_Internal_GetMadviseColdRegionsNoHugepage();
 ABSL_ATTRIBUTE_WEAK void TCMalloc_Internal_SetMadviseColdRegionsNoHugepage(
     bool v);
+[[nodiscard]] ABSL_ATTRIBUTE_WEAK
+    tcmalloc::tcmalloc_internal::MadviseSampledAllocations
+    TCMalloc_Internal_GetMadviseSampledAllocations();
+ABSL_ATTRIBUTE_WEAK void TCMalloc_Internal_SetMadviseSampledAllocations(
+    tcmalloc::tcmalloc_internal::MadviseSampledAllocations v);
 ABSL_ATTRIBUTE_WEAK int64_t TCMalloc_Internal_GetEventTraceMemoryLimit();
 ABSL_ATTRIBUTE_WEAK void TCMalloc_Internal_SetEventTraceMemoryLimit(int64_t v);
 ABSL_ATTRIBUTE_WEAK uint8_t TCMalloc_Internal_GetMinHotAccessHint();

tcmalloc/parameters.cc

@@ -231,6 +231,9 @@ ABSL_CONST_INIT std::atomic<int32_t> Parameters::back_size_threshold_bytes_(
 ABSL_CONST_INIT std::atomic<bool> Parameters::enable_unfiltered_collapse_(
     false);
 ABSL_CONST_INIT std::atomic<bool> Parameters::release_max_cold_pages_(false);
+ABSL_CONST_INIT std::atomic<MadviseSampledAllocations>
+    Parameters::madvise_sampled_allocations_(
+        MadviseSampledAllocations::kDisabled);
 ABSL_CONST_INIT std::atomic<int64_t> Parameters::event_trace_memory_limit_(
     16 << 20);
 ABSL_CONST_INIT
@@ -368,6 +371,7 @@ static bool want_disable_dynamic_slabs() {
 }  // namespace tcmalloc_internal
 }  // namespace tcmalloc
 
+using tcmalloc::tcmalloc_internal::MadviseSampledAllocations;
 using tcmalloc::tcmalloc_internal::Parameters;
 using tcmalloc::tcmalloc_internal::tc_globals;
 
@@ -663,6 +667,15 @@ void TCMalloc_Internal_SetMadviseColdRegionsNoHugepage(bool v) {
       std::memory_order_relaxed);
 }
 
+MadviseSampledAllocations TCMalloc_Internal_GetMadviseSampledAllocations() {
+  return Parameters::madvise_sampled_allocations();
+}
+
+void TCMalloc_Internal_SetMadviseSampledAllocations(
+    MadviseSampledAllocations v) {
+  Parameters::madvise_sampled_allocations_.store(v, std::memory_order_relaxed);
+}
+
 int64_t TCMalloc_Internal_GetEventTraceMemoryLimit() {
   return Parameters::event_trace_memory_limit();
 }

tcmalloc/parameters.h

@@ -148,6 +148,14 @@ class Parameters {
     TCMalloc_Internal_SetMadviseColdRegionsNoHugepage(value);
   }
 
+  [[nodiscard]] static MadviseSampledAllocations madvise_sampled_allocations() {
+    return madvise_sampled_allocations_.load(std::memory_order_relaxed);
+  }
+
+  static void set_madvise_sampled_allocations(MadviseSampledAllocations value) {
+    TCMalloc_Internal_SetMadviseSampledAllocations(value);
+  }
+
   static int64_t event_trace_memory_limit() {
     return event_trace_memory_limit_.load(std::memory_order_relaxed);
   }
@@ -256,6 +264,8 @@ class Parameters {
   friend void ::TCMalloc_Internal_SetEnableUnfilteredCollapse(bool v);
   friend void ::TCMalloc_Internal_SetHugeRegionAdaptiveReleaseEnabled(bool v);
   friend void ::TCMalloc_Internal_SetReleaseMaxColdPages(bool v);
+  friend void ::TCMalloc_Internal_SetMadviseSampledAllocations(
+      MadviseSampledAllocations v);
   friend void ::TCMalloc_Internal_SetEventTraceMemoryLimit(int64_t v);
   friend void ::TCMalloc_Internal_SetReleaseDrainedSlabMetadata(bool v);
 
@@ -278,6 +288,7 @@ class Parameters {
   static std::atomic<int32_t> back_size_threshold_bytes_;
   static std::atomic<bool> enable_unfiltered_collapse_;
   static std::atomic<bool> release_max_cold_pages_;
+  static std::atomic<MadviseSampledAllocations> madvise_sampled_allocations_;
   static std::atomic<int64_t> event_trace_memory_limit_;
   static std::atomic<bool> release_drained_slab_metadata_;
 };

tcmalloc/testing/BUILD

@@ -52,6 +52,7 @@ cc_library(
     deps = [
         "//tcmalloc:malloc_extension",
         "//tcmalloc/internal:logging",
+        "//tcmalloc/internal:parameter_accessors",
         "//tcmalloc/internal:percpu",
         "@com_github_google_benchmark//:benchmark",
         "@com_google_absl//absl/base:core_headers",
@@ -1011,13 +1012,17 @@ create_tcmalloc_testsuite(
     copts = TCMALLOC_DEFAULT_COPTS,
     deps = [
         ":test_allocator_harness",
+        ":testutil",
         ":thread_manager",
         "//tcmalloc:malloc_extension",
         "//tcmalloc:malloc_hook",
         "//tcmalloc/internal:config",
         "//tcmalloc/internal:logging",
+        "//tcmalloc/internal:memory_tag",
+        "//tcmalloc/internal:page_size",
         "//tcmalloc/internal:profile_builder",
         "//tcmalloc/internal:profile_cc_proto",
+        "//tcmalloc/internal:residency",
         "//tcmalloc/internal:sampled_allocation",
         "@com_google_absl//absl/base",
         "@com_google_absl//absl/base:core_headers",

tcmalloc/testing/get_stats_test.cc

@@ -40,6 +40,7 @@
 namespace tcmalloc {
 namespace {
 
+using tcmalloc_internal::MadviseSampledAllocations;
 using tcmalloc_internal::Parameters;
 using ::testing::AnyOf;
 using ::testing::ContainsRegex;
@@ -186,6 +187,7 @@ TEST_F(GetStatsTest, Parameters) {
   const absl::Duration old_skip_subrelease_long =
       Parameters::filler_skip_subrelease_long_interval();
   Parameters::set_filler_skip_subrelease_long_interval(absl::Seconds(3));
+  ScopedMadviseSampledAllocations s(MadviseSampledAllocations::kDisabled);
 
   auto using_hpaa = [](absl::string_view sv) {
     return absl::StrContains(sv, "HugePageAwareAllocator");
@@ -258,11 +260,15 @@ TEST_F(GetStatsTest, Parameters) {
       EXPECT_THAT(buf,
                   HasSubstr(R"(PARAMETER madvise_cold_regions_nohugepage 0)"));
     }
+    EXPECT_THAT(
+        buf, HasSubstr(R"(PARAMETER tcmalloc_madvise_sampled_allocations 0)"));
     if (using_hpaa(buf)) {
       EXPECT_THAT(buf, HasSubstr(R"(using_hpaa_subrelease: false)"));
     }
 
     EXPECT_THAT(pbtxt, HasSubstr(R"(guarded_sample_parameter: -1)"));
+    EXPECT_THAT(pbtxt,
+                HasSubstr(R"(tcmalloc_madvise_sampled_allocations: false)"));
 #ifdef TCMALLOC_DEPRECATED_PERTHREAD
     EXPECT_THAT(pbtxt, HasSubstr(R"(tcmalloc_per_cpu_caches: false)"));
 #endif  // TCMALLOC_DEPRECATED_PERTHREAD
@@ -307,6 +313,8 @@ TEST_F(GetStatsTest, Parameters) {
   Parameters::set_filler_skip_subrelease_long_interval(
       absl::Milliseconds(180375));
   Parameters::set_min_hot_access_hint(hot_cold_t{3});
+  Parameters::set_madvise_sampled_allocations(
+      MadviseSampledAllocations::kEnabled);
 
   buf = MallocExtension::GetStats();
   pbtxt = GetStatsInPbTxt();
@@ -335,6 +343,8 @@ TEST_F(GetStatsTest, Parameters) {
         buf,
         HasSubstr(
             R"(PARAMETER tcmalloc_skip_subrelease_long_interval 3m0.375s)"));
+    EXPECT_THAT(
+        buf, HasSubstr(R"(PARAMETER tcmalloc_madvise_sampled_allocations 1)"));
 
     if (using_hpaa(buf)) {
       EXPECT_THAT(pbtxt, HasSubstr(R"(using_hpaa_subrelease: true)"));
@@ -356,6 +366,8 @@ TEST_F(GetStatsTest, Parameters) {
         HasSubstr(
             R"(tcmalloc_skip_subrelease_long_interval_ns: 180375000000)"));
     EXPECT_THAT(pbtxt, HasSubstr(R"(min_hot_access_hint: 3)"));
+    EXPECT_THAT(pbtxt,
+                HasSubstr(R"(tcmalloc_madvise_sampled_allocations: true)"));
   }
 
   Parameters::set_hpaa_subrelease(old_hpaa_subrelease);