Memory exhaustion / Denial of Service (resource exhaustion)

Go PoC (demonstrates potential memory growth without the cap, illustrating how an attacker could exhaust memory by flooding with new, distinct objects): // Pseudo-code illustrating the vulnerability scenario prior to the fix. // This is not a full, runnable program but shows the exploit pattern against the ingester. package main import ( "fmt" "runtime" // Grafana internal packages are assumed to be importable in a real environment // "github.com/grafana/grafana/pkg/storage/unified/resource/usagestats" // "github.com/grafana/grafana/pkg/storage" ) func main() { // Setup: create a storage-backed ingester with a very small cap for demonstration // ing, _ := NewIngester(IngesterOptions{ // Store: store, // Leases: newTestLeases(t), // Reg: prometheus.NewRegistry(), // Now: fixedNow("2026-06-23"), // MaxBufferedObjects: 1, // }) // Pseudo: simulate flooding with distinct objects beyond the cap maxBuffered := 1 // would be configured via ingester option in real code // e.g., flood with 1000 distinct object keys for i := 0; i < 1000; i++ { key := fmt.Sprintf("obj-%d", i) // In real code: ing.mergeBack(objectRefFromKey(dashKey(key)), map[string]uint64{"views": 1}) // Here we just illustrate the pattern _ = key } // Observables to demonstrate impact (memory growth, not a real assertion) var m runtime.MemStats runtime.GC() runtime.ReadMemStats(&m) fmt.Printf("Allocated bytes after flood: %d\n", m.Alloc) _ = maxBuffered } Notes: - In a real world scenario, a flood of new, distinct object keys would cause the ingester to create new buffer entries up to the cap. Without the fix, this could lead to unbounded memory growth. With the fix, new objects beyond the cap are dropped and their deltas counted as dropped events, preventing memory exhaustion.

diff --git a/pkg/storage/unified/resource/usagestats/ingester.go b/pkg/storage/unified/resource/usagestats/ingester.go index 9782494ab5aa..cf6f956fc43c 100644 --- a/pkg/storage/unified/resource/usagestats/ingester.go +++ b/pkg/storage/unified/resource/usagestats/ingester.go @@ -200,6 +200,14 @@ func (i *Ingester) mergeBack(o objectRef, deltas map[string]uint64) { defer i.mu.Unlock() cur, ok := i.buffer[o] if !ok { + if len(i.buffer) >= i.maxBufferedObjects { + var dropped uint64 + for _, v := range deltas { + dropped += v + } + i.metrics.dropEvents(reasonBufferFull, int(dropped)) + return + } cur = map[string]uint64{} i.buffer[o] = cur } diff --git a/pkg/storage/unified/resource/usagestats/ingester_test.go b/pkg/storage/unified/resource/usagestats/ingester_test.go index ccfd3d85658d..77c91b94bb12 100644 --- a/pkg/storage/unified/resource/usagestats/ingester_test.go +++ b/pkg/storage/unified/resource/usagestats/ingester_test.go @@ -245,6 +245,36 @@ func TestIngesterBufferFull(t *testing.T) { }) } +func TestIngesterMergeBackHonorsBufferCap(t *testing.T) { + forEachBackend(t, func(t *testing.T, store *Store) { + ing, err := NewIngester(IngesterOptions{ + Store: store, + Leases: newTestLeases(t), + Reg: prometheus.NewRegistry(), + Now: fixedNow("2026-06-23"), + MaxBufferedObjects: 1, + }) + require.NoError(t, err) + + a := objectRefFromKey(dashKey("a")) + b := objectRefFromKey(dashKey("b")) + + // A new object rebuffered into an empty buffer is accepted. + ing.mergeBack(a, map[string]uint64{"views": 2}) + require.Len(t, ing.buffer, 1) + + // A second distinct object exceeds the cap and is dropped, with its + // deltas counted as dropped events. + ing.mergeBack(b, map[string]uint64{"views": 3, "queries": 1}) + require.Len(t, ing.buffer, 1) + require.Equal(t, float64(4), testutil.ToFloat64(ing.metrics.droppedEvents.WithLabelValues(reasonBufferFull))) + + // Already-buffered objects can still accumulate even when full. + ing.mergeBack(a, map[string]uint64{"views": 5}) + require.Equal(t, uint64(7), ing.buffer[a]["views"]) + }) +} + func TestIngesterGetResourceDailyStats(t *testing.T) { forEachBackend(t, func(t *testing.T, store *Store) { ctx := context.Background()