Denial of Service / Resource Exhaustion

PoC summary: Demonstrates how an attacker could cause resource exhaustion by embedding a very large panel content (e.g., a 1 MiB SQL query) into a dashboard that gets embedded by Grafana's unified storage path. The fix truncates Content to 4 KiB and Description to 2 KiB, preventing unbounded growth of the embedded payload. Reproduction steps (pre-fix vulnerability scenario): 1) Create a dashboard payload with a single panel that includes a very large query (e.g., 1 MiB SQL in targets[].rawSql). 2) Submit this payload to Grafana's embedding/indexing flow (the Extractor in pkg/storage/unified/search/embed/dashboard/extractor.go would process the panel content). 3) Observe that the resulting embedded item Content is extremely large (potentially exhausting memory or bandwidth) and could impact service stability when many such payloads are processed. Reproduction steps (post-fix with PoC validation): 1) Construct the same dashboard payload with a giant query as above. 2) Run the embedding extractor (as used by Grafana’s storage path). The extractor now truncates content to maxItemContentBytes (4 KiB) and truncates descriptions to maxDescriptionBytes (2 KiB) with UTF-8 safe boundaries. 3) Validate that the produced item.Content length is <= 4 KiB and that descriptions are capped appropriately, while ensuring the query portion remains searchable (i.e., not entirely trimmed). Go-like pseudo-proof-of-concept (illustrative, not guaranteed to compile in this environment): package main import ( "context" "encoding/json" "fmt" "strings" ) // This is illustrative pseudo-code showing how one would craft a dashboard payload // with a giant SQL query and feed it into Grafana's extractor to observe the cap. func main() { giantSQL := strings.Repeat("A", 1<<20) // ~1 MiB body := map[string]interface{}{ "uid": "poc-dashboard", "title": "Poc Dashboard", "panels": []interface{}{ map[string]interface{}{ "id": 1, "title": "Big Query", "datasource": map[string]interface{}{"uid": "ds-1", "type": "postgres"}, "targets": []interface{}{map[string]interface{}{"refId": "A", "rawSql": giantSQL}}, }, }, } payload, _ := json.Marshal(body) // In Grafana, you would call the Extractor here, e.g.: // items, err := New().Extract(context.Background(), &resourcepb.ResourceKey{Name: "poc"}, payload, "") // For the PoC, you would then check len(items[0].Content) <= 4*1024 and utf8 validity. fmt.Println(string(payload[:50])) // small glimpse } Notes: - The actual pre-fix exploit would produce Content length equal to the size of giantSQL, potentially causing higher memory usage and processor time when embedding/content generation occurs. - The fix ensures Content length is capped at 4 KiB, and Descriptions at 2 KiB, using truncateUTF8 to avoid breaking UTF-8 characters. - Tests added in the commit (e.g., TestExtractor_CapsHugePanelContent, TestExtractor_LongDescriptionKeepsQuery, TestTruncateUTF8_RuneBoundary) validate the caps and UTF-8-safe truncation.

diff --git a/pkg/storage/unified/search/embed/dashboard/extractor.go b/pkg/storage/unified/search/embed/dashboard/extractor.go index 8d53f595bca75..a455b81ce6996 100644 --- a/pkg/storage/unified/search/embed/dashboard/extractor.go +++ b/pkg/storage/unified/search/embed/dashboard/extractor.go @@ -11,6 +11,7 @@ import ( "sort" "strconv" "strings" + "unicode/utf8" "github.com/PaesslerAG/jsonpath" "go.opentelemetry.io/otel" @@ -27,6 +28,12 @@ var tracer = otel.Tracer("github.com/grafana/grafana/pkg/storage/unified/search/ // tokens on those outliers without affecting normal dashboards. const defaultMaxPanels = 200 +// maxItemContentBytes caps a panel's embeddable text so we keep our token count per batch reasonable +const maxItemContentBytes = 4 * 1024 + +// maxDescriptionBytes cap the panel desc at 2Kib to leave room for queries +const maxDescriptionBytes = 2 * 1024 + // Extractor produces one embed.Item per panel. type Extractor struct { logger *slog.Logger @@ -103,7 +110,7 @@ func buildEmbeddableItem(content *dashboardContent, p panelContent, uid string, parts = append(parts, p.Title) } if p.Description != "" { - parts = append(parts, p.Description) + parts = append(parts, truncateUTF8(p.Description, maxDescriptionBytes)) } breadcrumb := strings.Join(parts, " → ") @@ -159,12 +166,23 @@ func buildEmbeddableItem(content *dashboardContent, p panelContent, uid string, UID: uid, Title: displayTitle(content.DashboardTitle, p.Title, uid), Subresource: subresource(p.PanelID, idx), - Content: strings.Join(sections, "\n"), + Content: truncateUTF8(strings.Join(sections, "\n"), maxItemContentBytes), Metadata: mdJSON, Folder: content.FolderUID, }, true } +// truncateUTF8 caps s to at most n bytes without splitting a rune. +func truncateUTF8(s string, n int) string { + if len(s) <= n { + return s + } + for n > 0 && !utf8.RuneStart(s[n]) { + n-- + } + return s[:n] +} + // subresource is the unique sub-identifier for a panel within its dashboard. // Mirrors grafana-assistant-app/api/internal/memory/dashboards.buildPanelVectors: // non-zero panel ID wins; otherwise fall back to positional index. diff --git a/pkg/storage/unified/search/embed/dashboard/extractor_test.go b/pkg/storage/unified/search/embed/dashboard/extractor_test.go index 7e3e5f810b3fe..6a0297e3f2ab3 100644 --- a/pkg/storage/unified/search/embed/dashboard/extractor_test.go +++ b/pkg/storage/unified/search/embed/dashboard/extractor_test.go @@ -4,7 +4,9 @@ import ( "context" "encoding/json" "os" + "strings" "testing" + "unicode/utf8" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -306,6 +308,66 @@ func TestExtractor_SQLQueries(t *testing.T) { assert.Contains(t, items[0].Content, "SELECT date, SUM(amount) FROM orders") } +func TestExtractor_CapsHugePanelContent(t *testing.T) { + // A giant query must not produce an item that overflows the provider budget. + huge := strings.Repeat("a", 1<<20) // 1 MiB + body := map[string]any{ + "uid": "huge-dash", + "title": "Huge", + "panels": []any{ + map[string]any{ + "id": 1, + "title": "Big", + "datasource": map[string]any{"uid": "pg-1", "type": "postgres"}, + "targets": []any{map[string]any{"refId": "A", "rawSql": huge}}, + }, + }, + } + value, _ := json.Marshal(body) + items, err := New().Extract(context.Background(), + &resourcepb.ResourceKey{Name: "huge-dash"}, value, "") + require.NoError(t, err) + require.Len(t, items, 1) + assert.LessOrEqual(t, len(items[0].Content), maxItemContentBytes) +} + +func TestExtractor_LongDescriptionKeepsQuery(t *testing.T) { + // A verbose panel description must not eat the whole content budget and + // crowd out the query text — query-based search still needs to match. + body := map[string]any{ + "uid": "desc-dash", + "title": "Runbooks", + "panels": []any{ + map[string]any{ + "id": 1, + "title": "Error budget", + "description": strings.Repeat("verbose runbook prose. ", 1000), // ~23 KiB + "datasource": map[string]any{"uid": "prom-1", "type": "prometheus"}, + "targets": []any{ + map[string]any{"refId": "A", "expr": "sum(rate(http_requests_total{code=~\"5..\"}[5m]))"}, + }, + }, + }, + } + value, _ := json.Marshal(body) + items, err := New().Extract(context.Background(), + &resourcepb.ResourceKey{Name: "desc-dash"}, value, "") + require.NoError(t, err) + require.Len(t, items, 1) + assert.LessOrEqual(t, len(items[0].Content), maxItemContentBytes) + assert.Contains(t, items[0].Content, "sum(rate(http_requests_total") +} + +func TestTruncateUTF8_RuneBoundary(t *testing.T) { + s := strings.Repeat("é", 10) // 2 bytes per rune + // Cap at an odd byte count that lands mid-rune; result must stay valid. + got := truncateUTF8(s, 5) + assert.LessOrEqual(t, len(got), 5) + assert.True(t, utf8.ValidString(got)) + assert.Equal(t, "éé", got) // backed up to the last whole rune + assert.Equal(t, "short", truncateUTF8("short", 100)) +} + func TestExtractor_InvalidJSON(t *testing.T) { _, err := New().Extract(context.Background(), &resourcepb.ResourceKey{Name: "bad"}, []byte(`{not json`), "")