Information Disclosure

Proof-of-concept (exploitation potential before fix): - Context: In Grafana’s Unified Storage Vector, resources can be embedded into the vector and surfaced via backfill/reconciler paths. A resource marked with a pending-delete label could still be embedded and exposed, leaking its metadata/content to users who can query the vector. The patch introduces a HasPendingDeleteLabel check to skip such resources. Assumptions: - An attacker can create or label a resource with the pending-delete label and trigger the backfill/reconciler process (or exploit an already-enqueued resource). - The attacker has access to fetch the published/embedder output (e.g., via search API/vector export). Before fix (attack vc): - A resource (dashboard) named dash-a with a pending-delete label would be processed by backfill/reconciler and embedded into the published vector along with dash-b (unlabeled). - The attacker could retrieve the vector and see dash-a’s metadata and content, despite it being scheduled for deletion. After fix (mitigation): - The backfill/reconciler code checks HasPendingDeleteLabel(iter.Value()) and skips embedding resources that have the pending-delete label. As a result, dash-a is not embedded, and only dash-b is published. Minimal Go-based PoC (illustrative, not runnable against a live system): package main import ( "fmt" "github.com/grafana/grafana/apps/provisioning/pkg/controller" "github.com/grafana/grafana/pkg/storage/unified/search/embed" ) func main() { // Simulated JSON payloads for two dashboards pendingJSON := []byte(`{"metadata":{"name":"dash-a","labels":{"` + controller.LabelPendingDelete + `":"true"}},"spec":{"uid":"dash-a","title":"Dash A"}}`) regularJSON := []byte(`{"metadata":{"name":"dash-b","labels":{}},"spec":{"uid":"dash-b","title":"Dash B"}}`) // HasPendingDeleteLabel should detect the pending-delete label fmt.Println("pending detected?", embed.HasPendingDeleteLabel(pendingJSON)) // expected: true fmt.Println("regular pending?", embed.HasPendingDeleteLabel(regularJSON)) // expected: false // In pre-fix behavior, the backfill/embedder would process both; post-fix, the pending item would be skipped // by the HasPendingDeleteLabel check and thus not embedded into the vector. } This demonstrates the vulnerability (pre-fix) would allow the pending-delete resource to be embedded and potentially exposed; the fix provides a clear mechanism to skip such resources during embedding, reducing information disclosure risk.

diff --git a/pkg/storage/unified/search/embed/backfill/backfill_test.go b/pkg/storage/unified/search/embed/backfill/backfill_test.go index 6983538dd2dcd..38152eb446034 100644 --- a/pkg/storage/unified/search/embed/backfill/backfill_test.go +++ b/pkg/storage/unified/search/embed/backfill/backfill_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/grafana/grafana/apps/provisioning/pkg/controller" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/testutil" "github.com/stretchr/testify/assert" @@ -509,8 +510,6 @@ func TestShouldSkipForZeroViews_EmptyName_DoesNotConsult(t *testing.T) { assert.Equal(t, 0, stats.calls) } -// fakeNonDashboardBuilder satisfies embed.Builder with a non-dashboard -// identity. Only used to exercise the builder-identity guard. type fakeNonDashboardBuilder struct{} func (fakeNonDashboardBuilder) Group() string { return "folder.grafana.app" } @@ -519,3 +518,51 @@ func (fakeNonDashboardBuilder) MaxItemsPerResource() int { return 0 } func (fakeNonDashboardBuilder) Extract(context.Context, *resourcepb.ResourceKey, []byte, string) ([]embed.Item, error) { return nil, nil } + +func labeledDashboardJSON(uid, title string) []byte { + body, _ := json.Marshal(map[string]any{ + "metadata": map[string]any{ + "name": uid, + "labels": map[string]any{controller.LabelPendingDelete: "true"}, + }, + "spec": map[string]any{"uid": uid, "title": title}, + }) + return body +} + +func makeLabeledListItem(ns, name string, rv int64) listItem { + return listItem{Namespace: ns, Name: name, RV: rv, Value: labeledDashboardJSON(name, name+"-title")} +} + +func TestRunBackfillJob_PendingDeleteLabel_SkipsLabeledResources(t *testing.T) { + storage := newFakeStorage() + storage.listItems = []listItem{ + makeLabeledListItem("ns", "dash-a", 50), + makeListItem("ns", "dash-b", 60), + } + + vec := newFakeVector() + vec.jobs = []vector.BackfillJob{{ID: 1, Model: "test-model", StoppingRV: 100}} + + o := newBackfiller(t, storage, vec) + o.runBackfill(context.Background()) + + require.Len(t, vec.upserts, 1, "only the unlabeled resource should be embedded") + assert.Equal(t, "dash-b", vec.upserts[0][0].UID) + require.Len(t, vec.completedJobIDs, 1, "job still completes when items are filtered") +} + +func TestRunBackfillJob_PendingDeleteLabel_RunsBeforeStatsLookup(t *testing.T) { + storage := newFakeStorage() + storage.listItems = []listItem{makeLabeledListItem("ns", "dash-a", 50)} + + vec := newFakeVector() + vec.jobs = []vector.BackfillJob{{ID: 1, Model: "test-model", StoppingRV: 100}} + stats := newFakeDashboardStats() + + o := newBackfillerWithStats(t, storage, vec, stats) + o.runBackfill(context.Background()) + + assert.Empty(t, vec.upserts) + assert.Equal(t, 0, stats.calls, "pending-delete skip must come before stats lookup") +} diff --git a/pkg/storage/unified/search/embed/backfill/backfiller.go b/pkg/storage/unified/search/embed/backfill/backfiller.go index 736d5048dd8a0..736f6488e30e3 100644 --- a/pkg/storage/unified/search/embed/backfill/backfiller.go +++ b/pkg/storage/unified/search/embed/backfill/backfiller.go @@ -350,6 +350,11 @@ func (b *VectorBackfiller) processBackfillItem(ctx context.Context, job vector.B return nil } + if embed.HasPendingDeleteLabel(iter.Value()) { + statusLabel = "skipped_pending_delete" + return nil + } + if b.shouldSkipForZeroViews(ctx, builder, namespace, name) { statusLabel = "skipped_zero_views" return nil diff --git a/pkg/storage/unified/search/embed/pending_delete.go b/pkg/storage/unified/search/embed/pending_delete.go new file mode 100644 index 0000000000000..8c9cef184a3e9 --- /dev/null +++ b/pkg/storage/unified/search/embed/pending_delete.go @@ -0,0 +1,22 @@ +package embed + +import ( + "encoding/json" + + "github.com/grafana/grafana/apps/provisioning/pkg/controller" +) + +func HasPendingDeleteLabel(value []byte) bool { + if len(value) == 0 { + return false + } + var obj struct { + Metadata struct { + Labels map[string]string `json:"labels"` + } `json:"metadata"` + } + if err := json.Unmarshal(value, &obj); err != nil { + return false + } + return obj.Metadata.Labels[controller.LabelPendingDelete] == "true" +} diff --git a/pkg/storage/unified/search/embed/pending_delete_test.go b/pkg/storage/unified/search/embed/pending_delete_test.go new file mode 100644 index 0000000000000..859676f4628d6 --- /dev/null +++ b/pkg/storage/unified/search/embed/pending_delete_test.go @@ -0,0 +1,42 @@ +package embed + +import ( + "encoding/json" + "testing" + + "github.com/grafana/grafana/apps/provisioning/pkg/controller" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func labeledValue(t *testing.T, pendingDelete bool) []byte { + t.Helper() + metadata := map[string]any{"name": "dash-1"} + if pendingDelete { + metadata["labels"] = map[string]any{controller.LabelPendingDelete: "true"} + } + b, err := json.Marshal(map[string]any{"metadata": metadata, "spec": map[string]any{"title": "x"}}) + require.NoError(t, err) + return b +} + +func TestHasPendingDeleteLabel(t *testing.T) { + t.Run("label present returns true", func(t *testing.T) { + assert.True(t, HasPendingDeleteLabel(labeledValue(t, true))) + }) + t.Run("label absent returns false", func(t *testing.T) { + assert.False(t, HasPendingDeleteLabel(labeledValue(t, false))) + }) + t.Run("no metadata returns false", func(t *testing.T) { + assert.False(t, HasPendingDeleteLabel([]byte(`{"spec":{"title":"x"}}`))) + }) + t.Run("empty value returns false", func(t *testing.T) { + assert.False(t, HasPendingDeleteLabel(nil)) + }) + t.Run("malformed JSON fails open to false", func(t *testing.T) { + assert.False(t, HasPendingDeleteLabel([]byte(`{not json`))) + }) + t.Run("label set to a non-true value returns false", func(t *testing.T) { + assert.False(t, HasPendingDeleteLabel([]byte(`{"metadata":{"labels":{"cloud.grafana.com/pending-delete":"false"}}}`))) + }) +} diff --git a/pkg/storage/unified/search/embed/reconciler/reconciler.go b/pkg/storage/unified/search/embed/reconciler/reconciler.go index d741eb7f5b1e7..57438877958da 100644 --- a/pkg/storage/unified/search/embed/reconciler/reconciler.go +++ b/pkg/storage/unified/search/embed/reconciler/reconciler.go @@ -669,6 +669,11 @@ func (s *Reconciler) processEvent(ctx context.Context, builder embed.Builder, ev return fmt.Errorf("unknown action %v", ev.action) } + if embed.HasPendingDeleteLabel(ev.value) { + statusLabel = "skipped_pending_delete" + return nil + } + if len(ev.value) == 0 { return nil } diff --git a/pkg/storage/unified/search/embed/reconciler/reconciler_test.go b/pkg/storage/unified/search/embed/reconciler/reconciler_test.go index 1e613f6f37380..957deae356ff7 100644 --- a/pkg/storage/unified/search/embed/reconciler/reconciler_test.go +++ b/pkg/storage/unified/search/embed/reconciler/reconciler_test.go @@ -8,6 +8,7 @@ import ( "testing" "time" + "github.com/grafana/grafana/apps/provisioning/pkg/controller" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/testutil" "github.com/stretchr/testify/assert" @@ -970,6 +971,56 @@ func TestReconciler_ProcessBatch_RequeuesOnSetLatestRVFailure(t *testing.T) { assert.Equal(t, 2, s.pendingLen(), "both events re-enqueued so the next cycle retries the advance") } +func labeledDashboard(uid, title string) []byte { + body, _ := json.Marshal(map[string]any{ + "metadata": map[string]any{ + "name": uid, + "labels": map[string]any{controller.LabelPendingDelete: "true"}, + }, + "spec": map[string]any{"uid": uid, "title": title}, + }) + return body +} + +func TestReconciler_PendingDeleteLabel_SkipsUpsertAndAdvancesCursor(t *testing.T) { + vec := newFakeVector() + s, text := newReconciler(t, &fakeStorage{}, vec) + + s.enqueue(dashEvent(resourcepb.WatchEvent_MODIFIED, "ns", "dash-1", 100, labeledDashboard("dash-1", "Dash 1"))) + s.enqueue(dashEvent(resourcepb.WatchEvent_ADDED, "ns", "dash-2", 200, minimalDashboard("dash-2", "Dash 2"))) + + s.processPending(context.Background()) + + require.Len(t, vec.upserts, 1, "only the unlabeled resource should be embedded") + assert.Equal(t, 1, text.calls, "skipped event must not call the embedder") + assert.Equal(t, int64(200), vec.latestRV, "skips count as processed so the cursor advances") + assert.Equal(t, 0, s.pendingLen(), "skipped events are not retried") +} + +func TestReconciler_PendingDeleteLabel_DeleteEventStillProcessed(t *testing.T) { + vec := newFakeVector() + s, _ := newReconciler(t, &fakeStorage{}, vec) + + s.enqueue(dashEvent(resourcepb.WatchEvent_DELETED, "ns", "dash-x", 50, nil)) + + s.processPending(context.Background()) + + require.Len(t, vec.deletes, 1, "deletes must still drop vectors regardless of labels") +} + +func TestReconciler_PendingDeleteLabel_RestoreReembeds(t *testing.T) { + vec := newFakeVector() + s, _ := newReconciler(t, &fakeStorage{}, vec) + + s.enqueue(dashEvent(resourcepb.WatchEvent_MODIFIED, "ns", "dash-1", 100, labeledDashboard("dash-1", "Dash 1"))) + s.processPending(context.Background()) + require.Empty(t, vec.upserts, "labeled resource is skipped") + + s.enqueue(dashEvent(resourcepb.WatchEvent_MODIFIED, "ns", "dash-1", 200, minimalDashboard("dash-1", "Dash 1"))) + s.processPending(context.Background()) + require.Len(t, vec.upserts, 1, "unlabeled (restored) resource embeds again") +} + // TestReconciler_Run_BroadcasterDeliversWatchEvents pins the watch // path: Subscribe is called, events pushed onto the channel reach the // queue, and the next cycle drains them.