Denial of Service (Resource Exhaustion) through quota enforcement surface

PoC (conceptual and actionable steps): Prerequisites: - Grafana deployment with Unified Storage quotas enabled. - Before patch scenario: global quota enforcement enabled (EnforceQuotas = true) with no per-resource granularity, or EnforcedQuotaResources misconfigured to cover all resources. - After patch scenario: per-resource enforcement via EnforcedQuotaResources mapped keys, e.g., {"dashboard.grafana.app/dashboards": true}. 1) Before fix (global enforcement, potential DoS): - Configure quotas so that any resource write above the limit is rejected globally. - Quota limit per resource: L (e.g., L = 1). - Attempt to create two dashboard resources in the same group/resource: bash TOKEN=<your_bearer_token> BASE=https://grafana.example.com RESOURCE='dashboards' GROUP='dashboard.grafana.app' for i in {1..5}; do curl -s -X POST "$BASE/api/unifiedstorage/write" \ -H "Authorization: Bearer $TOKEN" \ -d '{"group":"'$GROUP'","resource":"'$RESOURCE'","payload":{}}' >/dev/null done echo DONE Expected result: Once the quota limit is reached, subsequent writes fail with a quota exceeded error, potentially blocking other resources if enforcement is global. 2) After fix (per-resource enforcement): - Configure quotas with EnforcedQuotaResources including only selected resources, e.g.: enforce_quotas_resources = dashboard.grafana.app/dashboards - Attempt to write to dashboards (enforced) and to a non-enforced resource (e.g., folders): bash TOKEN=<your_bearer_token> BASE=https://grafana.example.com ENFORCED='dashboard.grafana.app/dashboards' # Enforced resource attempt (should be blocked after quota is exceeded for that resource) for i in {1..5}; do curl -s -X POST "$BASE/api/unifiedstorage/write" \ -H "Authorization: Bearer $TOKEN" \ -d '{"group":"dashboard.grafana.app","resource":"dashboards","payload":{}}' >/dev/null done # Unenforced resource attempt (should NOT be blocked by quota in this patch) for i in {1..5}; do curl -s -X POST "$BASE/api/unifiedstorage/write" \ -H "Authorization: Bearer $TOKEN" \ -d '{"group":"dashboard.grafana.app","resource":"folders","payload":{}}' >/dev/null done echo DONE Expected result: Once the quota is exhausted, only the dashboards resource is blocked; folders writes continue to succeed. This demonstrates per-resource enforcement reducing DoS blast radius. Notes: - The exact API path (/api/unifiedstorage/write) and payload schema are illustrative; adapt to your Grafana deployment endpoints. The core idea is to show that global enforcement blocks all resources, while per-resource enforcement confines blocking to the configured resources.

diff --git a/pkg/setting/setting.go b/pkg/setting/setting.go index 118584c67954b..84d3fce1e0528 100644 --- a/pkg/setting/setting.go +++ b/pkg/setting/setting.go @@ -655,7 +655,7 @@ type Cfg struct { EnableSearchClient bool OverridesFilePath string OverridesReloadInterval time.Duration - EnforceQuotas bool + EnforcedQuotaResources []string QuotasErrorMessageSupportInfo string EnableSQLKVBackend bool EnableSQLKVCompatibilityMode bool diff --git a/pkg/setting/setting_unified_storage.go b/pkg/setting/setting_unified_storage.go index 8c5a2b90a978c..a1f279415b833 100644 --- a/pkg/setting/setting_unified_storage.go +++ b/pkg/setting/setting_unified_storage.go @@ -181,7 +181,7 @@ func (cfg *Cfg) setUnifiedStorageConfig() { // quotas/limits config cfg.OverridesFilePath = section.Key("overrides_path").String() cfg.OverridesReloadInterval = section.Key("overrides_reload_period").MustDuration(30 * time.Second) - cfg.EnforceQuotas = section.Key("enforce_quotas").MustBool(false) + cfg.EnforcedQuotaResources = parseCommaSeparatedList(section.Key("enforce_quotas_resources").MustString("")) cfg.QuotasErrorMessageSupportInfo = section.Key("quotas_error_message_support_info").MustString("Please contact your administrator to increase it.") // tenant watcher @@ -279,6 +279,21 @@ func (cfg *Cfg) ShouldRunMigrations() bool { isTargetEligibleForMigrations(cfg.Target) } +func parseCommaSeparatedList(s string) []string { + if s == "" { + return nil + } + parts := strings.Split(s, ",") + result := make([]string, 0, len(parts)) + for _, p := range parts { + p = strings.TrimSpace(p) + if p != "" { + result = append(result, p) + } + } + return result +} + // UnifiedStorageType returns the configured storage type without creating or mutating keys. // Precedence: env > ini > default ("unified"). // Used to decide unified storage behavior early without side effects. diff --git a/pkg/setting/setting_unified_storage_test.go b/pkg/setting/setting_unified_storage_test.go index a3f93f751210d..f907b3b794ff7 100644 --- a/pkg/setting/setting_unified_storage_test.go +++ b/pkg/setting/setting_unified_storage_test.go @@ -284,6 +284,27 @@ func TestApplyMigrationEnforcements(t *testing.T) { }) } +func TestParseCommaSeparatedList(t *testing.T) { + tests := []struct { + name string + input string + expected []string + }{ + {name: "empty string", input: "", expected: nil}, + {name: "single value", input: "dashboard.grafana.app/dashboards", expected: []string{"dashboard.grafana.app/dashboards"}}, + {name: "multiple values", input: "dashboard.grafana.app/dashboards,folder.grafana.app/folders", expected: []string{"dashboard.grafana.app/dashboards", "folder.grafana.app/folders"}}, + {name: "with whitespace", input: " dashboard.grafana.app/dashboards , folder.grafana.app/folders ", expected: []string{"dashboard.grafana.app/dashboards", "folder.grafana.app/folders"}}, + {name: "trailing comma", input: "dashboard.grafana.app/dashboards,", expected: []string{"dashboard.grafana.app/dashboards"}}, + {name: "only commas", input: ",,", expected: []string{}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expected, parseCommaSeparatedList(tt.input)) + }) + } +} + func TestIsTargetEligibleForMigrations(t *testing.T) { tests := []struct { name string diff --git a/pkg/storage/unified/resource/quotas.go b/pkg/storage/unified/resource/quotas.go index c00ecb4c5923f..42ddd3988bd6b 100644 --- a/pkg/storage/unified/resource/quotas.go +++ b/pkg/storage/unified/resource/quotas.go @@ -19,8 +19,14 @@ import ( const DEFAULT_RESOURCE_LIMIT = 1000 type QuotasConfig struct { - EnforceQuotas bool - SupportMessage string + EnforcedResources map[string]bool // group/resource keys to enforce, e.g. {"dashboard.grafana.app/dashboards": true} + SupportMessage string +} + +// ShouldEnforce returns true if quota enforcement should block writes for +// the given group/resource pair. +func (c QuotasConfig) ShouldEnforce(group, resource string) bool { + return c.EnforcedResources[group+"/"+resource] } type QuotaExceededError struct { diff --git a/pkg/storage/unified/resource/server.go b/pkg/storage/unified/resource/server.go index d07ed6af02347..df17777440bed 100644 --- a/pkg/storage/unified/resource/server.go +++ b/pkg/storage/unified/resource/server.go @@ -1898,7 +1898,7 @@ func (s *server) checkQuota(ctx context.Context, nsr NamespacedResource) error { if len(stats) > 0 && stats[0].Count >= int64(quota.Limit) { s.log.FromContext(ctx).Info("Quota exceeded on create", "namespace", nsr.Namespace, "group", nsr.Group, "resource", nsr.Resource, "quota", quota.Limit, "count", stats[0].Count, "stats_resource", stats[0].Resource) - if s.quotasConfig.EnforceQuotas { + if s.quotasConfig.ShouldEnforce(nsr.Group, nsr.Resource) { return QuotaExceededError{ Resource: nsr.Resource, Used: stats[0].Count, diff --git a/pkg/storage/unified/resource/server_test.go b/pkg/storage/unified/resource/server_test.go index aee7fccb05c55..dc80b9efe3188 100644 --- a/pkg/storage/unified/resource/server_test.go +++ b/pkg/storage/unified/resource/server_test.go @@ -703,19 +703,34 @@ func TestGetQuotaUsage(t *testing.T) { func TestCheckQuotas(t *testing.T) { tests := []struct { - name string - limit int - expectError bool + name string + limit int + enforcedResources map[string]bool + expectError bool }{ { - name: "will return error if quota exceeded", - limit: 1, - expectError: true, + name: "enforced resource returns error if quota exceeded", + limit: 1, + enforcedResources: map[string]bool{"grafana.dashboard.app/dashboards": true}, + expectError: true, }, { - name: "will return nil if within quota", - limit: 2, - expectError: false, + name: "enforced resource returns nil if within quota", + limit: 2, + enforcedResources: map[string]bool{"grafana.dashboard.app/dashboards": true}, + expectError: false, + }, + { + name: "unlisted resource is not enforced even if over quota", + limit: 1, + enforcedResources: map[string]bool{"grafana.folder.app/folders": true}, + expectError: false, + }, + { + name: "empty enforced resources means no enforcement", + limit: 1, + enforcedResources: nil, + expectError: false, }, } @@ -752,7 +767,7 @@ func TestCheckQuotas(t *testing.T) { }}, }, OverridesService: overridesService, - QuotasConfig: QuotasConfig{EnforceQuotas: true}, + QuotasConfig: QuotasConfig{EnforcedResources: tt.enforcedResources}, }) require.NoError(t, err) t.Cleanup(func() { @@ -769,6 +784,51 @@ func TestCheckQuotas(t *testing.T) { } } +func TestShouldEnforce(t *testing.T) { + tests := []struct { + name string + config QuotasConfig + group string + resource string + expected bool + }{ + { + name: "returns true for listed resource", + config: QuotasConfig{EnforcedResources: map[string]bool{"dashboard.grafana.app/dashboards": true}}, + group: "dashboard.grafana.app", + resource: "dashboards", + expected: true, + }, + { + name: "returns false for unlisted resource", + config: QuotasConfig{EnforcedResources: map[string]bool{"dashboard.grafana.app/dashboards": true}}, + group: "folder.grafana.app", + resource: "folders", + expected: false, + }, + { + name: "returns false when map is nil", + config: QuotasConfig{}, + group: "dashboard.grafana.app", + resource: "dashboards", + expected: false, + }, + { + name: "returns false when map is empty", + config: QuotasConfig{EnforcedResources: map[string]bool{}}, + group: "dashboard.grafana.app", + resource: "dashboards", + expected: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expected, tt.config.ShouldEnforce(tt.group, tt.resource)) + }) + } +} + func Test_resourceVersionTime(t *testing.T) { // Reference time: 2026-01-15 12:00:00 UTC refTime := time.Date(2026, 1, 15, 12, 0, 0, 0, time.UTC) diff --git a/pkg/storage/unified/sql/server.go b/pkg/storage/unified/sql/server.go index e061fc2730fd5..1a6ad1bb210a9 100644 --- a/pkg/storage/unified/sql/server.go +++ b/pkg/storage/unified/sql/server.go @@ -204,9 +204,13 @@ func withOverridesService(opts *ServerOptions, resourceOpts *resource.ResourceSe } func withQuotaConfig(opts *ServerOptions, resourceOpts *resource.ResourceServerOptions) error { + enforced := make(map[string]bool, len(opts.Cfg.EnforcedQuotaResources)) + for _, r := range opts.Cfg.EnforcedQuotaResources { + enforced[r] = true + } resourceOpts.QuotasConfig = resource.QuotasConfig{ - EnforceQuotas: opts.Cfg.EnforceQuotas, - SupportMessage: opts.Cfg.QuotasErrorMessageSupportInfo, + EnforcedResources: enforced, + SupportMessage: opts.Cfg.QuotasErrorMessageSupportInfo, } return nil }