Denial of Service (DoS) / Memory exhaustion

PoC (pre-fix exploit path): A malicious actor could attempt to push a provisioning file payload larger than the old hard-coded 5 MiB cap and trigger heavy memory usage or crash due to unbounded reads/parsing. The commit replaces the hard-coded cap with a configurable max_file_size and enforces the limit at the repository boundary, surfacing HTTP 413 for oversized payloads. Prerequisites: - Grafana instance with provisioning enabled (older versions prior to this fix). - Access to the provisioning API (authentication as required). - A large payload file to upload (e.g., large.bin). Attack steps (example, endpoint path may vary with Grafana version): 1) Start Grafana with provisioning enabled and default settings. 2) Create or reference a repository/file path for provisioning, e.g., README.md in a test repo. 3) Send an oversized POST/PUT to the provisioning files API: curl -X POST -H "Content-Type: application/octet-stream" --data-binary @large.bin \ https://grafana.example/api/provisioning/v1/files/README.md 4) Observe the server behavior prior to the fix: it would attempt to read the full payload, potentially causing memory exhaustion or long GC pauses, leading to degraded service or crash if limits are not properly enforced. With the fix in place (as of this commit): the server enforces a maximum file size at the repository boundary and returns HTTP 413 for payloads larger than max_file_size; a 0 or negative value disables the cap (unlimited). A read path also enforces limits, returning 413 for oversized reads. Postfix note: Endpoint paths vary by Grafana version and API configuration. Adapt the curl target to the actual provisioning API path used in your deployment.

diff --git a/conf/defaults.ini b/conf/defaults.ini index 89f3fd534f9a1..90afb1dbd71b8 100644 --- a/conf/defaults.ini +++ b/conf/defaults.ini @@ -2433,6 +2433,12 @@ folders_api_version = v1 # Default is 100. If set to 0, the size check is disabled. max_incremental_changes = 100 +# Maximum file size in bytes for files read from or written to a provisioning repository +# through the files API. Applies symmetrically to GET (reads) and POST/PUT writes. +# Default is 5242880 bytes (5 MiB, i.e. 5 * 1024 * 1024). Set to 0 (or any +# non-positive value) for unlimited. +max_file_size = 5242880 + # Public-facing root URL of this Grafana instance, used by provisioning to construct URLs # that must be reachable from external systems (e.g. GitHub PR-comment image fetchers and # GitHub webhook deliveries). When empty, falls back to [server] root_url. Set this when diff --git a/pkg/registry/apis/provisioning/files.go b/pkg/registry/apis/provisioning/files.go index fc4fb68d3c41a..cfe8b79332351 100644 --- a/pkg/registry/apis/provisioning/files.go +++ b/pkg/registry/apis/provisioning/files.go @@ -23,11 +23,6 @@ import ( "github.com/grafana/grafana/pkg/registry/apis/provisioning/resources" ) -const ( - // Files endpoint max size for dashboards etc (5MB) - filesMaxBodySize = 5 * 1024 * 1024 -) - type filesConnector struct { getter RepoGetter access auth.AccessChecker @@ -35,9 +30,12 @@ type filesConnector struct { clients resources.ClientFactory folderMetadataEnabled bool folderAPIVersion string + // maxFileSize caps the size in bytes of files read from or written to the + // repository through this connector. <=0 disables the check. + maxFileSize int64 } -func NewFilesConnector(getter RepoGetter, parsers resources.ParserFactory, clients resources.ClientFactory, access auth.AccessChecker, folderMetadataEnabled bool, folderAPIVersion string) *filesConnector { +func NewFilesConnector(getter RepoGetter, parsers resources.ParserFactory, clients resources.ClientFactory, access auth.AccessChecker, folderMetadataEnabled bool, folderAPIVersion string, maxFileSize int64) *filesConnector { return &filesConnector{ getter: getter, parsers: parsers, @@ -45,7 +43,39 @@ func NewFilesConnector(getter RepoGetter, parsers resources.ParserFactory, clien access: access, folderMetadataEnabled: folderMetadataEnabled, folderAPIVersion: folderAPIVersion, + maxFileSize: maxFileSize, + } +} + +// sizeLimitedReaderWriter wraps a repository.ReaderWriter and rejects reads +// that exceed maxBytes. The check fires immediately after the underlying +// repository returns the file bytes, so callers (including DualReadWriter +// which parses the result) never see oversized payloads. +type sizeLimitedReaderWriter struct { + repository.ReaderWriter + maxBytes int64 +} + +func (s *sizeLimitedReaderWriter) Read(ctx context.Context, path, ref string) (*repository.FileInfo, error) { + info, err := s.ReaderWriter.Read(ctx, path, ref) + if err != nil { + return info, err + } + if s.maxBytes > 0 && info != nil && int64(len(info.Data)) > s.maxBytes { + return nil, apierrors.NewRequestEntityTooLargeError( + fmt.Sprintf("file %q is %d bytes; max allowed is %d bytes", info.Path, len(info.Data), s.maxBytes), + ) + } + return info, nil +} + +// withSizeLimit returns rw wrapped so its Read method enforces the connector's +// configured max file size. Returns rw unchanged when the cap is disabled. +func (c *filesConnector) withSizeLimit(rw repository.ReaderWriter) repository.ReaderWriter { + if c.maxFileSize <= 0 { + return rw } + return &sizeLimitedReaderWriter{ReaderWriter: rw, maxBytes: c.maxFileSize} } func (*filesConnector) New() runtime.Object { @@ -105,6 +135,10 @@ func (c *filesConnector) handleRequest(ctx context.Context, name string, r *http responder.Error(apierrors.NewBadRequest("repository does not support read-writing")) return } + // Enforce max_file_size right at the repo boundary so oversized payloads + // are rejected before parsing/DryRun runs in DualReadWriter.Read or before + // the bytes are streamed back from handleGetRawFile. + readWriter = c.withSizeLimit(readWriter) dualReadWriter, authorizer, err := c.createDualReadWriter(ctx, repo, readWriter) if err != nil { @@ -310,7 +344,7 @@ func (c *filesConnector) handlePost(ctx context.Context, r *http.Request, opts r return dualReadWriter.CreateFolder(ctx, opts) } - data, err := readBody(r, filesMaxBodySize) + data, err := readBody(r, c.maxFileSize) if err != nil { return nil, err } @@ -326,7 +360,7 @@ func (c *filesConnector) handlePost(ctx context.Context, r *http.Request, opts r func (c *filesConnector) handleMove(ctx context.Context, r *http.Request, opts resources.DualWriteOptions, isDir bool, dualReadWriter *resources.DualReadWriter) (*provisioning.ResourceWrapper, error) { // For move operations, only read body for file moves (not directory moves) if !isDir { - data, err := readBody(r, filesMaxBodySize) + data, err := readBody(r, c.maxFileSize) if err != nil { return nil, err } @@ -348,7 +382,7 @@ func (c *filesConnector) handlePut(ctx context.Context, r *http.Request, opts re return nil, apierrors.NewMethodNotSupported(provisioning.RepositoryResourceInfo.GroupResource(), r.Method) } - data, err := readBody(r, filesMaxBodySize) + data, err := readBody(r, c.maxFileSize) if err != nil { return nil, err } @@ -362,7 +396,7 @@ func (c *filesConnector) handlePut(ctx context.Context, r *http.Request, opts re } func (c *filesConnector) handleFolderMetadataUpdate(ctx context.Context, r *http.Request, opts resources.DualWriteOptions, dualReadWriter *resources.DualReadWriter) (*provisioning.ResourceWrapper, error) { - data, err := readBody(r, filesMaxBodySize) + data, err := readBody(r, c.maxFileSize) if err != nil { return nil, err } diff --git a/pkg/registry/apis/provisioning/files_test.go b/pkg/registry/apis/provisioning/files_test.go index b02b0ede902f4..6ebc5c4366400 100644 --- a/pkg/registry/apis/provisioning/files_test.go +++ b/pkg/registry/apis/provisioning/files_test.go @@ -536,6 +536,66 @@ func TestHandleGetRawFile_FolderScopedAuth(t *testing.T) { }) } +func TestHandleGetRawFile_MaxFileSize(t *testing.T) { + const path = "README.md" + repo := &provisioningapi.Repository{ + ObjectMeta: metav1.ObjectMeta{Name: "test-repo"}, + Spec: provisioningapi.RepositorySpec{ + Sync: provisioningapi.SyncOptions{Target: provisioningapi.SyncTargetTypeFolder}, + }, + } + + tests := []struct { + name string + maxFileSize int64 + dataSize int + wantTooBig bool + }{ + {name: "under limit", maxFileSize: 1024, dataSize: 512}, + {name: "exactly at limit", maxFileSize: 1024, dataSize: 1024}, + {name: "over limit", maxFileSize: 1024, dataSize: 2048, wantTooBig: true}, + {name: "zero disables limit", maxFileSize: 0, dataSize: 10 * 1024 * 1024}, + {name: "negative disables limit", maxFileSize: -1, dataSize: 10 * 1024 * 1024}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + mockReadWriter := repository.NewMockReaderWriter(t) + mockAccess := auth.NewMockAccessChecker(t) + mockAccess.EXPECT().Check(mock.Anything, mock.Anything, mock.Anything).Return(nil).Maybe() + + mockReadWriter.EXPECT().Config().Return(repo).Maybe() + authorizer := resources.NewAuthorizer(repo, mockReadWriter, mockAccess, false) + + mockReadWriter.EXPECT().Read(mock.Anything, path, "").Return(&repository.FileInfo{ + Path: path, + Data: make([]byte, tc.dataSize), + Ref: "main", + }, nil) + + connector := &filesConnector{access: mockAccess, maxFileSize: tc.maxFileSize} + // handleRequest wraps readWriter with the size limiter at the + // connector boundary. Mirror that here so the unit test exercises + // the same enforcement point as production. + limited := connector.withSizeLimit(mockReadWriter) + + _, err := connector.handleGetRawFile( + context.Background(), + resources.DualWriteOptions{Path: path}, + limited, + authorizer, + ) + + if tc.wantTooBig { + require.Error(t, err) + assert.True(t, apierrors.IsRequestEntityTooLargeError(err), "expected 413 RequestEntityTooLarge, got %v", err) + } else { + require.NoError(t, err) + } + }) + } +} + func TestIsRawFileIntegration(t *testing.T) { tests := []struct { name string diff --git a/pkg/registry/apis/provisioning/register.go b/pkg/registry/apis/provisioning/register.go index 6114aa7c1f775..065ea447d358f 100644 --- a/pkg/registry/apis/provisioning/register.go +++ b/pkg/registry/apis/provisioning/register.go @@ -139,6 +139,7 @@ type APIBuilder struct { registry prometheus.Registerer quotaGetter quotas.QuotaGetter folderMetadataEnabled bool + maxFileSize int64 incrementalPolicy repository.IncrementalSyncPolicy folderAPIVersion string webhookSecretRotationInterval time.Duration @@ -241,6 +242,10 @@ func NewAPIBuilder( folderMetadataEnabled: folderMetadataEnabled, folderAPIVersion: folderAPIVersion, incrementalPolicy: incrementalPolicy, + // Default cap for the files API. Callers (e.g. RegisterAPIService) + // may overwrite b.maxFileSize after construction; any non-positive + // value (<=0) disables the cap. + maxFileSize: setting.ProvisioningMaxFileSizeDefault, } for _, builder := range extraBuilders { @@ -315,6 +320,7 @@ func RegisterAPIService( jobHistoryConfig := createJobHistoryConfigFromSettings(cfg) folderMetadataEnabled := features.IsEnabledGlobally(featuremgmt.FlagProvisioningFolderMetadata) //nolint:staticcheck folderAPIVersion := cfg.ProvisioningFolderAPIVersion + maxFileSize := cfg.ProvisioningMaxFileSize incrementalPolicy := repository.NewIncrementalSyncPolicy(folderMetadataEnabled, cfg.ProvisioningMaxIncrementalChanges) // Register v0alpha1 (preferred version) @@ -353,6 +359,7 @@ func RegisterAPIService( return nil, err } builder.webhookSecretRotationInterval = cfg.ProvisioningWebhookSecretRotationInterval + builder.maxFileSize = maxFileSize apiregistration.RegisterAPI(builder) // Register v1beta1 @@ -391,6 +398,7 @@ func RegisterAPIService( return nil, err } v1beta1Builder.webhookSecretRotationInterval = cfg.ProvisioningWebhookSecretRotationInterval + v1beta1Builder.maxFileSize = maxFileSize apiregistration.RegisterAPI(v1beta1Builder) // Return the preferred (v0alpha1) builder since it runs controllers/workers @@ -802,7 +810,7 @@ func (b *APIBuilder) UpdateAPIGroupInfo(apiGroupInfo *genericapiserver.APIGroupI // connector via ProvisioningAuthorizer.AuthorizeResource, and repository- // level operations remain Admin-gated by authorizeRepositorySubresource. filesAccess := auth.NewVerbAwareAccessChecker(b.accessWithViewer, b.accessWithEditor) - storage[provisioning.RepositoryResourceInfo.StoragePath("files")] = NewFilesConnector(b, b.parsers, b.clients, filesAccess, b.folderMetadataEnabled, b.folderAPIVersion) + storage[provisioning.RepositoryResourceInfo.StoragePath("files")] = NewFilesConnector(b, b.parsers, b.clients, filesAccess, b.folderMetadataEnabled, b.folderAPIVersion, b.maxFileSize) storage[provisioning.RepositoryResourceInfo.StoragePath("refs")] = NewRefsConnector(b) storage[provisioning.RepositoryResourceInfo.StoragePath("resources")] = &listConnector{ getter: b, diff --git a/pkg/registry/apis/provisioning/request.go b/pkg/registry/apis/provisioning/request.go index 7eb8def59bb4b..60be986cd7057 100644 --- a/pkg/registry/apis/provisioning/request.go +++ b/pkg/registry/apis/provisioning/request.go @@ -7,6 +7,8 @@ import ( "io" "net/http" "strings" + + apierrors "k8s.io/apimachinery/pkg/api/errors" ) const ( @@ -18,14 +20,24 @@ const ( errMsgRequestTooLarge = "request body too large" ) -// readBody reads the request body and limits the size +// readBody reads the request body and limits the size. A non-positive +// maxSize disables the limit. func readBody(r *http.Request, maxSize int64) ([]byte, error) { + if maxSize <= 0 { + body, err := io.ReadAll(r.Body) + if err != nil { + return nil, fmt.Errorf("error reading request body: %w", err) + } + return body, nil + } limitedBody := http.MaxBytesReader(nil, r.Body, maxSize) body, err := io.ReadAll(limitedBody) if err != nil { var maxBytesError *http.MaxBytesError if errors.As(err, &maxBytesError) { - return nil, fmt.Errorf("%s: max size %d bytes", errMsgRequestTooLarge, maxSize) + return nil, apierrors.NewRequestEntityTooLargeError( + fmt.Sprintf("%s: max size %d bytes", errMsgRequestTooLarge, maxSize), + ) } return nil, fmt.Errorf("error reading request body: %w", err) } diff --git a/pkg/registry/apis/provisioning/request_test.go b/pkg/registry/apis/provisioning/request_test.go index 3fedd8d550b4f..aa4b727741443 100644 --- a/pkg/registry/apis/provisioning/request_test.go +++ b/pkg/registry/apis/provisioning/request_test.go @@ -31,6 +31,16 @@ func TestReadBody(t *testing.T) { body: "1234567890", maxSize: 10, }, + { + name: "zero maxSize disables the cap", + body: "anything goes when the cap is unlimited", + maxSize: 0, + }, + { + name: "negative maxSize disables the cap", + body: "anything goes when the cap is unlimited", + maxSize: -1, + }, } for _, tt := range tests { diff --git a/pkg/setting/setting.go b/pkg/setting/setting.go index c7dafa1afd69a..c2d00a27fec92 100644 --- a/pkg/setting/setting.go +++ b/pkg/setting/setting.go @@ -65,6 +65,11 @@ const zoneInfo = "ZONEINFO" // Default renderer auth token from [rendering]renderer_token. const DefaultRendererAuthToken = "-" +// ProvisioningMaxFileSizeDefault is the default value for the +// [provisioning] max_file_size key (5 MiB). It bounds files read from or +// written to a provisioning repository through the files API. +const ProvisioningMaxFileSizeDefault int64 = 5 * 1024 * 1024 + var ( customInitPath = "conf/custom.ini" @@ -163,6 +168,7 @@ type Cfg struct { ProvisioningMaxRepositories int64 // default 10, 0 in config = unlimited (converted to -1 internally) ProvisioningFolderAPIVersion string // "v1" (default for on-prem) or "v1beta1" ProvisioningMaxIncrementalChanges int // default 100, 0 in config = unlimited + ProvisioningMaxFileSize int64 // bytes; default 5 MiB (5242880); <=0 = unlimited ProvisioningWebhookSecretRotationInterval time.Duration // default 30 days ProvisioningPublicRootURL string // public-facing root URL of this Grafana instance for provisioning consumers (webhooks, screenshots); falls back to AppURL when empty DataPath string @@ -2469,6 +24 ... [truncated]