Resource exhaustion (DoS) / Memory safety

PoC: Demonstrating DoS potential via per-target max_scrape_size override Prereqs: - A VictoriaMetrics vmagent setup with a target that serves a large metrics payload. - Ability to inject per-target relabels that set __max_scrape_size__. 1) Run a metrics endpoint that emits a large number of metrics, e.g. a local Python server: python3 - << 'PY' from http.server import BaseHTTPRequestHandler, HTTPServer class MetricsHandler(BaseHTTPRequestHandler): def do_GET(self): if self.path == '/metrics': self.send_response(200) self.end_headers() # Emit a large payload to stress memory when scraped for i in range(1000000): self.wfile.write(f"metric_{i} {i}\n".encode()) httpd = HTTPServer(("0.0.0.0", 9100), MetricsHandler) httpd.serve_forever() PY 2) Configure vmagent to scrape the endpoint and override max_scrape_size per target using a relabel rule: scrape_configs: - job_name: large static_configs: - targets: ["127.0.0.1:9100"] relabel_configs: - source_labels: [__address__] regex: 127.0.0.1:9100 target_label: __max_scrape_size__ replacement: 1024MiB 3) Run vmagent with this config and observe memory usage. The per-target max_scrape_size__ is set to 1 GiB, allowing a much larger scrape payload for this target than the global default. If the endpoint can be fed with a very large response, vmagent may allocate proportional memory to hold the scraped data, potentially leading to memory exhaustion or DoS under load. 4) Verification steps: - Monitor vmagent process memory (e.g., top/htop or pmap) before, during, and after a few scrape cycles. - If memory usage spikes significantly when the large payload is scraped, this demonstrates a potential DoS risk if per-target overrides can be manipulated by an attacker. Note: This PoC is intended for controlled testing in a safe environment. Do not run against production systems.

diff --git a/lib/promscrape/config.go b/lib/promscrape/config.go index ce14a4d85bef5..a111a3500631b 100644 --- a/lib/promscrape/config.go +++ b/lib/promscrape/config.go @@ -1271,6 +1271,17 @@ func (swc *scrapeWorkConfig) getScrapeWork(target string, extraLabels, metaLabel } scrapeTimeout = d } + // Read max_scrape_size option from __max_scrape_size__ label. + targetMaxScrapeSize := swc.maxScrapeSize + if s := labels.Get("__max_scrape_size__"); len(s) > 0 { + n, err := flagutil.ParseBytes(s) + if err != nil { + return nil, fmt.Errorf("cannot parse __max_scrape_size__=%q: %w", s, err) + } + if n > 0 { + targetMaxScrapeSize = n + } + } // Read series_limit option from __series_limit__ label. // See https://docs.victoriametrics.com/victoriametrics/vmagent/#cardinality-limiter seriesLimit := swc.seriesLimit @@ -1333,7 +1344,7 @@ func (swc *scrapeWorkConfig) getScrapeWork(target string, extraLabels, metaLabel ScrapeURL: scrapeURL, ScrapeInterval: scrapeInterval, ScrapeTimeout: scrapeTimeout, - MaxScrapeSize: swc.maxScrapeSize, + MaxScrapeSize: targetMaxScrapeSize, HonorLabels: swc.honorLabels, HonorTimestamps: swc.honorTimestamps, DenyRedirects: swc.denyRedirects, diff --git a/lib/promscrape/config_test.go b/lib/promscrape/config_test.go index 0af0786a1379a..73ac545c74514 100644 --- a/lib/promscrape/config_test.go +++ b/lib/promscrape/config_test.go @@ -1153,6 +1153,57 @@ scrape_configs: }) f(` scrape_configs: +- job_name: foo + max_scrape_size: 8MiB + relabel_configs: + - source_labels: [__address__] + regex: foo1:.* + target_label: __max_scrape_size__ + replacement: 2.5MiB + - source_labels: [__address__] + regex: foo2:.* + target_label: __max_scrape_size__ + replacement: -1 + static_configs: + - targets: ["foo1:1234", "foo2:1234", "foo3:1234"] +`, []*ScrapeWork{ + { + ScrapeURL: "http://foo1:1234/metrics", + ScrapeInterval: defaultScrapeInterval, + ScrapeTimeout: defaultScrapeTimeout, + MaxScrapeSize: 2.5 * 1024 * 1024, + Labels: promutil.NewLabelsFromMap(map[string]string{ + "instance": "foo1:1234", + "job": "foo", + }), + jobNameOriginal: "foo", + }, + // invalid __max_scrape_size__ will be ignored + { + ScrapeURL: "http://foo2:1234/metrics", + ScrapeInterval: defaultScrapeInterval, + ScrapeTimeout: defaultScrapeTimeout, + MaxScrapeSize: 8 * 1024 * 1024, + Labels: promutil.NewLabelsFromMap(map[string]string{ + "instance": "foo2:1234", + "job": "foo", + }), + jobNameOriginal: "foo", + }, + { + ScrapeURL: "http://foo3:1234/metrics", + ScrapeInterval: defaultScrapeInterval, + ScrapeTimeout: defaultScrapeTimeout, + MaxScrapeSize: 8 * 1024 * 1024, + Labels: promutil.NewLabelsFromMap(map[string]string{ + "instance": "foo3:1234", + "job": "foo", + }), + jobNameOriginal: "foo", + }, + }) + f(` +scrape_configs: - job_name: foo static_configs: - targets: ["foo.bar:1234"]