scrape: provide a fallback format (prometheus#15136)

scrape: Remove implicit fallback to the Prometheus text format

Remove implicit fallback to the Prometheus text format in case of invalid/missing Content-Type and fail the scrape instead. Add ability to specify a `fallback_scrape_protocol` in the scrape config.

---------

Signed-off-by: alexgreenbank <[email protected]>
Signed-off-by: Alex Greenbank <[email protected]>
Co-authored-by: Björn Rabenstein <[email protected]>

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## unreleased
 
+* [CHANGE] Scraping: Remove implicit fallback to the Prometheus text format in case of invalid/missing Content-Type and fail the scrape instead. Add ability to specify a `fallback_scrape_protocol` in the scrape config. #15136
 * [BUGFIX] PromQL: Fix stddev+stdvar aggregations to always ignore native histograms. #14941
 * [BUGFIX] PromQL: Fix stddev+stdvar aggregations to treat Infinity consistently. #14941
 

config/config.go

@@ -17,6 +17,7 @@ import (
 	"errors"
 	"fmt"
 	"log/slog"
+	"mime"
 	"net/url"
 	"os"
 	"path/filepath"
@@ -473,16 +474,30 @@ func (s ScrapeProtocol) Validate() error {
 	return nil
 }
 
+// HeaderMediaType returns the MIME mediaType for a particular ScrapeProtocol.
+func (s ScrapeProtocol) HeaderMediaType() string {
+	if _, ok := ScrapeProtocolsHeaders[s]; !ok {
+		return ""
+	}
+	mediaType, _, err := mime.ParseMediaType(ScrapeProtocolsHeaders[s])
+	if err != nil {
+		return ""
+	}
+	return mediaType
+}
+
 var (
 	PrometheusProto      ScrapeProtocol = "PrometheusProto"
 	PrometheusText0_0_4  ScrapeProtocol = "PrometheusText0.0.4"
+	PrometheusText1_0_0  ScrapeProtocol = "PrometheusText1.0.0"
 	OpenMetricsText0_0_1 ScrapeProtocol = "OpenMetricsText0.0.1"
 	OpenMetricsText1_0_0 ScrapeProtocol = "OpenMetricsText1.0.0"
 	UTF8NamesHeader      string         = model.EscapingKey + "=" + model.AllowUTF8
 
 	ScrapeProtocolsHeaders = map[ScrapeProtocol]string{
 		PrometheusProto:      "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited",
 		PrometheusText0_0_4:  "text/plain;version=0.0.4",
+		PrometheusText1_0_0:  "text/plain;version=1.0.0;escaping=allow-utf-8",
 		OpenMetricsText0_0_1: "application/openmetrics-text;version=0.0.1",
 		OpenMetricsText1_0_0: "application/openmetrics-text;version=1.0.0",
 	}
@@ -492,6 +507,7 @@ var (
 	DefaultScrapeProtocols = []ScrapeProtocol{
 		OpenMetricsText1_0_0,
 		OpenMetricsText0_0_1,
+		PrometheusText1_0_0,
 		PrometheusText0_0_4,
 	}
 
@@ -503,6 +519,7 @@ var (
 		PrometheusProto,
 		OpenMetricsText1_0_0,
 		OpenMetricsText0_0_1,
+		PrometheusText1_0_0,
 		PrometheusText0_0_4,
 	}
 )
@@ -629,8 +646,15 @@ type ScrapeConfig struct {
 	// The protocols to negotiate during a scrape. It tells clients what
 	// protocol are accepted by Prometheus and with what preference (most wanted is first).
 	// Supported values (case sensitive): PrometheusProto, OpenMetricsText0.0.1,
-	// OpenMetricsText1.0.0, PrometheusText0.0.4.
+	// OpenMetricsText1.0.0, PrometheusText1.0.0, PrometheusText0.0.4.
 	ScrapeProtocols []ScrapeProtocol `yaml:"scrape_protocols,omitempty"`
+	// The fallback protocol to use if the Content-Type provided by the target
+	// is not provided, blank, or not one of the expected values.
+	// Supported values (case sensitive): PrometheusProto, OpenMetricsText0.0.1,
+	// OpenMetricsText1.0.0, PrometheusText1.0.0, PrometheusText0.0.4.
+	ScrapeFallbackProtocol ScrapeProtocol `yaml:"fallback_scrape_protocol,omitempty"`
+	// Whether to scrape a classic histogram that is also exposed as a native histogram.
+	ScrapeClassicHistograms bool `yaml:"scrape_classic_histograms,omitempty"`
 	// Whether to scrape a classic histogram, even if it is also exposed as a native histogram.
 	AlwaysScrapeClassicHistograms bool `yaml:"always_scrape_classic_histograms,omitempty"`
 	// File to which scrape failures are logged.
@@ -780,6 +804,12 @@ func (c *ScrapeConfig) Validate(globalConfig GlobalConfig) error {
 		return fmt.Errorf("%w for scrape config with job name %q", err, c.JobName)
 	}
 
+	if c.ScrapeFallbackProtocol != "" {
+		if err := c.ScrapeFallbackProtocol.Validate(); err != nil {
+			return fmt.Errorf("invalid fallback_scrape_protocol for scrape config with job name %q: %w", c.JobName, err)
+		}
+	}
+
 	switch globalConfig.MetricNameValidationScheme {
 	case LegacyValidationConfig:
 	case "", UTF8ValidationConfig:

config/config_test.go

@@ -206,19 +206,20 @@ var expectedConf = &Config{
 		{
 			JobName: "prometheus",
 
-			HonorLabels:           true,
-			HonorTimestamps:       true,
-			ScrapeInterval:        model.Duration(15 * time.Second),
-			ScrapeTimeout:         DefaultGlobalConfig.ScrapeTimeout,
-			EnableCompression:     true,
-			BodySizeLimit:         globBodySizeLimit,
-			SampleLimit:           globSampleLimit,
-			TargetLimit:           globTargetLimit,
-			LabelLimit:            globLabelLimit,
-			LabelNameLengthLimit:  globLabelNameLengthLimit,
-			LabelValueLengthLimit: globLabelValueLengthLimit,
-			ScrapeProtocols:       DefaultGlobalConfig.ScrapeProtocols,
-			ScrapeFailureLogFile:  "testdata/fail_prom.log",
+			HonorLabels:            true,
+			HonorTimestamps:        true,
+			ScrapeInterval:         model.Duration(15 * time.Second),
+			ScrapeTimeout:          DefaultGlobalConfig.ScrapeTimeout,
+			EnableCompression:      true,
+			BodySizeLimit:          globBodySizeLimit,
+			SampleLimit:            globSampleLimit,
+			TargetLimit:            globTargetLimit,
+			LabelLimit:             globLabelLimit,
+			LabelNameLengthLimit:   globLabelNameLengthLimit,
+			LabelValueLengthLimit:  globLabelValueLengthLimit,
+			ScrapeProtocols:        DefaultGlobalConfig.ScrapeProtocols,
+			ScrapeFallbackProtocol: PrometheusText0_0_4,
+			ScrapeFailureLogFile:   "testdata/fail_prom.log",
 
 			MetricsPath: DefaultScrapeConfig.MetricsPath,
 			Scheme:      DefaultScrapeConfig.Scheme,
@@ -2086,12 +2087,20 @@ var expectedErrors = []struct {
 	},
 	{
 		filename: "scrape_config_files_scrape_protocols.bad.yml",
-		errMsg:   `parsing YAML file testdata/scrape_config_files_scrape_protocols.bad.yml: scrape_protocols: unknown scrape protocol prometheusproto, supported: [OpenMetricsText0.0.1 OpenMetricsText1.0.0 PrometheusProto PrometheusText0.0.4] for scrape config with job name "node"`,
+		errMsg:   `parsing YAML file testdata/scrape_config_files_scrape_protocols.bad.yml: scrape_protocols: unknown scrape protocol prometheusproto, supported: [OpenMetricsText0.0.1 OpenMetricsText1.0.0 PrometheusProto PrometheusText0.0.4 PrometheusText1.0.0] for scrape config with job name "node"`,
 	},
 	{
 		filename: "scrape_config_files_scrape_protocols2.bad.yml",
 		errMsg:   `parsing YAML file testdata/scrape_config_files_scrape_protocols2.bad.yml: duplicated protocol in scrape_protocols, got [OpenMetricsText1.0.0 PrometheusProto OpenMetricsText1.0.0] for scrape config with job name "node"`,
 	},
+	{
+		filename: "scrape_config_files_fallback_scrape_protocol1.bad.yml",
+		errMsg:   `parsing YAML file testdata/scrape_config_files_fallback_scrape_protocol1.bad.yml: invalid fallback_scrape_protocol for scrape config with job name "node": unknown scrape protocol prometheusproto, supported: [OpenMetricsText0.0.1 OpenMetricsText1.0.0 PrometheusProto PrometheusText0.0.4 PrometheusText1.0.0]`,
+	},
+	{
+		filename: "scrape_config_files_fallback_scrape_protocol2.bad.yml",
+		errMsg:   `unmarshal errors`,
+	},
 }
 
 func TestBadConfigs(t *testing.T) {
@@ -2412,3 +2421,54 @@ func TestScrapeConfigNameValidationSettings(t *testing.T) {
 		})
 	}
 }
+
+func TestScrapeProtocolHeader(t *testing.T) {
+	tests := []struct {
+		name          string
+		proto         ScrapeProtocol
+		expectedValue string
+	}{
+		{
+			name:          "blank",
+			proto:         ScrapeProtocol(""),
+			expectedValue: "",
+		},
+		{
+			name:          "invalid",
+			proto:         ScrapeProtocol("invalid"),
+			expectedValue: "",
+		},
+		{
+			name:          "prometheus protobuf",
+			proto:         PrometheusProto,
+			expectedValue: "application/vnd.google.protobuf",
+		},
+		{
+			name:          "prometheus text 0.0.4",
+			proto:         PrometheusText0_0_4,
+			expectedValue: "text/plain",
+		},
+		{
+			name:          "prometheus text 1.0.0",
+			proto:         PrometheusText1_0_0,
+			expectedValue: "text/plain",
+		},
+		{
+			name:          "openmetrics 0.0.1",
+			proto:         OpenMetricsText0_0_1,
+			expectedValue: "application/openmetrics-text",
+		},
+		{
+			name:          "openmetrics 1.0.0",
+			proto:         OpenMetricsText1_0_0,
+			expectedValue: "application/openmetrics-text",
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			mediaType := tc.proto.HeaderMediaType()
+
+			require.Equal(t, tc.expectedValue, mediaType)
+		})
+	}
+}

config/testdata/conf.good.yml

@@ -74,6 +74,8 @@ scrape_configs:
     # metrics_path defaults to '/metrics'
     # scheme defaults to 'http'.
 
+    fallback_scrape_protocol: PrometheusText0.0.4
+
     scrape_failure_log_file: fail_prom.log
     file_sd_configs:
       - files:

config/testdata/scrape_config_files_fallback_scrape_protocol1.bad.yml

@@ -0,0 +1,5 @@
+scrape_configs:
+  - job_name: node
+    fallback_scrape_protocol: "prometheusproto"
+    static_configs:
+      - targets: ['localhost:8080']

config/testdata/scrape_config_files_fallback_scrape_protocol2.bad.yml

@@ -0,0 +1,5 @@
+scrape_configs:
+  - job_name: node
+    fallback_scrape_protocol: ["OpenMetricsText1.0.0", "PrometheusText0.0.4"]
+    static_configs:
+      - targets: ['localhost:8080']

docs/configuration/configuration.md

@@ -212,9 +212,15 @@ job_name: <job_name>
 
 # The protocols to negotiate during a scrape with the client.
 # Supported values (case sensitive): PrometheusProto, OpenMetricsText0.0.1,
-# OpenMetricsText1.0.0, PrometheusText0.0.4.
+# OpenMetricsText1.0.0, PrometheusText0.0.4, PrometheusText1.0.0.
 [ scrape_protocols: [<string>, ...] | default = <global_config.scrape_protocols> ]
 
+# Fallback protocol to use if a scrape returns blank, unparseable, or otherwise
+# invalid Content-Type.
+# Supported values (case sensitive): PrometheusProto, OpenMetricsText0.0.1,
+# OpenMetricsText1.0.0, PrometheusText0.0.4, PrometheusText1.0.0.
+[ fallback_scrape_protocol: <string> ]
+
 # Whether to scrape a classic histogram, even if it is also exposed as a native
 # histogram (has no effect without --enable-feature=native-histograms).
 [ always_scrape_classic_histograms: <boolean> | default = false ]

model/textparse/interface.go

@@ -14,6 +14,8 @@
 package textparse
 
 import (
+	"errors"
+	"fmt"
 	"mime"
 
 	"github.com/prometheus/common/model"
@@ -78,28 +80,65 @@ type Parser interface {
 	Next() (Entry, error)
 }
 
-// New returns a new parser of the byte slice.
-//
-// This function always returns a valid parser, but might additionally
-// return an error if the content type cannot be parsed.
-func New(b []byte, contentType string, parseClassicHistograms, skipOMCTSeries bool, st *labels.SymbolTable) (Parser, error) {
+// extractMediaType returns the mediaType of a required parser. It tries first to
+// extract a valid and supported mediaType from contentType. If that fails,
+// the provided fallbackType (possibly an empty string) is returned, together with
+// an error. fallbackType is used as-is without further validation.
+func extractMediaType(contentType, fallbackType string) (string, error) {
 	if contentType == "" {
-		return NewPromParser(b, st), nil
+		if fallbackType == "" {
+			return "", errors.New("non-compliant scrape target sending blank Content-Type and no fallback_scrape_protocol specified for target")
+		}
+		return fallbackType, fmt.Errorf("non-compliant scrape target sending blank Content-Type, using fallback_scrape_protocol %q", fallbackType)
 	}
 
+	// We have a contentType, parse it.
 	mediaType, _, err := mime.ParseMediaType(contentType)
 	if err != nil {
-		return NewPromParser(b, st), err
+		if fallbackType == "" {
+			retErr := fmt.Errorf("cannot parse Content-Type %q and no fallback_scrape_protocol for target", contentType)
+			return "", errors.Join(retErr, err)
+		}
+		retErr := fmt.Errorf("could not parse received Content-Type %q, using fallback_scrape_protocol %q", contentType, fallbackType)
+		return fallbackType, errors.Join(retErr, err)
+	}
+
+	// We have a valid media type, either we recognise it and can use it
+	// or we have to error.
+	switch mediaType {
+	case "application/openmetrics-text", "application/vnd.google.protobuf", "text/plain":
+		return mediaType, nil
+	}
+	// We're here because we have no recognised mediaType.
+	if fallbackType == "" {
+		return "", fmt.Errorf("received unsupported Content-Type %q and no fallback_scrape_protocol specified for target", contentType)
 	}
+	return fallbackType, fmt.Errorf("received unsupported Content-Type %q, using fallback_scrape_protocol %q", contentType, fallbackType)
+}
+
+// New returns a new parser of the byte slice.
+//
+// This function no longer guarantees to return a valid parser.
+//
+// It only returns a valid parser if the supplied contentType and fallbackType allow.
+// An error may also be returned if fallbackType had to be used or there was some
+// other error parsing the supplied Content-Type.
+// If the returned parser is nil then the scrape must fail.
+func New(b []byte, contentType, fallbackType string, parseClassicHistograms, skipOMCTSeries bool, st *labels.SymbolTable) (Parser, error) {
+	mediaType, err := extractMediaType(contentType, fallbackType)
+	// err may be nil or something we want to warn about.
+
 	switch mediaType {
 	case "application/openmetrics-text":
 		return NewOpenMetricsParser(b, st, func(o *openMetricsParserOptions) {
 			o.SkipCTSeries = skipOMCTSeries
-		}), nil
+		}), err
 	case "application/vnd.google.protobuf":
-		return NewProtobufParser(b, parseClassicHistograms, st), nil
+		return NewProtobufParser(b, parseClassicHistograms, st), err
+	case "text/plain":
+		return NewPromParser(b, st), err
 	default:
-		return NewPromParser(b, st), nil
+		return nil, err
 	}
 }