From a264ec0d85600decfb0681d00ed1566186bebfb3 Mon Sep 17 00:00:00 2001 From: Colin Watson Date: Wed, 26 Nov 2025 19:39:29 +0000 Subject: [PATCH 01/11] Don't interleave histogram metrics in multi-process collector (#1148) The OpenMetrics exposition format requires that samples for a given Metric (i.e. metric name and label set) are not interleaved, but the way that the multi-process collector handled accumulating histogram metrics could end up interleaving them. Restructure it slightly to guarantee that all the samples for a given Metric are kept together. Fixes: #1147 Signed-off-by: Colin Watson --- prometheus_client/multiprocess.py | 51 +++++++++++++--------- tests/test_multiprocess.py | 70 +++++++++++++++++++++++++++---- 2 files changed, 94 insertions(+), 27 deletions(-) diff --git a/prometheus_client/multiprocess.py b/prometheus_client/multiprocess.py index 2682190a..db55874e 100644 --- a/prometheus_client/multiprocess.py +++ b/prometheus_client/multiprocess.py @@ -88,32 +88,42 @@ def _parse_key(key): @staticmethod def _accumulate_metrics(metrics, accumulate): for metric in metrics.values(): - samples = defaultdict(float) - sample_timestamps = defaultdict(float) + samples = defaultdict(lambda: defaultdict(float)) + sample_timestamps = defaultdict(lambda: defaultdict(float)) buckets = defaultdict(lambda: defaultdict(float)) - samples_setdefault = samples.setdefault for s in metric.samples: name, labels, value, timestamp, exemplar, native_histogram_value = s + + if ( + metric.type == 'gauge' + and metric._multiprocess_mode in ( + 'min', 'livemin', + 'max', 'livemax', + 'sum', 'livesum', + 'mostrecent', 'livemostrecent', + ) + ): + labels = tuple(l for l in labels if l[0] != 'pid') + if metric.type == 'gauge': - without_pid_key = (name, tuple(l for l in labels if l[0] != 'pid')) if metric._multiprocess_mode in ('min', 'livemin'): - current = samples_setdefault(without_pid_key, value) + current = samples[labels].setdefault((name, labels), value) if value < current: - samples[without_pid_key] = value + samples[labels][(name, labels)] = value elif metric._multiprocess_mode in ('max', 'livemax'): - current = samples_setdefault(without_pid_key, value) + current = samples[labels].setdefault((name, labels), value) if value > current: - samples[without_pid_key] = value + samples[labels][(name, labels)] = value elif metric._multiprocess_mode in ('sum', 'livesum'): - samples[without_pid_key] += value + samples[labels][(name, labels)] += value elif metric._multiprocess_mode in ('mostrecent', 'livemostrecent'): - current_timestamp = sample_timestamps[without_pid_key] + current_timestamp = sample_timestamps[labels][name] timestamp = float(timestamp or 0) if current_timestamp < timestamp: - samples[without_pid_key] = value - sample_timestamps[without_pid_key] = timestamp + samples[labels][(name, labels)] = value + sample_timestamps[labels][name] = timestamp else: # all/liveall - samples[(name, labels)] = value + samples[labels][(name, labels)] = value elif metric.type == 'histogram': # A for loop with early exit is faster than a genexpr @@ -127,10 +137,10 @@ def _accumulate_metrics(metrics, accumulate): break else: # did not find the `le` key # _sum/_count - samples[(name, labels)] += value + samples[labels][(name, labels)] += value else: # Counter and Summary. - samples[(name, labels)] += value + samples[labels][(name, labels)] += value # Accumulate bucket values. if metric.type == 'histogram': @@ -143,14 +153,17 @@ def _accumulate_metrics(metrics, accumulate): ) if accumulate: acc += value - samples[sample_key] = acc + samples[labels][sample_key] = acc else: - samples[sample_key] = value + samples[labels][sample_key] = value if accumulate: - samples[(metric.name + '_count', labels)] = acc + samples[labels][(metric.name + '_count', labels)] = acc # Convert to correct sample format. - metric.samples = [Sample(name_, dict(labels), value) for (name_, labels), value in samples.items()] + metric.samples = [] + for _, samples_by_labels in samples.items(): + for (name_, labels), value in samples_by_labels.items(): + metric.samples.append(Sample(name_, dict(labels), value)) return metrics.values() def collect(self): diff --git a/tests/test_multiprocess.py b/tests/test_multiprocess.py index e7ca154e..c2f71d26 100644 --- a/tests/test_multiprocess.py +++ b/tests/test_multiprocess.py @@ -276,10 +276,8 @@ def add_label(key, value): Sample('g', add_label('pid', '1'), 1.0), ]) - metrics['h'].samples.sort( - key=lambda x: (x[0], float(x[1].get('le', 0))) - ) expected_histogram = [ + Sample('h_sum', labels, 6.0), Sample('h_bucket', add_label('le', '0.005'), 0.0), Sample('h_bucket', add_label('le', '0.01'), 0.0), Sample('h_bucket', add_label('le', '0.025'), 0.0), @@ -296,7 +294,66 @@ def add_label(key, value): Sample('h_bucket', add_label('le', '10.0'), 2.0), Sample('h_bucket', add_label('le', '+Inf'), 2.0), Sample('h_count', labels, 2.0), - Sample('h_sum', labels, 6.0), + ] + + self.assertEqual(metrics['h'].samples, expected_histogram) + + def test_collect_histogram_ordering(self): + pid = 0 + values.ValueClass = MultiProcessValue(lambda: pid) + labels = {i: i for i in 'abcd'} + + def add_label(key, value): + l = labels.copy() + l[key] = value + return l + + h = Histogram('h', 'help', labelnames=['view'], registry=None) + + h.labels(view='view1').observe(1) + + pid = 1 + + h.labels(view='view1').observe(5) + h.labels(view='view2').observe(1) + + metrics = {m.name: m for m in self.collector.collect()} + + expected_histogram = [ + Sample('h_sum', {'view': 'view1'}, 6.0), + Sample('h_bucket', {'view': 'view1', 'le': '0.005'}, 0.0), + Sample('h_bucket', {'view': 'view1', 'le': '0.01'}, 0.0), + Sample('h_bucket', {'view': 'view1', 'le': '0.025'}, 0.0), + Sample('h_bucket', {'view': 'view1', 'le': '0.05'}, 0.0), + Sample('h_bucket', {'view': 'view1', 'le': '0.075'}, 0.0), + Sample('h_bucket', {'view': 'view1', 'le': '0.1'}, 0.0), + Sample('h_bucket', {'view': 'view1', 'le': '0.25'}, 0.0), + Sample('h_bucket', {'view': 'view1', 'le': '0.5'}, 0.0), + Sample('h_bucket', {'view': 'view1', 'le': '0.75'}, 0.0), + Sample('h_bucket', {'view': 'view1', 'le': '1.0'}, 1.0), + Sample('h_bucket', {'view': 'view1', 'le': '2.5'}, 1.0), + Sample('h_bucket', {'view': 'view1', 'le': '5.0'}, 2.0), + Sample('h_bucket', {'view': 'view1', 'le': '7.5'}, 2.0), + Sample('h_bucket', {'view': 'view1', 'le': '10.0'}, 2.0), + Sample('h_bucket', {'view': 'view1', 'le': '+Inf'}, 2.0), + Sample('h_count', {'view': 'view1'}, 2.0), + Sample('h_sum', {'view': 'view2'}, 1.0), + Sample('h_bucket', {'view': 'view2', 'le': '0.005'}, 0.0), + Sample('h_bucket', {'view': 'view2', 'le': '0.01'}, 0.0), + Sample('h_bucket', {'view': 'view2', 'le': '0.025'}, 0.0), + Sample('h_bucket', {'view': 'view2', 'le': '0.05'}, 0.0), + Sample('h_bucket', {'view': 'view2', 'le': '0.075'}, 0.0), + Sample('h_bucket', {'view': 'view2', 'le': '0.1'}, 0.0), + Sample('h_bucket', {'view': 'view2', 'le': '0.25'}, 0.0), + Sample('h_bucket', {'view': 'view2', 'le': '0.5'}, 0.0), + Sample('h_bucket', {'view': 'view2', 'le': '0.75'}, 0.0), + Sample('h_bucket', {'view': 'view2', 'le': '1.0'}, 1.0), + Sample('h_bucket', {'view': 'view2', 'le': '2.5'}, 1.0), + Sample('h_bucket', {'view': 'view2', 'le': '5.0'}, 1.0), + Sample('h_bucket', {'view': 'view2', 'le': '7.5'}, 1.0), + Sample('h_bucket', {'view': 'view2', 'le': '10.0'}, 1.0), + Sample('h_bucket', {'view': 'view2', 'le': '+Inf'}, 1.0), + Sample('h_count', {'view': 'view2'}, 1.0), ] self.assertEqual(metrics['h'].samples, expected_histogram) @@ -347,10 +404,8 @@ def add_label(key, value): m.name: m for m in self.collector.merge(files, accumulate=False) } - metrics['h'].samples.sort( - key=lambda x: (x[0], float(x[1].get('le', 0))) - ) expected_histogram = [ + Sample('h_sum', labels, 6.0), Sample('h_bucket', add_label('le', '0.005'), 0.0), Sample('h_bucket', add_label('le', '0.01'), 0.0), Sample('h_bucket', add_label('le', '0.025'), 0.0), @@ -366,7 +421,6 @@ def add_label(key, value): Sample('h_bucket', add_label('le', '7.5'), 0.0), Sample('h_bucket', add_label('le', '10.0'), 0.0), Sample('h_bucket', add_label('le', '+Inf'), 0.0), - Sample('h_sum', labels, 6.0), ] self.assertEqual(metrics['h'].samples, expected_histogram) From 13df12421e1ba9c621246b9084229e24fda4074e Mon Sep 17 00:00:00 2001 From: Colin Watson Date: Wed, 26 Nov 2025 19:41:33 +0000 Subject: [PATCH 02/11] Relax registry type annotations for exposition (#1149) * Turn Collector into a Protocol We require Python >= 3.9 now, so there's no reason to avoid this any more. Signed-off-by: Colin Watson * Relax registry type annotations for exposition Anything with a suitable `collect` method will do: for instance, it's sometimes useful to be able to define a class whose `collect` method yields all metrics from a registry whose names have a given prefix, and such a class doesn't need to inherit from `CollectorRegistry`. Signed-off-by: Colin Watson --------- Signed-off-by: Colin Watson --- prometheus_client/aiohttp/exposition.py | 4 ++-- prometheus_client/asgi.py | 4 ++-- prometheus_client/bridge/graphite.py | 4 ++-- prometheus_client/exposition.py | 28 ++++++++++++------------- prometheus_client/registry.py | 13 +++++------- 5 files changed, 25 insertions(+), 28 deletions(-) diff --git a/prometheus_client/aiohttp/exposition.py b/prometheus_client/aiohttp/exposition.py index 914fb26f..c1ae254d 100644 --- a/prometheus_client/aiohttp/exposition.py +++ b/prometheus_client/aiohttp/exposition.py @@ -4,11 +4,11 @@ from aiohttp.typedefs import Handler from ..exposition import _bake_output -from ..registry import CollectorRegistry, REGISTRY +from ..registry import Collector, REGISTRY def make_aiohttp_handler( - registry: CollectorRegistry = REGISTRY, + registry: Collector = REGISTRY, disable_compression: bool = False, ) -> Handler: """Create a aiohttp handler which serves the metrics from a registry.""" diff --git a/prometheus_client/asgi.py b/prometheus_client/asgi.py index affd9844..6e527ca9 100644 --- a/prometheus_client/asgi.py +++ b/prometheus_client/asgi.py @@ -2,10 +2,10 @@ from urllib.parse import parse_qs from .exposition import _bake_output -from .registry import CollectorRegistry, REGISTRY +from .registry import Collector, REGISTRY -def make_asgi_app(registry: CollectorRegistry = REGISTRY, disable_compression: bool = False) -> Callable: +def make_asgi_app(registry: Collector = REGISTRY, disable_compression: bool = False) -> Callable: """Create a ASGI app which serves the metrics from a registry.""" async def prometheus_app(scope, receive, send): diff --git a/prometheus_client/bridge/graphite.py b/prometheus_client/bridge/graphite.py index 8cadbedc..235324b2 100755 --- a/prometheus_client/bridge/graphite.py +++ b/prometheus_client/bridge/graphite.py @@ -8,7 +8,7 @@ from timeit import default_timer from typing import Callable, Tuple -from ..registry import CollectorRegistry, REGISTRY +from ..registry import Collector, REGISTRY # Roughly, have to keep to what works as a file name. # We also remove periods, so labels can be distinguished. @@ -48,7 +48,7 @@ def run(self): class GraphiteBridge: def __init__(self, address: Tuple[str, int], - registry: CollectorRegistry = REGISTRY, + registry: Collector = REGISTRY, timeout_seconds: float = 30, _timer: Callable[[], float] = time.time, tags: bool = False, diff --git a/prometheus_client/exposition.py b/prometheus_client/exposition.py index 0d471707..9cb74faa 100644 --- a/prometheus_client/exposition.py +++ b/prometheus_client/exposition.py @@ -19,7 +19,7 @@ from wsgiref.simple_server import make_server, WSGIRequestHandler, WSGIServer from .openmetrics import exposition as openmetrics -from .registry import CollectorRegistry, REGISTRY +from .registry import Collector, REGISTRY from .utils import floatToGoString, parse_version __all__ = ( @@ -118,7 +118,7 @@ def _bake_output(registry, accept_header, accept_encoding_header, params, disabl return '200 OK', headers, output -def make_wsgi_app(registry: CollectorRegistry = REGISTRY, disable_compression: bool = False) -> Callable: +def make_wsgi_app(registry: Collector = REGISTRY, disable_compression: bool = False) -> Callable: """Create a WSGI app which serves the metrics from a registry.""" def prometheus_app(environ, start_response): @@ -223,7 +223,7 @@ def _get_ssl_ctx( def start_wsgi_server( port: int, addr: str = '0.0.0.0', - registry: CollectorRegistry = REGISTRY, + registry: Collector = REGISTRY, certfile: Optional[str] = None, keyfile: Optional[str] = None, client_cafile: Optional[str] = None, @@ -252,12 +252,12 @@ class TmpServer(ThreadingWSGIServer): start_http_server = start_wsgi_server -def generate_latest(registry: CollectorRegistry = REGISTRY, escaping: str = openmetrics.UNDERSCORES) -> bytes: +def generate_latest(registry: Collector = REGISTRY, escaping: str = openmetrics.UNDERSCORES) -> bytes: """ Generates the exposition format using the basic Prometheus text format. Params: - registry: CollectorRegistry to export data from. + registry: Collector to export data from. escaping: Escaping scheme used for metric and label names. Returns: UTF-8 encoded string containing the metrics in text format. @@ -330,7 +330,7 @@ def sample_line(samples): return ''.join(output).encode('utf-8') -def choose_encoder(accept_header: str) -> Tuple[Callable[[CollectorRegistry], bytes], str]: +def choose_encoder(accept_header: str) -> Tuple[Callable[[Collector], bytes], str]: # Python client library accepts a narrower range of content-types than # Prometheus does. accept_header = accept_header or '' @@ -408,7 +408,7 @@ def gzip_accepted(accept_encoding_header: str) -> bool: class MetricsHandler(BaseHTTPRequestHandler): """HTTP handler that gives metrics from ``REGISTRY``.""" - registry: CollectorRegistry = REGISTRY + registry: Collector = REGISTRY def do_GET(self) -> None: # Prepare parameters @@ -429,7 +429,7 @@ def log_message(self, format: str, *args: Any) -> None: """Log nothing.""" @classmethod - def factory(cls, registry: CollectorRegistry) -> type: + def factory(cls, registry: Collector) -> type: """Returns a dynamic MetricsHandler class tied to the passed registry. """ @@ -444,7 +444,7 @@ def factory(cls, registry: CollectorRegistry) -> type: return MyMetricsHandler -def write_to_textfile(path: str, registry: CollectorRegistry, escaping: str = openmetrics.ALLOWUTF8, tmpdir: Optional[str] = None) -> None: +def write_to_textfile(path: str, registry: Collector, escaping: str = openmetrics.ALLOWUTF8, tmpdir: Optional[str] = None) -> None: """Write metrics to the given path. This is intended for use with the Node exporter textfile collector. @@ -592,7 +592,7 @@ def tls_auth_handler( def push_to_gateway( gateway: str, job: str, - registry: CollectorRegistry, + registry: Collector, grouping_key: Optional[Dict[str, Any]] = None, timeout: Optional[float] = 30, handler: Callable = default_handler, @@ -603,7 +603,7 @@ def push_to_gateway( 'http://pushgateway.local', or 'pushgateway.local'. Scheme defaults to 'http' if none is provided `job` is the job label to be attached to all pushed metrics - `registry` is an instance of CollectorRegistry + `registry` is a Collector, normally an instance of CollectorRegistry `grouping_key` please see the pushgateway documentation for details. Defaults to None `timeout` is how long push will attempt to connect before giving up. @@ -641,7 +641,7 @@ def push_to_gateway( def pushadd_to_gateway( gateway: str, job: str, - registry: Optional[CollectorRegistry], + registry: Optional[Collector], grouping_key: Optional[Dict[str, Any]] = None, timeout: Optional[float] = 30, handler: Callable = default_handler, @@ -652,7 +652,7 @@ def pushadd_to_gateway( 'http://pushgateway.local', or 'pushgateway.local'. Scheme defaults to 'http' if none is provided `job` is the job label to be attached to all pushed metrics - `registry` is an instance of CollectorRegistry + `registry` is a Collector, normally an instance of CollectorRegistry `grouping_key` please see the pushgateway documentation for details. Defaults to None `timeout` is how long push will attempt to connect before giving up. @@ -702,7 +702,7 @@ def _use_gateway( method: str, gateway: str, job: str, - registry: Optional[CollectorRegistry], + registry: Optional[Collector], grouping_key: Optional[Dict[str, Any]], timeout: Optional[float], handler: Callable, diff --git a/prometheus_client/registry.py b/prometheus_client/registry.py index 8de4ce91..9934117d 100644 --- a/prometheus_client/registry.py +++ b/prometheus_client/registry.py @@ -1,24 +1,21 @@ -from abc import ABC, abstractmethod import copy from threading import Lock -from typing import Dict, Iterable, List, Optional +from typing import Dict, Iterable, List, Optional, Protocol from .metrics_core import Metric -# Ideally this would be a Protocol, but Protocols are only available in Python >= 3.8. -class Collector(ABC): - @abstractmethod +class Collector(Protocol): def collect(self) -> Iterable[Metric]: - pass + """Collect metrics.""" -class _EmptyCollector(Collector): +class _EmptyCollector: def collect(self) -> Iterable[Metric]: return [] -class CollectorRegistry(Collector): +class CollectorRegistry: """Metric collector registry. Collectors must have a no-argument method 'collect' that returns a list of From 7b9959209492c06968785c66bc6ea2316d156f91 Mon Sep 17 00:00:00 2001 From: ritesh-avesha <104001014+ritesh-avesha@users.noreply.github.com> Date: Fri, 12 Dec 2025 22:30:45 +0530 Subject: [PATCH 03/11] Added compression support in pushgateway (#1144) * feat(): Added compression support in pushgateway Signed-off-by: ritesh-avesha * fix(): Incorporated changes for PR review comments Signed-off-by: ritesh-avesha * fix(): Incorporated changes for PR review comments, lint issues Signed-off-by: ritesh-avesha * fix(): lint issues Signed-off-by: ritesh-avesha --------- Signed-off-by: ritesh-avesha --- docs/content/exporting/pushgateway.md | 14 ++++++ prometheus_client/exposition.py | 66 +++++++++++++++++++++++---- tests/test_exposition.py | 25 ++++++++++ tox.ini | 1 + 4 files changed, 96 insertions(+), 10 deletions(-) diff --git a/docs/content/exporting/pushgateway.md b/docs/content/exporting/pushgateway.md index bf5eb112..d9f9a945 100644 --- a/docs/content/exporting/pushgateway.md +++ b/docs/content/exporting/pushgateway.md @@ -54,6 +54,20 @@ g.set_to_current_time() push_to_gateway('localhost:9091', job='batchA', registry=registry, handler=my_auth_handler) ``` +# Compressing data before sending to pushgateway +Pushgateway (version >= 1.5.0) supports gzip and snappy compression (v > 1.6.0). This can help in network constrained environments. +To compress a push request, set the `compression` argument to `'gzip'` or `'snappy'`: +```python +push_to_gateway( + 'localhost:9091', + job='batchA', + registry=registry, + handler=my_auth_handler, + compression='gzip', +) +``` +Snappy compression requires the optional [`python-snappy`](https://github.com/andrix/python-snappy) package. + TLS Auth is also supported when using the push gateway with a special handler. ```python diff --git a/prometheus_client/exposition.py b/prometheus_client/exposition.py index 9cb74faa..ca06d916 100644 --- a/prometheus_client/exposition.py +++ b/prometheus_client/exposition.py @@ -9,7 +9,9 @@ import ssl import sys import threading -from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, Union +from typing import ( + Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple, Union, +) from urllib.error import HTTPError from urllib.parse import parse_qs, quote_plus, urlparse from urllib.request import ( @@ -22,6 +24,13 @@ from .registry import Collector, REGISTRY from .utils import floatToGoString, parse_version +try: + import snappy # type: ignore + SNAPPY_AVAILABLE = True +except ImportError: + snappy = None # type: ignore + SNAPPY_AVAILABLE = False + __all__ = ( 'CONTENT_TYPE_LATEST', 'CONTENT_TYPE_PLAIN_0_0_4', @@ -46,6 +55,7 @@ """Content type of the latest format""" CONTENT_TYPE_LATEST = CONTENT_TYPE_PLAIN_1_0_0 +CompressionType = Optional[Literal['gzip', 'snappy']] class _PrometheusRedirectHandler(HTTPRedirectHandler): @@ -596,6 +606,7 @@ def push_to_gateway( grouping_key: Optional[Dict[str, Any]] = None, timeout: Optional[float] = 30, handler: Callable = default_handler, + compression: CompressionType = None, ) -> None: """Push metrics to the given pushgateway. @@ -632,10 +643,12 @@ def push_to_gateway( failure. 'content' is the data which should be used to form the HTTP Message Body. + `compression` selects the payload compression. Supported values are 'gzip' + and 'snappy'. Defaults to None (no compression). This overwrites all metrics with the same job and grouping_key. This uses the PUT HTTP method.""" - _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) + _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler, compression) def pushadd_to_gateway( @@ -645,6 +658,7 @@ def pushadd_to_gateway( grouping_key: Optional[Dict[str, Any]] = None, timeout: Optional[float] = 30, handler: Callable = default_handler, + compression: CompressionType = None, ) -> None: """PushAdd metrics to the given pushgateway. @@ -663,10 +677,12 @@ def pushadd_to_gateway( will be carried out by a default handler. See the 'prometheus_client.push_to_gateway' documentation for implementation requirements. + `compression` selects the payload compression. Supported values are 'gzip' + and 'snappy'. Defaults to None (no compression). This replaces metrics with the same name, job and grouping_key. This uses the POST HTTP method.""" - _use_gateway('POST', gateway, job, registry, grouping_key, timeout, handler) + _use_gateway('POST', gateway, job, registry, grouping_key, timeout, handler, compression) def delete_from_gateway( @@ -706,6 +722,7 @@ def _use_gateway( grouping_key: Optional[Dict[str, Any]], timeout: Optional[float], handler: Callable, + compression: CompressionType = None, ) -> None: gateway_url = urlparse(gateway) # See https://bugs.python.org/issue27657 for details on urlparse in py>=3.7.6. @@ -715,24 +732,53 @@ def _use_gateway( gateway = gateway.rstrip('/') url = '{}/metrics/{}/{}'.format(gateway, *_escape_grouping_key("job", job)) - data = b'' - if method != 'DELETE': - if registry is None: - registry = REGISTRY - data = generate_latest(registry) - if grouping_key is None: grouping_key = {} url += ''.join( '/{}/{}'.format(*_escape_grouping_key(str(k), str(v))) for k, v in sorted(grouping_key.items())) + data = b'' + headers: List[Tuple[str, str]] = [] + if method != 'DELETE': + if registry is None: + registry = REGISTRY + data = generate_latest(registry) + data, headers = _compress_payload(data, compression) + else: + # DELETE requests still need Content-Type header per test expectations + headers = [('Content-Type', CONTENT_TYPE_PLAIN_0_0_4)] + if compression is not None: + raise ValueError('Compression is not supported for DELETE requests.') + handler( url=url, method=method, timeout=timeout, - headers=[('Content-Type', CONTENT_TYPE_PLAIN_0_0_4)], data=data, + headers=headers, data=data, )() +def _compress_payload(data: bytes, compression: CompressionType) -> Tuple[bytes, List[Tuple[str, str]]]: + headers = [('Content-Type', CONTENT_TYPE_PLAIN_0_0_4)] + if compression is None: + return data, headers + + encoding = compression.lower() + if encoding == 'gzip': + headers.append(('Content-Encoding', 'gzip')) + return gzip.compress(data), headers + if encoding == 'snappy': + if not SNAPPY_AVAILABLE: + raise RuntimeError('Snappy compression requires the python-snappy package to be installed.') + headers.append(('Content-Encoding', 'snappy')) + compressor = snappy.StreamCompressor() + compressed = compressor.compress(data) + flush = getattr(compressor, 'flush', None) + if callable(flush): + compressed += flush() + return compressed, headers + raise ValueError(f"Unsupported compression type: {compression}") + + def _escape_grouping_key(k, v): if v == "": # Per https://github.com/prometheus/pushgateway/pull/346. diff --git a/tests/test_exposition.py b/tests/test_exposition.py index 3dd5e378..aceff738 100644 --- a/tests/test_exposition.py +++ b/tests/test_exposition.py @@ -1,3 +1,4 @@ +import gzip from http.server import BaseHTTPRequestHandler, HTTPServer import os import threading @@ -404,6 +405,30 @@ def test_push_with_trailing_slash(self): self.assertNotIn('//', self.requests[0][0].path) + def test_push_with_gzip_compression(self): + push_to_gateway(self.address, "my_job", self.registry, compression='gzip') + request, body = self.requests[0] + self.assertEqual(request.headers.get('content-encoding'), 'gzip') + decompressed = gzip.decompress(body) + self.assertEqual(decompressed, b'# HELP g help\n# TYPE g gauge\ng 0.0\n') + + def test_push_with_snappy_compression(self): + snappy = pytest.importorskip('snappy') + push_to_gateway(self.address, "my_job", self.registry, compression='snappy') + request, body = self.requests[0] + self.assertEqual(request.headers.get('content-encoding'), 'snappy') + decompressor = snappy.StreamDecompressor() + decompressed = decompressor.decompress(body) + flush = getattr(decompressor, 'flush', None) + if callable(flush): + decompressed += flush() + self.assertEqual(decompressed, b'# HELP g help\n# TYPE g gauge\ng 0.0\n') + + def test_push_with_invalid_compression(self): + with self.assertRaisesRegex(ValueError, 'Unsupported compression type'): + push_to_gateway(self.address, "my_job", self.registry, compression='brotli') + self.assertEqual(self.requests, []) + def test_instance_ip_grouping_key(self): self.assertTrue('' != instance_ip_grouping_key()['instance']) diff --git a/tox.ini b/tox.ini index ccf95cc2..45a6baf3 100644 --- a/tox.ini +++ b/tox.ini @@ -10,6 +10,7 @@ deps = attrs {py3.9,pypy3.9}: twisted {py3.9,pypy3.9}: aiohttp + {py3.9}: python-snappy commands = coverage run --parallel -m pytest {posargs} [testenv:py3.9-nooptionals] From e1cdc203b1cf5f15c7b9a64d79fccc7907a62ca3 Mon Sep 17 00:00:00 2001 From: Julie Rymer Date: Mon, 5 Jan 2026 22:59:43 +0100 Subject: [PATCH 04/11] Add Django exporter (#1088) (#1143) Signed-off-by: Julie Rymer --- docs/content/exporting/http/django.md | 47 +++++++++++++++++++++++++ mypy.ini | 2 +- prometheus_client/django/__init__.py | 5 +++ prometheus_client/django/exposition.py | 44 +++++++++++++++++++++++ pyproject.toml | 3 ++ tests/test_django.py | 48 ++++++++++++++++++++++++++ tox.ini | 1 + 7 files changed, 149 insertions(+), 1 deletion(-) create mode 100644 docs/content/exporting/http/django.md create mode 100644 prometheus_client/django/__init__.py create mode 100644 prometheus_client/django/exposition.py create mode 100644 tests/test_django.py diff --git a/docs/content/exporting/http/django.md b/docs/content/exporting/http/django.md new file mode 100644 index 00000000..a900a3a2 --- /dev/null +++ b/docs/content/exporting/http/django.md @@ -0,0 +1,47 @@ +--- +title: Django +weight: 5 +--- + +To use Prometheus with [Django](https://www.djangoproject.com/) you can use the provided view class +to add a metrics endpoint to your app. + +```python +# urls.py + +from django.urls import path +from prometheus_client.django import PrometheusDjangoView + +urlpatterns = [ + # ... any other urls that you want + path("metrics/", PrometheusDjangoView.as_view(), name="prometheus-metrics"), + # ... still more urls +] +``` + +By default, Multiprocessing support is activated if environment variable `PROMETHEUS_MULTIPROC_DIR` is set. +You can override this through the view arguments: + +```python +from django.conf import settings + +urlpatterns = [ + path( + "metrics/", + PrometheusDjangoView.as_view( + multiprocess_mode=settings.YOUR_SETTING # or any boolean value + ), + name="prometheus-metrics", + ), +] +``` + +Full multiprocessing instructions are provided [here]({{< ref "/multiprocess" >}}). + +# django-prometheus + +The included `PrometheusDjangoView` is useful if you want to define your own metrics from scratch. + +An external package called [django-prometheus](https://github.com/django-commons/django-prometheus/) +can be used instead if you want to get a bunch of ready-made monitoring metrics for your Django application +and easily benefit from utilities such as models monitoring. diff --git a/mypy.ini b/mypy.ini index fe372d07..3aa142c1 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,5 +1,5 @@ [mypy] -exclude = prometheus_client/decorator.py|prometheus_client/twisted|tests/test_twisted.py +exclude = prometheus_client/decorator.py|prometheus_client/twisted|tests/test_twisted.py|prometheus_client/django|tests/test_django.py implicit_reexport = False disallow_incomplete_defs = True diff --git a/prometheus_client/django/__init__.py b/prometheus_client/django/__init__.py new file mode 100644 index 00000000..280dbfb0 --- /dev/null +++ b/prometheus_client/django/__init__.py @@ -0,0 +1,5 @@ +from .exposition import PrometheusDjangoView + +__all__ = [ + "PrometheusDjangoView", +] diff --git a/prometheus_client/django/exposition.py b/prometheus_client/django/exposition.py new file mode 100644 index 00000000..085e8fcf --- /dev/null +++ b/prometheus_client/django/exposition.py @@ -0,0 +1,44 @@ +import os + +from django.http import HttpResponse +from django.views import View + +import prometheus_client +from prometheus_client import multiprocess +from prometheus_client.exposition import _bake_output +from prometheus_client.registry import registry + + +class PrometheusDjangoView(View): + multiprocess_mode: bool = "PROMETHEUS_MULTIPROC_DIR" in os.environ or "prometheus_multiproc_dir" in os.environ + registry: prometheus_client.CollectorRegistry = None + + def get(self, request, *args, **kwargs): + if self.registry is None: + if self.multiprocess_mode: + self.registry = prometheus_client.CollectorRegistry() + multiprocess.MultiProcessCollector(registry) + else: + self.registry = prometheus_client.REGISTRY + accept_header = request.headers.get("Accept") + accept_encoding_header = request.headers.get("Accept-Encoding") + # Bake output + status, headers, output = _bake_output( + registry=self.registry, + accept_header=accept_header, + accept_encoding_header=accept_encoding_header, + params=request.GET, + disable_compression=False, + ) + status = int(status.split(" ")[0]) + return HttpResponse( + output, + status=status, + headers=headers, + ) + + def options(self, request, *args, **kwargs): + return HttpResponse( + status=200, + headers={"Allow": "OPTIONS,GET"}, + ) diff --git a/pyproject.toml b/pyproject.toml index 1d8527a0..3961482f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,6 +47,9 @@ twisted = [ aiohttp = [ "aiohttp", ] +django = [ + "django", +] [project.urls] Homepage = "https://github.com/prometheus/client_python" diff --git a/tests/test_django.py b/tests/test_django.py new file mode 100644 index 00000000..659bb3f6 --- /dev/null +++ b/tests/test_django.py @@ -0,0 +1,48 @@ +from unittest import skipUnless + +from prometheus_client import CollectorRegistry, Counter, generate_latest +from prometheus_client.openmetrics.exposition import ALLOWUTF8 + +try: + import django + from django.test import RequestFactory, TestCase + + from prometheus_client.django import PrometheusDjangoView + + HAVE_DJANGO = True +except ImportError: + from unittest import TestCase + + HAVE_DJANGO = False + +else: + from django.conf import settings + + if not settings.configured: + settings.configure( + DATABASES={ + "default": { + "ENGINE": "django.db.backends.sqlite3", + 'NAME': ':memory:' + } + }, + INSTALLED_APPS=[], + ) + django.setup() + + +class MetricsResourceTest(TestCase): + @skipUnless(HAVE_DJANGO, "Don't have django installed.") + def setUp(self): + self.registry = CollectorRegistry() + self.factory = RequestFactory() + + def test_reports_metrics(self): + c = Counter('cc', 'A counter', registry=self.registry) + c.inc() + + request = self.factory.get("/metrics") + + response = PrometheusDjangoView.as_view(registry=self.registry)(request) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.content, generate_latest(self.registry, ALLOWUTF8)) diff --git a/tox.ini b/tox.ini index 45a6baf3..992bd0a7 100644 --- a/tox.ini +++ b/tox.ini @@ -10,6 +10,7 @@ deps = attrs {py3.9,pypy3.9}: twisted {py3.9,pypy3.9}: aiohttp + {py3.9,pypy3.9}: django {py3.9}: python-snappy commands = coverage run --parallel -m pytest {posargs} From c5024d310fbfcba45a5e9db62e337a3a7930ea16 Mon Sep 17 00:00:00 2001 From: Chris Marchbanks Date: Mon, 12 Jan 2026 13:10:03 -0700 Subject: [PATCH 05/11] Release 0.24.0 Signed-off-by: Chris Marchbanks --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 3961482f..d45ef12d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "prometheus_client" -version = "0.23.1" +version = "0.24.0" description = "Python client for the Prometheus monitoring system." readme = "README.md" license = "Apache-2.0 AND BSD-2-Clause" From 6f0e967c1f7a408b75861d6833a8d303874be95d Mon Sep 17 00:00:00 2001 From: Jelle van der Waa Date: Wed, 14 Jan 2026 16:23:36 +0100 Subject: [PATCH 06/11] Pass correct registry to MultiProcessCollector (#1152) `registry` does not exists in prometheus_client.registry, as that causes an ImportError the test was skipped in the 3.9 scenario. Signed-off-by: Jelle van der Waa --- prometheus_client/django/exposition.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/prometheus_client/django/exposition.py b/prometheus_client/django/exposition.py index 085e8fcf..71fc8d8a 100644 --- a/prometheus_client/django/exposition.py +++ b/prometheus_client/django/exposition.py @@ -6,7 +6,6 @@ import prometheus_client from prometheus_client import multiprocess from prometheus_client.exposition import _bake_output -from prometheus_client.registry import registry class PrometheusDjangoView(View): @@ -17,7 +16,7 @@ def get(self, request, *args, **kwargs): if self.registry is None: if self.multiprocess_mode: self.registry = prometheus_client.CollectorRegistry() - multiprocess.MultiProcessCollector(registry) + multiprocess.MultiProcessCollector(self.registry) else: self.registry = prometheus_client.REGISTRY accept_header = request.headers.get("Accept") From f417f6ea8f058165a1934e368fed245e91aafc14 Mon Sep 17 00:00:00 2001 From: Chris Marchbanks Date: Wed, 14 Jan 2026 08:24:50 -0700 Subject: [PATCH 07/11] Release 0.24.1 Signed-off-by: Chris Marchbanks --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index d45ef12d..ed3ef389 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "prometheus_client" -version = "0.24.0" +version = "0.24.1" description = "Python client for the Prometheus monitoring system." readme = "README.md" license = "Apache-2.0 AND BSD-2-Clause" From a8541354519d04852d24688845f1d2d495eef59c Mon Sep 17 00:00:00 2001 From: Chris Marchbanks Date: Wed, 21 Jan 2026 16:41:10 -0700 Subject: [PATCH 08/11] Migrate to Github Actions (#1153) * Migrate to Github Actions * Pin github actions versions --------- Signed-off-by: Chris Marchbanks --- .circleci/config.yml | 93 ----------------------- .github/workflows/ci.yaml | 110 ++++++++++++++++++++++++++++ .github/workflows/github-pages.yaml | 18 +++-- 3 files changed, 121 insertions(+), 100 deletions(-) delete mode 100644 .circleci/config.yml create mode 100644 .github/workflows/ci.yaml diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index f29bd265..00000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,93 +0,0 @@ ---- -version: 2.1 - -executors: - python: - docker: - - image: cimg/python:3.9 - -jobs: - flake8_lint: - executor: python - steps: - - checkout - - run: pip install tox - - run: tox -e flake8 - isort_lint: - executor: python - steps: - - checkout - - run: pip install tox - - run: tox -e isort - mypy_lint: - executor: python - steps: - - checkout - - run: pip install tox - - run: tox -e mypy - test: - parameters: - python: - type: string - docker: - - image: cimg/python:<< parameters.python >> - environment: - TOXENV: "py<< parameters.python >>" - steps: - - checkout - - run: echo 'export PATH=$HOME/.local/bin:$PATH' >> $BASH_ENV - - run: pip install --user tox "virtualenv<20.22.0" - - run: tox - test_nooptionals: - parameters: - python: - type: string - docker: - - image: cimg/python:<< parameters.python >> - environment: - TOXENV: "py<< parameters.python >>-nooptionals" - steps: - - checkout - - run: pip install tox - - run: tox - test_pypy: - parameters: - python: - type: string - docker: - - image: pypy:<< parameters.python >> - environment: - TOXENV: "pypy<< parameters.python >>" - steps: - - checkout - - run: pip install tox - - run: tox - - -workflows: - version: 2 - client_python: - jobs: - - flake8_lint - - isort_lint - - mypy_lint - - test: - matrix: - parameters: - python: - - "3.9.18" - - "3.10" - - "3.11" - - "3.12" - - "3.13" - - "3.14" - - test_nooptionals: - matrix: - parameters: - python: - - "3.9" - - test_pypy: - matrix: - parameters: - python: - - "3.9" diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 00000000..a7e4e094 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,110 @@ +name: CI + +on: + push: + branches: [ master ] + pull_request: + +permissions: + contents: read + +jobs: + flake8_lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - name: Set up Python + uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 + with: + python-version: '3.9' + - name: Install tox + run: pip install tox + - name: Run flake8 + run: tox -e flake8 + + isort_lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - name: Set up Python + uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 + with: + python-version: '3.9' + - name: Install tox + run: pip install tox + - name: Run isort + run: tox -e isort + + mypy_lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - name: Set up Python + uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 + with: + python-version: '3.9' + - name: Install tox + run: pip install tox + - name: Run mypy + run: tox -e mypy + + test: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.9', '3.10', '3.11', '3.12', '3.13', '3.14'] + steps: + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + pip install --user tox "virtualenv<20.22.0" + echo "$HOME/.local/bin" >> $GITHUB_PATH + - name: Set tox environment + id: toxenv + run: | + VERSION="${{ matrix.python-version }}" + # Extract major.minor version (strip patch if present) + TOX_VERSION=$(echo "$VERSION" | cut -d. -f1,2) + echo "toxenv=py${TOX_VERSION}" >> $GITHUB_OUTPUT + - name: Run tests + run: tox + env: + TOXENV: ${{ steps.toxenv.outputs.toxenv }} + + test_nooptionals: + runs-on: ubuntu-latest + env: + PYTHON_VERSION: '3.9' + steps: + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - name: Set up Python + uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 + with: + python-version: ${{ env.PYTHON_VERSION }} + - name: Install tox + run: pip install tox + - name: Run tests without optional dependencies + run: tox + env: + TOXENV: py${{ env.PYTHON_VERSION }}-nooptionals + + test_pypy: + runs-on: ubuntu-latest + env: + PYTHON_VERSION: '3.9' + steps: + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - name: Set up PyPy + uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 + with: + python-version: pypy-${{ env.PYTHON_VERSION }} + - name: Install tox + run: pip install tox + - name: Run tests with PyPy + run: tox + env: + TOXENV: pypy${{ env.PYTHON_VERSION }} diff --git a/.github/workflows/github-pages.yaml b/.github/workflows/github-pages.yaml index 621f2d73..d8db8cbc 100644 --- a/.github/workflows/github-pages.yaml +++ b/.github/workflows/github-pages.yaml @@ -11,9 +11,6 @@ on: # Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages permissions: contents: read - pages: write - id-token: write - actions: read # Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. # However, do NOT cancel in-progress runs as we want to allow these production deployments to complete. @@ -32,6 +29,9 @@ jobs: runs-on: ubuntu-latest env: HUGO_VERSION: 0.145.0 + permissions: + pages: write + id-token: write steps: - name: Install Hugo CLI run: | @@ -40,13 +40,13 @@ jobs: #- name: Install Dart Sass # run: sudo snap install dart-sass - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: submodules: recursive fetch-depth: 0 - name: Setup Pages id: pages - uses: actions/configure-pages@v5 + uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b # v5.0.0 - name: Install Node.js dependencies run: "[[ -f package-lock.json || -f npm-shrinkwrap.json ]] && npm ci || true" working-directory: ./docs @@ -62,7 +62,7 @@ jobs: --baseURL "${{ steps.pages.outputs.base_url }}/" working-directory: ./docs - name: Upload artifact - uses: actions/upload-pages-artifact@v3 + uses: actions/upload-pages-artifact@7b1f4a764d45c48632c6b24a0339c27f5614fb0b # v4.0.0 with: path: ./docs/public @@ -73,7 +73,11 @@ jobs: url: ${{ steps.deployment.outputs.page_url }} runs-on: ubuntu-latest needs: build + permissions: + pages: write + id-token: write + actions: read steps: - name: Deploy to GitHub Pages id: deployment - uses: actions/deploy-pages@v4 + uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4.0.5 From 1cf53feae63b6ecb0bd76eee80582a0fba957e09 Mon Sep 17 00:00:00 2001 From: Varun Chawla <34209028+veeceey@users.noreply.github.com> Date: Mon, 9 Feb 2026 07:04:33 -0800 Subject: [PATCH 09/11] Fix server shutdown documentation (#1155) Add server.server_close() call to shutdown example to properly release the port. Without this call, attempting to restart the server on the same port results in "Address already in use" error. Fixes #1068 Signed-off-by: Varun Chawla --- docs/content/exporting/http/_index.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/content/exporting/http/_index.md b/docs/content/exporting/http/_index.md index dc1b8f2c..f7a6aac6 100644 --- a/docs/content/exporting/http/_index.md +++ b/docs/content/exporting/http/_index.md @@ -24,6 +24,7 @@ to shutdown the server gracefully: ```python server, t = start_http_server(8000) server.shutdown() +server.server_close() t.join() ``` From 671f75c6f1f04838995fadd57cda21beee01838b Mon Sep 17 00:00:00 2001 From: Varun Chawla <34209028+veeceey@users.noreply.github.com> Date: Mon, 9 Feb 2026 08:05:29 -0800 Subject: [PATCH 10/11] Fix spaces in grouping key values for push_to_gateway (#1156) Use base64 encoding for grouping key values containing spaces, similar to how values with slashes are handled. This prevents spaces from being converted to '+' signs by quote_plus(). Fixes #1064 Signed-off-by: Varun Chawla --- prometheus_client/exposition.py | 3 ++- tests/test_exposition.py | 7 +++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/prometheus_client/exposition.py b/prometheus_client/exposition.py index ca06d916..2d402a0f 100644 --- a/prometheus_client/exposition.py +++ b/prometheus_client/exposition.py @@ -783,8 +783,9 @@ def _escape_grouping_key(k, v): if v == "": # Per https://github.com/prometheus/pushgateway/pull/346. return k + "@base64", "=" - elif '/' in v: + elif '/' in v or ' ' in v: # Added in Pushgateway 0.9.0. + # Use base64 encoding for values containing slashes or spaces return k + "@base64", base64.urlsafe_b64encode(v.encode("utf-8")).decode("utf-8") else: return k, quote_plus(v) diff --git a/tests/test_exposition.py b/tests/test_exposition.py index aceff738..a3c97820 100644 --- a/tests/test_exposition.py +++ b/tests/test_exposition.py @@ -301,6 +301,13 @@ def test_push_with_groupingkey_empty_label(self): self.assertEqual(self.requests[0][0].headers.get('content-type'), CONTENT_TYPE_PLAIN_0_0_4) self.assertEqual(self.requests[0][1], b'# HELP g help\n# TYPE g gauge\ng 0.0\n') + def test_push_with_groupingkey_with_spaces(self): + push_to_gateway(self.address, "my_job", self.registry, {'label': 'value with spaces'}) + self.assertEqual(self.requests[0][0].command, 'PUT') + self.assertEqual(self.requests[0][0].path, '/metrics/job/my_job/label@base64/dmFsdWUgd2l0aCBzcGFjZXM=') + self.assertEqual(self.requests[0][0].headers.get('content-type'), CONTENT_TYPE_PLAIN_0_0_4) + self.assertEqual(self.requests[0][1], b'# HELP g help\n# TYPE g gauge\ng 0.0\n') + def test_push_with_complex_groupingkey(self): push_to_gateway(self.address, "my_job", self.registry, {'a': 9, 'b': 'a/ z'}) self.assertEqual(self.requests[0][0].command, 'PUT') From 8673912276bdca7ddbca5d163eb11422b546bffb Mon Sep 17 00:00:00 2001 From: Mathias Kende Date: Wed, 18 Feb 2026 21:56:45 +0100 Subject: [PATCH 11/11] Support MultiProcessCollector in RestrictedRegistry. (#1150) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Support MultiProcessCollector in RestrictedRegistry. This change makes it so that the RestrictedRegistry will always attempt to collect metrics from a collector for which it couldn’t find any metrics name. Although this can be used generally, this is meant to be used with MultiProcessCollector. This changes the current behavior of the code but should be somehow safe as it enables filtering in case where it was not working previously. If this is an issue, an alternative approach with an explicit flag could be used (set either in the MultiProcessCollector or in the registry). The intent here is to allow collecting a subset of metrics from production fastapi servers (running in multiprocess mode). So not having to change the library usage in these servers is advantageous to have filtering work out-of-the-box with this change. Signed-off-by: Mathias Kende * Make the new support for collectors without names be explicit. This adds a parameters to the constructor of CollectorRegistry to allow that new behavior rather than make it be the default. Signed-off-by: Mathias Kende * Fix comments Signed-off-by: Mathias Kende --------- Signed-off-by: Mathias Kende --- docs/content/multiprocess/_index.md | 7 +++++-- prometheus_client/registry.py | 9 +++++++-- tests/test_asgi.py | 29 +++++++++++++++++++++++++++ tests/test_core.py | 18 +++++++++++++++++ tests/test_multiprocess.py | 31 ++++++++++++++++++++++++++++- 5 files changed, 89 insertions(+), 5 deletions(-) diff --git a/docs/content/multiprocess/_index.md b/docs/content/multiprocess/_index.md index 33507cd9..42ea6a67 100644 --- a/docs/content/multiprocess/_index.md +++ b/docs/content/multiprocess/_index.md @@ -10,9 +10,12 @@ it's common to have processes rather than threads to handle large workloads. To handle this the client library can be put in multiprocess mode. This comes with a number of limitations: -- Registries can not be used as normal, all instantiated metrics are exported +- Registries can not be used as normal: + - all instantiated metrics are collected - Registering metrics to a registry later used by a `MultiProcessCollector` may cause duplicate metrics to be exported + - Filtering on metrics works if and only if the constructor was called with + `support_collectors_without_names=True` and it but might be inefficient. - Custom collectors do not work (e.g. cpu and memory metrics) - Gauges cannot use `set_function` - Info and Enum metrics do not work @@ -49,7 +52,7 @@ MY_COUNTER = Counter('my_counter', 'Description of my counter') # Expose metrics. def app(environ, start_response): - registry = CollectorRegistry() + registry = CollectorRegistry(support_collectors_without_names=True) multiprocess.MultiProcessCollector(registry) data = generate_latest(registry) status = '200 OK' diff --git a/prometheus_client/registry.py b/prometheus_client/registry.py index 9934117d..c2b55d15 100644 --- a/prometheus_client/registry.py +++ b/prometheus_client/registry.py @@ -23,12 +23,15 @@ class CollectorRegistry: exposition formats. """ - def __init__(self, auto_describe: bool = False, target_info: Optional[Dict[str, str]] = None): + def __init__(self, auto_describe: bool = False, target_info: Optional[Dict[str, str]] = None, + support_collectors_without_names: bool = False): self._collector_to_names: Dict[Collector, List[str]] = {} self._names_to_collectors: Dict[str, Collector] = {} self._auto_describe = auto_describe self._lock = Lock() self._target_info: Optional[Dict[str, str]] = {} + self._support_collectors_without_names = support_collectors_without_names + self._collectors_without_names: List[Collector] = [] self.set_target_info(target_info) def register(self, collector: Collector) -> None: @@ -43,6 +46,8 @@ def register(self, collector: Collector) -> None: for name in names: self._names_to_collectors[name] = collector self._collector_to_names[collector] = names + if self._support_collectors_without_names and not names: + self._collectors_without_names.append(collector) def unregister(self, collector: Collector) -> None: """Remove a collector from the registry.""" @@ -145,7 +150,7 @@ def __init__(self, names: Iterable[str], registry: CollectorRegistry): self._registry = registry def collect(self) -> Iterable[Metric]: - collectors = set() + collectors = set(self._registry._collectors_without_names) target_info_metric = None with self._registry._lock: if 'target_info' in self._name_set and self._registry._target_info: diff --git a/tests/test_asgi.py b/tests/test_asgi.py index d4933cec..6e795e21 100644 --- a/tests/test_asgi.py +++ b/tests/test_asgi.py @@ -223,3 +223,32 @@ def test_qs_parsing(self): asyncio.new_event_loop().run_until_complete( self.communicator.wait() ) + + def test_qs_parsing_multi(self): + """Only metrics that match the 'name[]' query string param appear""" + + app = make_asgi_app(self.registry) + metrics = [ + ("asdf", "first test metric", 1), + ("bsdf", "second test metric", 2), + ("csdf", "third test metric", 3) + ] + + for m in metrics: + self.increment_metrics(*m) + + self.seed_app(app) + self.scope['query_string'] = "&".join([f"name[]={m[0]}_total" for m in metrics[0:2]]).encode("utf-8") + self.send_default_request() + + outputs = self.get_all_output() + response_body = outputs[1] + output = response_body['body'].decode('utf8') + + self.assert_metrics(output, *metrics[0]) + self.assert_metrics(output, *metrics[1]) + self.assert_not_metrics(output, *metrics[2]) + + asyncio.new_event_loop().run_until_complete( + self.communicator.wait() + ) diff --git a/tests/test_core.py b/tests/test_core.py index c7c9c14f..66492c6f 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -1024,6 +1024,24 @@ def test_restricted_registry_does_not_call_extra(self): self.assertEqual([m], list(registry.restricted_registry(['s_sum']).collect())) mock_collector.collect.assert_not_called() + def test_restricted_registry_ignore_no_names_collectors(self): + from unittest.mock import MagicMock + registry = CollectorRegistry() + mock_collector = MagicMock() + mock_collector.describe.return_value = [] + registry.register(mock_collector) + self.assertEqual(list(registry.restricted_registry(['metric']).collect()), []) + mock_collector.collect.assert_not_called() + + def test_restricted_registry_collects_no_names_collectors(self): + from unittest.mock import MagicMock + registry = CollectorRegistry(support_collectors_without_names=True) + mock_collector = MagicMock() + mock_collector.describe.return_value = [] + registry.register(mock_collector) + self.assertEqual(list(registry.restricted_registry(['metric']).collect()), []) + mock_collector.collect.assert_called() + def test_restricted_registry_does_not_yield_while_locked(self): registry = CollectorRegistry(target_info={'foo': 'bar'}) Summary('s', 'help', registry=registry).observe(7) diff --git a/tests/test_multiprocess.py b/tests/test_multiprocess.py index c2f71d26..ee0c7423 100644 --- a/tests/test_multiprocess.py +++ b/tests/test_multiprocess.py @@ -52,7 +52,7 @@ def setUp(self): self.tempdir = tempfile.mkdtemp() os.environ['PROMETHEUS_MULTIPROC_DIR'] = self.tempdir values.ValueClass = MultiProcessValue(lambda: 123) - self.registry = CollectorRegistry() + self.registry = CollectorRegistry(support_collectors_without_names=True) self.collector = MultiProcessCollector(self.registry) @property @@ -358,6 +358,35 @@ def add_label(key, value): self.assertEqual(metrics['h'].samples, expected_histogram) + def test_restrict(self): + pid = 0 + values.ValueClass = MultiProcessValue(lambda: pid) + labels = {i: i for i in 'abcd'} + + def add_label(key, value): + l = labels.copy() + l[key] = value + return l + + c = Counter('c', 'help', labelnames=labels.keys(), registry=None) + g = Gauge('g', 'help', labelnames=labels.keys(), registry=None) + + c.labels(**labels).inc(1) + g.labels(**labels).set(1) + + pid = 1 + + c.labels(**labels).inc(1) + g.labels(**labels).set(1) + + metrics = {m.name: m for m in self.registry.restricted_registry(['c_total']).collect()} + + self.assertEqual(metrics.keys(), {'c'}) + + self.assertEqual( + metrics['c'].samples, [Sample('c_total', labels, 2.0)] + ) + def test_collect_preserves_help(self): pid = 0 values.ValueClass = MultiProcessValue(lambda: pid)