diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index f29bd265..00000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,93 +0,0 @@ ---- -version: 2.1 - -executors: - python: - docker: - - image: cimg/python:3.9 - -jobs: - flake8_lint: - executor: python - steps: - - checkout - - run: pip install tox - - run: tox -e flake8 - isort_lint: - executor: python - steps: - - checkout - - run: pip install tox - - run: tox -e isort - mypy_lint: - executor: python - steps: - - checkout - - run: pip install tox - - run: tox -e mypy - test: - parameters: - python: - type: string - docker: - - image: cimg/python:<< parameters.python >> - environment: - TOXENV: "py<< parameters.python >>" - steps: - - checkout - - run: echo 'export PATH=$HOME/.local/bin:$PATH' >> $BASH_ENV - - run: pip install --user tox "virtualenv<20.22.0" - - run: tox - test_nooptionals: - parameters: - python: - type: string - docker: - - image: cimg/python:<< parameters.python >> - environment: - TOXENV: "py<< parameters.python >>-nooptionals" - steps: - - checkout - - run: pip install tox - - run: tox - test_pypy: - parameters: - python: - type: string - docker: - - image: pypy:<< parameters.python >> - environment: - TOXENV: "pypy<< parameters.python >>" - steps: - - checkout - - run: pip install tox - - run: tox - - -workflows: - version: 2 - client_python: - jobs: - - flake8_lint - - isort_lint - - mypy_lint - - test: - matrix: - parameters: - python: - - "3.9.18" - - "3.10" - - "3.11" - - "3.12" - - "3.13" - - "3.14" - - test_nooptionals: - matrix: - parameters: - python: - - "3.9" - - test_pypy: - matrix: - parameters: - python: - - "3.9" diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 00000000..a7e4e094 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,110 @@ +name: CI + +on: + push: + branches: [ master ] + pull_request: + +permissions: + contents: read + +jobs: + flake8_lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - name: Set up Python + uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 + with: + python-version: '3.9' + - name: Install tox + run: pip install tox + - name: Run flake8 + run: tox -e flake8 + + isort_lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - name: Set up Python + uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 + with: + python-version: '3.9' + - name: Install tox + run: pip install tox + - name: Run isort + run: tox -e isort + + mypy_lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - name: Set up Python + uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 + with: + python-version: '3.9' + - name: Install tox + run: pip install tox + - name: Run mypy + run: tox -e mypy + + test: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.9', '3.10', '3.11', '3.12', '3.13', '3.14'] + steps: + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + pip install --user tox "virtualenv<20.22.0" + echo "$HOME/.local/bin" >> $GITHUB_PATH + - name: Set tox environment + id: toxenv + run: | + VERSION="${{ matrix.python-version }}" + # Extract major.minor version (strip patch if present) + TOX_VERSION=$(echo "$VERSION" | cut -d. -f1,2) + echo "toxenv=py${TOX_VERSION}" >> $GITHUB_OUTPUT + - name: Run tests + run: tox + env: + TOXENV: ${{ steps.toxenv.outputs.toxenv }} + + test_nooptionals: + runs-on: ubuntu-latest + env: + PYTHON_VERSION: '3.9' + steps: + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - name: Set up Python + uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 + with: + python-version: ${{ env.PYTHON_VERSION }} + - name: Install tox + run: pip install tox + - name: Run tests without optional dependencies + run: tox + env: + TOXENV: py${{ env.PYTHON_VERSION }}-nooptionals + + test_pypy: + runs-on: ubuntu-latest + env: + PYTHON_VERSION: '3.9' + steps: + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - name: Set up PyPy + uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 + with: + python-version: pypy-${{ env.PYTHON_VERSION }} + - name: Install tox + run: pip install tox + - name: Run tests with PyPy + run: tox + env: + TOXENV: pypy${{ env.PYTHON_VERSION }} diff --git a/.github/workflows/github-pages.yaml b/.github/workflows/github-pages.yaml index 621f2d73..d8db8cbc 100644 --- a/.github/workflows/github-pages.yaml +++ b/.github/workflows/github-pages.yaml @@ -11,9 +11,6 @@ on: # Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages permissions: contents: read - pages: write - id-token: write - actions: read # Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. # However, do NOT cancel in-progress runs as we want to allow these production deployments to complete. @@ -32,6 +29,9 @@ jobs: runs-on: ubuntu-latest env: HUGO_VERSION: 0.145.0 + permissions: + pages: write + id-token: write steps: - name: Install Hugo CLI run: | @@ -40,13 +40,13 @@ jobs: #- name: Install Dart Sass # run: sudo snap install dart-sass - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: submodules: recursive fetch-depth: 0 - name: Setup Pages id: pages - uses: actions/configure-pages@v5 + uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b # v5.0.0 - name: Install Node.js dependencies run: "[[ -f package-lock.json || -f npm-shrinkwrap.json ]] && npm ci || true" working-directory: ./docs @@ -62,7 +62,7 @@ jobs: --baseURL "${{ steps.pages.outputs.base_url }}/" working-directory: ./docs - name: Upload artifact - uses: actions/upload-pages-artifact@v3 + uses: actions/upload-pages-artifact@7b1f4a764d45c48632c6b24a0339c27f5614fb0b # v4.0.0 with: path: ./docs/public @@ -73,7 +73,11 @@ jobs: url: ${{ steps.deployment.outputs.page_url }} runs-on: ubuntu-latest needs: build + permissions: + pages: write + id-token: write + actions: read steps: - name: Deploy to GitHub Pages id: deployment - uses: actions/deploy-pages@v4 + uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4.0.5 diff --git a/docs/content/exporting/http/_index.md b/docs/content/exporting/http/_index.md index dc1b8f2c..f7a6aac6 100644 --- a/docs/content/exporting/http/_index.md +++ b/docs/content/exporting/http/_index.md @@ -24,6 +24,7 @@ to shutdown the server gracefully: ```python server, t = start_http_server(8000) server.shutdown() +server.server_close() t.join() ``` diff --git a/docs/content/exporting/http/django.md b/docs/content/exporting/http/django.md new file mode 100644 index 00000000..a900a3a2 --- /dev/null +++ b/docs/content/exporting/http/django.md @@ -0,0 +1,47 @@ +--- +title: Django +weight: 5 +--- + +To use Prometheus with [Django](https://www.djangoproject.com/) you can use the provided view class +to add a metrics endpoint to your app. + +```python +# urls.py + +from django.urls import path +from prometheus_client.django import PrometheusDjangoView + +urlpatterns = [ + # ... any other urls that you want + path("metrics/", PrometheusDjangoView.as_view(), name="prometheus-metrics"), + # ... still more urls +] +``` + +By default, Multiprocessing support is activated if environment variable `PROMETHEUS_MULTIPROC_DIR` is set. +You can override this through the view arguments: + +```python +from django.conf import settings + +urlpatterns = [ + path( + "metrics/", + PrometheusDjangoView.as_view( + multiprocess_mode=settings.YOUR_SETTING # or any boolean value + ), + name="prometheus-metrics", + ), +] +``` + +Full multiprocessing instructions are provided [here]({{< ref "/multiprocess" >}}). + +# django-prometheus + +The included `PrometheusDjangoView` is useful if you want to define your own metrics from scratch. + +An external package called [django-prometheus](https://github.com/django-commons/django-prometheus/) +can be used instead if you want to get a bunch of ready-made monitoring metrics for your Django application +and easily benefit from utilities such as models monitoring. diff --git a/docs/content/exporting/pushgateway.md b/docs/content/exporting/pushgateway.md index bf5eb112..d9f9a945 100644 --- a/docs/content/exporting/pushgateway.md +++ b/docs/content/exporting/pushgateway.md @@ -54,6 +54,20 @@ g.set_to_current_time() push_to_gateway('localhost:9091', job='batchA', registry=registry, handler=my_auth_handler) ``` +# Compressing data before sending to pushgateway +Pushgateway (version >= 1.5.0) supports gzip and snappy compression (v > 1.6.0). This can help in network constrained environments. +To compress a push request, set the `compression` argument to `'gzip'` or `'snappy'`: +```python +push_to_gateway( + 'localhost:9091', + job='batchA', + registry=registry, + handler=my_auth_handler, + compression='gzip', +) +``` +Snappy compression requires the optional [`python-snappy`](https://github.com/andrix/python-snappy) package. + TLS Auth is also supported when using the push gateway with a special handler. ```python diff --git a/docs/content/multiprocess/_index.md b/docs/content/multiprocess/_index.md index 33507cd9..42ea6a67 100644 --- a/docs/content/multiprocess/_index.md +++ b/docs/content/multiprocess/_index.md @@ -10,9 +10,12 @@ it's common to have processes rather than threads to handle large workloads. To handle this the client library can be put in multiprocess mode. This comes with a number of limitations: -- Registries can not be used as normal, all instantiated metrics are exported +- Registries can not be used as normal: + - all instantiated metrics are collected - Registering metrics to a registry later used by a `MultiProcessCollector` may cause duplicate metrics to be exported + - Filtering on metrics works if and only if the constructor was called with + `support_collectors_without_names=True` and it but might be inefficient. - Custom collectors do not work (e.g. cpu and memory metrics) - Gauges cannot use `set_function` - Info and Enum metrics do not work @@ -49,7 +52,7 @@ MY_COUNTER = Counter('my_counter', 'Description of my counter') # Expose metrics. def app(environ, start_response): - registry = CollectorRegistry() + registry = CollectorRegistry(support_collectors_without_names=True) multiprocess.MultiProcessCollector(registry) data = generate_latest(registry) status = '200 OK' diff --git a/mypy.ini b/mypy.ini index fe372d07..3aa142c1 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,5 +1,5 @@ [mypy] -exclude = prometheus_client/decorator.py|prometheus_client/twisted|tests/test_twisted.py +exclude = prometheus_client/decorator.py|prometheus_client/twisted|tests/test_twisted.py|prometheus_client/django|tests/test_django.py implicit_reexport = False disallow_incomplete_defs = True diff --git a/prometheus_client/aiohttp/exposition.py b/prometheus_client/aiohttp/exposition.py index 914fb26f..c1ae254d 100644 --- a/prometheus_client/aiohttp/exposition.py +++ b/prometheus_client/aiohttp/exposition.py @@ -4,11 +4,11 @@ from aiohttp.typedefs import Handler from ..exposition import _bake_output -from ..registry import CollectorRegistry, REGISTRY +from ..registry import Collector, REGISTRY def make_aiohttp_handler( - registry: CollectorRegistry = REGISTRY, + registry: Collector = REGISTRY, disable_compression: bool = False, ) -> Handler: """Create a aiohttp handler which serves the metrics from a registry.""" diff --git a/prometheus_client/asgi.py b/prometheus_client/asgi.py index affd9844..6e527ca9 100644 --- a/prometheus_client/asgi.py +++ b/prometheus_client/asgi.py @@ -2,10 +2,10 @@ from urllib.parse import parse_qs from .exposition import _bake_output -from .registry import CollectorRegistry, REGISTRY +from .registry import Collector, REGISTRY -def make_asgi_app(registry: CollectorRegistry = REGISTRY, disable_compression: bool = False) -> Callable: +def make_asgi_app(registry: Collector = REGISTRY, disable_compression: bool = False) -> Callable: """Create a ASGI app which serves the metrics from a registry.""" async def prometheus_app(scope, receive, send): diff --git a/prometheus_client/bridge/graphite.py b/prometheus_client/bridge/graphite.py index 8cadbedc..235324b2 100755 --- a/prometheus_client/bridge/graphite.py +++ b/prometheus_client/bridge/graphite.py @@ -8,7 +8,7 @@ from timeit import default_timer from typing import Callable, Tuple -from ..registry import CollectorRegistry, REGISTRY +from ..registry import Collector, REGISTRY # Roughly, have to keep to what works as a file name. # We also remove periods, so labels can be distinguished. @@ -48,7 +48,7 @@ def run(self): class GraphiteBridge: def __init__(self, address: Tuple[str, int], - registry: CollectorRegistry = REGISTRY, + registry: Collector = REGISTRY, timeout_seconds: float = 30, _timer: Callable[[], float] = time.time, tags: bool = False, diff --git a/prometheus_client/django/__init__.py b/prometheus_client/django/__init__.py new file mode 100644 index 00000000..280dbfb0 --- /dev/null +++ b/prometheus_client/django/__init__.py @@ -0,0 +1,5 @@ +from .exposition import PrometheusDjangoView + +__all__ = [ + "PrometheusDjangoView", +] diff --git a/prometheus_client/django/exposition.py b/prometheus_client/django/exposition.py new file mode 100644 index 00000000..71fc8d8a --- /dev/null +++ b/prometheus_client/django/exposition.py @@ -0,0 +1,43 @@ +import os + +from django.http import HttpResponse +from django.views import View + +import prometheus_client +from prometheus_client import multiprocess +from prometheus_client.exposition import _bake_output + + +class PrometheusDjangoView(View): + multiprocess_mode: bool = "PROMETHEUS_MULTIPROC_DIR" in os.environ or "prometheus_multiproc_dir" in os.environ + registry: prometheus_client.CollectorRegistry = None + + def get(self, request, *args, **kwargs): + if self.registry is None: + if self.multiprocess_mode: + self.registry = prometheus_client.CollectorRegistry() + multiprocess.MultiProcessCollector(self.registry) + else: + self.registry = prometheus_client.REGISTRY + accept_header = request.headers.get("Accept") + accept_encoding_header = request.headers.get("Accept-Encoding") + # Bake output + status, headers, output = _bake_output( + registry=self.registry, + accept_header=accept_header, + accept_encoding_header=accept_encoding_header, + params=request.GET, + disable_compression=False, + ) + status = int(status.split(" ")[0]) + return HttpResponse( + output, + status=status, + headers=headers, + ) + + def options(self, request, *args, **kwargs): + return HttpResponse( + status=200, + headers={"Allow": "OPTIONS,GET"}, + ) diff --git a/prometheus_client/exposition.py b/prometheus_client/exposition.py index 0d471707..2d402a0f 100644 --- a/prometheus_client/exposition.py +++ b/prometheus_client/exposition.py @@ -9,7 +9,9 @@ import ssl import sys import threading -from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, Union +from typing import ( + Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple, Union, +) from urllib.error import HTTPError from urllib.parse import parse_qs, quote_plus, urlparse from urllib.request import ( @@ -19,9 +21,16 @@ from wsgiref.simple_server import make_server, WSGIRequestHandler, WSGIServer from .openmetrics import exposition as openmetrics -from .registry import CollectorRegistry, REGISTRY +from .registry import Collector, REGISTRY from .utils import floatToGoString, parse_version +try: + import snappy # type: ignore + SNAPPY_AVAILABLE = True +except ImportError: + snappy = None # type: ignore + SNAPPY_AVAILABLE = False + __all__ = ( 'CONTENT_TYPE_LATEST', 'CONTENT_TYPE_PLAIN_0_0_4', @@ -46,6 +55,7 @@ """Content type of the latest format""" CONTENT_TYPE_LATEST = CONTENT_TYPE_PLAIN_1_0_0 +CompressionType = Optional[Literal['gzip', 'snappy']] class _PrometheusRedirectHandler(HTTPRedirectHandler): @@ -118,7 +128,7 @@ def _bake_output(registry, accept_header, accept_encoding_header, params, disabl return '200 OK', headers, output -def make_wsgi_app(registry: CollectorRegistry = REGISTRY, disable_compression: bool = False) -> Callable: +def make_wsgi_app(registry: Collector = REGISTRY, disable_compression: bool = False) -> Callable: """Create a WSGI app which serves the metrics from a registry.""" def prometheus_app(environ, start_response): @@ -223,7 +233,7 @@ def _get_ssl_ctx( def start_wsgi_server( port: int, addr: str = '0.0.0.0', - registry: CollectorRegistry = REGISTRY, + registry: Collector = REGISTRY, certfile: Optional[str] = None, keyfile: Optional[str] = None, client_cafile: Optional[str] = None, @@ -252,12 +262,12 @@ class TmpServer(ThreadingWSGIServer): start_http_server = start_wsgi_server -def generate_latest(registry: CollectorRegistry = REGISTRY, escaping: str = openmetrics.UNDERSCORES) -> bytes: +def generate_latest(registry: Collector = REGISTRY, escaping: str = openmetrics.UNDERSCORES) -> bytes: """ Generates the exposition format using the basic Prometheus text format. Params: - registry: CollectorRegistry to export data from. + registry: Collector to export data from. escaping: Escaping scheme used for metric and label names. Returns: UTF-8 encoded string containing the metrics in text format. @@ -330,7 +340,7 @@ def sample_line(samples): return ''.join(output).encode('utf-8') -def choose_encoder(accept_header: str) -> Tuple[Callable[[CollectorRegistry], bytes], str]: +def choose_encoder(accept_header: str) -> Tuple[Callable[[Collector], bytes], str]: # Python client library accepts a narrower range of content-types than # Prometheus does. accept_header = accept_header or '' @@ -408,7 +418,7 @@ def gzip_accepted(accept_encoding_header: str) -> bool: class MetricsHandler(BaseHTTPRequestHandler): """HTTP handler that gives metrics from ``REGISTRY``.""" - registry: CollectorRegistry = REGISTRY + registry: Collector = REGISTRY def do_GET(self) -> None: # Prepare parameters @@ -429,7 +439,7 @@ def log_message(self, format: str, *args: Any) -> None: """Log nothing.""" @classmethod - def factory(cls, registry: CollectorRegistry) -> type: + def factory(cls, registry: Collector) -> type: """Returns a dynamic MetricsHandler class tied to the passed registry. """ @@ -444,7 +454,7 @@ def factory(cls, registry: CollectorRegistry) -> type: return MyMetricsHandler -def write_to_textfile(path: str, registry: CollectorRegistry, escaping: str = openmetrics.ALLOWUTF8, tmpdir: Optional[str] = None) -> None: +def write_to_textfile(path: str, registry: Collector, escaping: str = openmetrics.ALLOWUTF8, tmpdir: Optional[str] = None) -> None: """Write metrics to the given path. This is intended for use with the Node exporter textfile collector. @@ -592,10 +602,11 @@ def tls_auth_handler( def push_to_gateway( gateway: str, job: str, - registry: CollectorRegistry, + registry: Collector, grouping_key: Optional[Dict[str, Any]] = None, timeout: Optional[float] = 30, handler: Callable = default_handler, + compression: CompressionType = None, ) -> None: """Push metrics to the given pushgateway. @@ -603,7 +614,7 @@ def push_to_gateway( 'http://pushgateway.local', or 'pushgateway.local'. Scheme defaults to 'http' if none is provided `job` is the job label to be attached to all pushed metrics - `registry` is an instance of CollectorRegistry + `registry` is a Collector, normally an instance of CollectorRegistry `grouping_key` please see the pushgateway documentation for details. Defaults to None `timeout` is how long push will attempt to connect before giving up. @@ -632,19 +643,22 @@ def push_to_gateway( failure. 'content' is the data which should be used to form the HTTP Message Body. + `compression` selects the payload compression. Supported values are 'gzip' + and 'snappy'. Defaults to None (no compression). This overwrites all metrics with the same job and grouping_key. This uses the PUT HTTP method.""" - _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) + _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler, compression) def pushadd_to_gateway( gateway: str, job: str, - registry: Optional[CollectorRegistry], + registry: Optional[Collector], grouping_key: Optional[Dict[str, Any]] = None, timeout: Optional[float] = 30, handler: Callable = default_handler, + compression: CompressionType = None, ) -> None: """PushAdd metrics to the given pushgateway. @@ -652,7 +666,7 @@ def pushadd_to_gateway( 'http://pushgateway.local', or 'pushgateway.local'. Scheme defaults to 'http' if none is provided `job` is the job label to be attached to all pushed metrics - `registry` is an instance of CollectorRegistry + `registry` is a Collector, normally an instance of CollectorRegistry `grouping_key` please see the pushgateway documentation for details. Defaults to None `timeout` is how long push will attempt to connect before giving up. @@ -663,10 +677,12 @@ def pushadd_to_gateway( will be carried out by a default handler. See the 'prometheus_client.push_to_gateway' documentation for implementation requirements. + `compression` selects the payload compression. Supported values are 'gzip' + and 'snappy'. Defaults to None (no compression). This replaces metrics with the same name, job and grouping_key. This uses the POST HTTP method.""" - _use_gateway('POST', gateway, job, registry, grouping_key, timeout, handler) + _use_gateway('POST', gateway, job, registry, grouping_key, timeout, handler, compression) def delete_from_gateway( @@ -702,10 +718,11 @@ def _use_gateway( method: str, gateway: str, job: str, - registry: Optional[CollectorRegistry], + registry: Optional[Collector], grouping_key: Optional[Dict[str, Any]], timeout: Optional[float], handler: Callable, + compression: CompressionType = None, ) -> None: gateway_url = urlparse(gateway) # See https://bugs.python.org/issue27657 for details on urlparse in py>=3.7.6. @@ -715,30 +732,60 @@ def _use_gateway( gateway = gateway.rstrip('/') url = '{}/metrics/{}/{}'.format(gateway, *_escape_grouping_key("job", job)) - data = b'' - if method != 'DELETE': - if registry is None: - registry = REGISTRY - data = generate_latest(registry) - if grouping_key is None: grouping_key = {} url += ''.join( '/{}/{}'.format(*_escape_grouping_key(str(k), str(v))) for k, v in sorted(grouping_key.items())) + data = b'' + headers: List[Tuple[str, str]] = [] + if method != 'DELETE': + if registry is None: + registry = REGISTRY + data = generate_latest(registry) + data, headers = _compress_payload(data, compression) + else: + # DELETE requests still need Content-Type header per test expectations + headers = [('Content-Type', CONTENT_TYPE_PLAIN_0_0_4)] + if compression is not None: + raise ValueError('Compression is not supported for DELETE requests.') + handler( url=url, method=method, timeout=timeout, - headers=[('Content-Type', CONTENT_TYPE_PLAIN_0_0_4)], data=data, + headers=headers, data=data, )() +def _compress_payload(data: bytes, compression: CompressionType) -> Tuple[bytes, List[Tuple[str, str]]]: + headers = [('Content-Type', CONTENT_TYPE_PLAIN_0_0_4)] + if compression is None: + return data, headers + + encoding = compression.lower() + if encoding == 'gzip': + headers.append(('Content-Encoding', 'gzip')) + return gzip.compress(data), headers + if encoding == 'snappy': + if not SNAPPY_AVAILABLE: + raise RuntimeError('Snappy compression requires the python-snappy package to be installed.') + headers.append(('Content-Encoding', 'snappy')) + compressor = snappy.StreamCompressor() + compressed = compressor.compress(data) + flush = getattr(compressor, 'flush', None) + if callable(flush): + compressed += flush() + return compressed, headers + raise ValueError(f"Unsupported compression type: {compression}") + + def _escape_grouping_key(k, v): if v == "": # Per https://github.com/prometheus/pushgateway/pull/346. return k + "@base64", "=" - elif '/' in v: + elif '/' in v or ' ' in v: # Added in Pushgateway 0.9.0. + # Use base64 encoding for values containing slashes or spaces return k + "@base64", base64.urlsafe_b64encode(v.encode("utf-8")).decode("utf-8") else: return k, quote_plus(v) diff --git a/prometheus_client/multiprocess.py b/prometheus_client/multiprocess.py index 2682190a..db55874e 100644 --- a/prometheus_client/multiprocess.py +++ b/prometheus_client/multiprocess.py @@ -88,32 +88,42 @@ def _parse_key(key): @staticmethod def _accumulate_metrics(metrics, accumulate): for metric in metrics.values(): - samples = defaultdict(float) - sample_timestamps = defaultdict(float) + samples = defaultdict(lambda: defaultdict(float)) + sample_timestamps = defaultdict(lambda: defaultdict(float)) buckets = defaultdict(lambda: defaultdict(float)) - samples_setdefault = samples.setdefault for s in metric.samples: name, labels, value, timestamp, exemplar, native_histogram_value = s + + if ( + metric.type == 'gauge' + and metric._multiprocess_mode in ( + 'min', 'livemin', + 'max', 'livemax', + 'sum', 'livesum', + 'mostrecent', 'livemostrecent', + ) + ): + labels = tuple(l for l in labels if l[0] != 'pid') + if metric.type == 'gauge': - without_pid_key = (name, tuple(l for l in labels if l[0] != 'pid')) if metric._multiprocess_mode in ('min', 'livemin'): - current = samples_setdefault(without_pid_key, value) + current = samples[labels].setdefault((name, labels), value) if value < current: - samples[without_pid_key] = value + samples[labels][(name, labels)] = value elif metric._multiprocess_mode in ('max', 'livemax'): - current = samples_setdefault(without_pid_key, value) + current = samples[labels].setdefault((name, labels), value) if value > current: - samples[without_pid_key] = value + samples[labels][(name, labels)] = value elif metric._multiprocess_mode in ('sum', 'livesum'): - samples[without_pid_key] += value + samples[labels][(name, labels)] += value elif metric._multiprocess_mode in ('mostrecent', 'livemostrecent'): - current_timestamp = sample_timestamps[without_pid_key] + current_timestamp = sample_timestamps[labels][name] timestamp = float(timestamp or 0) if current_timestamp < timestamp: - samples[without_pid_key] = value - sample_timestamps[without_pid_key] = timestamp + samples[labels][(name, labels)] = value + sample_timestamps[labels][name] = timestamp else: # all/liveall - samples[(name, labels)] = value + samples[labels][(name, labels)] = value elif metric.type == 'histogram': # A for loop with early exit is faster than a genexpr @@ -127,10 +137,10 @@ def _accumulate_metrics(metrics, accumulate): break else: # did not find the `le` key # _sum/_count - samples[(name, labels)] += value + samples[labels][(name, labels)] += value else: # Counter and Summary. - samples[(name, labels)] += value + samples[labels][(name, labels)] += value # Accumulate bucket values. if metric.type == 'histogram': @@ -143,14 +153,17 @@ def _accumulate_metrics(metrics, accumulate): ) if accumulate: acc += value - samples[sample_key] = acc + samples[labels][sample_key] = acc else: - samples[sample_key] = value + samples[labels][sample_key] = value if accumulate: - samples[(metric.name + '_count', labels)] = acc + samples[labels][(metric.name + '_count', labels)] = acc # Convert to correct sample format. - metric.samples = [Sample(name_, dict(labels), value) for (name_, labels), value in samples.items()] + metric.samples = [] + for _, samples_by_labels in samples.items(): + for (name_, labels), value in samples_by_labels.items(): + metric.samples.append(Sample(name_, dict(labels), value)) return metrics.values() def collect(self): diff --git a/prometheus_client/registry.py b/prometheus_client/registry.py index 8de4ce91..c2b55d15 100644 --- a/prometheus_client/registry.py +++ b/prometheus_client/registry.py @@ -1,24 +1,21 @@ -from abc import ABC, abstractmethod import copy from threading import Lock -from typing import Dict, Iterable, List, Optional +from typing import Dict, Iterable, List, Optional, Protocol from .metrics_core import Metric -# Ideally this would be a Protocol, but Protocols are only available in Python >= 3.8. -class Collector(ABC): - @abstractmethod +class Collector(Protocol): def collect(self) -> Iterable[Metric]: - pass + """Collect metrics.""" -class _EmptyCollector(Collector): +class _EmptyCollector: def collect(self) -> Iterable[Metric]: return [] -class CollectorRegistry(Collector): +class CollectorRegistry: """Metric collector registry. Collectors must have a no-argument method 'collect' that returns a list of @@ -26,12 +23,15 @@ class CollectorRegistry(Collector): exposition formats. """ - def __init__(self, auto_describe: bool = False, target_info: Optional[Dict[str, str]] = None): + def __init__(self, auto_describe: bool = False, target_info: Optional[Dict[str, str]] = None, + support_collectors_without_names: bool = False): self._collector_to_names: Dict[Collector, List[str]] = {} self._names_to_collectors: Dict[str, Collector] = {} self._auto_describe = auto_describe self._lock = Lock() self._target_info: Optional[Dict[str, str]] = {} + self._support_collectors_without_names = support_collectors_without_names + self._collectors_without_names: List[Collector] = [] self.set_target_info(target_info) def register(self, collector: Collector) -> None: @@ -46,6 +46,8 @@ def register(self, collector: Collector) -> None: for name in names: self._names_to_collectors[name] = collector self._collector_to_names[collector] = names + if self._support_collectors_without_names and not names: + self._collectors_without_names.append(collector) def unregister(self, collector: Collector) -> None: """Remove a collector from the registry.""" @@ -148,7 +150,7 @@ def __init__(self, names: Iterable[str], registry: CollectorRegistry): self._registry = registry def collect(self) -> Iterable[Metric]: - collectors = set() + collectors = set(self._registry._collectors_without_names) target_info_metric = None with self._registry._lock: if 'target_info' in self._name_set and self._registry._target_info: diff --git a/pyproject.toml b/pyproject.toml index 1d8527a0..ed3ef389 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "prometheus_client" -version = "0.23.1" +version = "0.24.1" description = "Python client for the Prometheus monitoring system." readme = "README.md" license = "Apache-2.0 AND BSD-2-Clause" @@ -47,6 +47,9 @@ twisted = [ aiohttp = [ "aiohttp", ] +django = [ + "django", +] [project.urls] Homepage = "https://github.com/prometheus/client_python" diff --git a/tests/test_asgi.py b/tests/test_asgi.py index d4933cec..6e795e21 100644 --- a/tests/test_asgi.py +++ b/tests/test_asgi.py @@ -223,3 +223,32 @@ def test_qs_parsing(self): asyncio.new_event_loop().run_until_complete( self.communicator.wait() ) + + def test_qs_parsing_multi(self): + """Only metrics that match the 'name[]' query string param appear""" + + app = make_asgi_app(self.registry) + metrics = [ + ("asdf", "first test metric", 1), + ("bsdf", "second test metric", 2), + ("csdf", "third test metric", 3) + ] + + for m in metrics: + self.increment_metrics(*m) + + self.seed_app(app) + self.scope['query_string'] = "&".join([f"name[]={m[0]}_total" for m in metrics[0:2]]).encode("utf-8") + self.send_default_request() + + outputs = self.get_all_output() + response_body = outputs[1] + output = response_body['body'].decode('utf8') + + self.assert_metrics(output, *metrics[0]) + self.assert_metrics(output, *metrics[1]) + self.assert_not_metrics(output, *metrics[2]) + + asyncio.new_event_loop().run_until_complete( + self.communicator.wait() + ) diff --git a/tests/test_core.py b/tests/test_core.py index c7c9c14f..66492c6f 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -1024,6 +1024,24 @@ def test_restricted_registry_does_not_call_extra(self): self.assertEqual([m], list(registry.restricted_registry(['s_sum']).collect())) mock_collector.collect.assert_not_called() + def test_restricted_registry_ignore_no_names_collectors(self): + from unittest.mock import MagicMock + registry = CollectorRegistry() + mock_collector = MagicMock() + mock_collector.describe.return_value = [] + registry.register(mock_collector) + self.assertEqual(list(registry.restricted_registry(['metric']).collect()), []) + mock_collector.collect.assert_not_called() + + def test_restricted_registry_collects_no_names_collectors(self): + from unittest.mock import MagicMock + registry = CollectorRegistry(support_collectors_without_names=True) + mock_collector = MagicMock() + mock_collector.describe.return_value = [] + registry.register(mock_collector) + self.assertEqual(list(registry.restricted_registry(['metric']).collect()), []) + mock_collector.collect.assert_called() + def test_restricted_registry_does_not_yield_while_locked(self): registry = CollectorRegistry(target_info={'foo': 'bar'}) Summary('s', 'help', registry=registry).observe(7) diff --git a/tests/test_django.py b/tests/test_django.py new file mode 100644 index 00000000..659bb3f6 --- /dev/null +++ b/tests/test_django.py @@ -0,0 +1,48 @@ +from unittest import skipUnless + +from prometheus_client import CollectorRegistry, Counter, generate_latest +from prometheus_client.openmetrics.exposition import ALLOWUTF8 + +try: + import django + from django.test import RequestFactory, TestCase + + from prometheus_client.django import PrometheusDjangoView + + HAVE_DJANGO = True +except ImportError: + from unittest import TestCase + + HAVE_DJANGO = False + +else: + from django.conf import settings + + if not settings.configured: + settings.configure( + DATABASES={ + "default": { + "ENGINE": "django.db.backends.sqlite3", + 'NAME': ':memory:' + } + }, + INSTALLED_APPS=[], + ) + django.setup() + + +class MetricsResourceTest(TestCase): + @skipUnless(HAVE_DJANGO, "Don't have django installed.") + def setUp(self): + self.registry = CollectorRegistry() + self.factory = RequestFactory() + + def test_reports_metrics(self): + c = Counter('cc', 'A counter', registry=self.registry) + c.inc() + + request = self.factory.get("/metrics") + + response = PrometheusDjangoView.as_view(registry=self.registry)(request) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.content, generate_latest(self.registry, ALLOWUTF8)) diff --git a/tests/test_exposition.py b/tests/test_exposition.py index 3dd5e378..a3c97820 100644 --- a/tests/test_exposition.py +++ b/tests/test_exposition.py @@ -1,3 +1,4 @@ +import gzip from http.server import BaseHTTPRequestHandler, HTTPServer import os import threading @@ -300,6 +301,13 @@ def test_push_with_groupingkey_empty_label(self): self.assertEqual(self.requests[0][0].headers.get('content-type'), CONTENT_TYPE_PLAIN_0_0_4) self.assertEqual(self.requests[0][1], b'# HELP g help\n# TYPE g gauge\ng 0.0\n') + def test_push_with_groupingkey_with_spaces(self): + push_to_gateway(self.address, "my_job", self.registry, {'label': 'value with spaces'}) + self.assertEqual(self.requests[0][0].command, 'PUT') + self.assertEqual(self.requests[0][0].path, '/metrics/job/my_job/label@base64/dmFsdWUgd2l0aCBzcGFjZXM=') + self.assertEqual(self.requests[0][0].headers.get('content-type'), CONTENT_TYPE_PLAIN_0_0_4) + self.assertEqual(self.requests[0][1], b'# HELP g help\n# TYPE g gauge\ng 0.0\n') + def test_push_with_complex_groupingkey(self): push_to_gateway(self.address, "my_job", self.registry, {'a': 9, 'b': 'a/ z'}) self.assertEqual(self.requests[0][0].command, 'PUT') @@ -404,6 +412,30 @@ def test_push_with_trailing_slash(self): self.assertNotIn('//', self.requests[0][0].path) + def test_push_with_gzip_compression(self): + push_to_gateway(self.address, "my_job", self.registry, compression='gzip') + request, body = self.requests[0] + self.assertEqual(request.headers.get('content-encoding'), 'gzip') + decompressed = gzip.decompress(body) + self.assertEqual(decompressed, b'# HELP g help\n# TYPE g gauge\ng 0.0\n') + + def test_push_with_snappy_compression(self): + snappy = pytest.importorskip('snappy') + push_to_gateway(self.address, "my_job", self.registry, compression='snappy') + request, body = self.requests[0] + self.assertEqual(request.headers.get('content-encoding'), 'snappy') + decompressor = snappy.StreamDecompressor() + decompressed = decompressor.decompress(body) + flush = getattr(decompressor, 'flush', None) + if callable(flush): + decompressed += flush() + self.assertEqual(decompressed, b'# HELP g help\n# TYPE g gauge\ng 0.0\n') + + def test_push_with_invalid_compression(self): + with self.assertRaisesRegex(ValueError, 'Unsupported compression type'): + push_to_gateway(self.address, "my_job", self.registry, compression='brotli') + self.assertEqual(self.requests, []) + def test_instance_ip_grouping_key(self): self.assertTrue('' != instance_ip_grouping_key()['instance']) diff --git a/tests/test_multiprocess.py b/tests/test_multiprocess.py index e7ca154e..ee0c7423 100644 --- a/tests/test_multiprocess.py +++ b/tests/test_multiprocess.py @@ -52,7 +52,7 @@ def setUp(self): self.tempdir = tempfile.mkdtemp() os.environ['PROMETHEUS_MULTIPROC_DIR'] = self.tempdir values.ValueClass = MultiProcessValue(lambda: 123) - self.registry = CollectorRegistry() + self.registry = CollectorRegistry(support_collectors_without_names=True) self.collector = MultiProcessCollector(self.registry) @property @@ -276,10 +276,8 @@ def add_label(key, value): Sample('g', add_label('pid', '1'), 1.0), ]) - metrics['h'].samples.sort( - key=lambda x: (x[0], float(x[1].get('le', 0))) - ) expected_histogram = [ + Sample('h_sum', labels, 6.0), Sample('h_bucket', add_label('le', '0.005'), 0.0), Sample('h_bucket', add_label('le', '0.01'), 0.0), Sample('h_bucket', add_label('le', '0.025'), 0.0), @@ -296,11 +294,99 @@ def add_label(key, value): Sample('h_bucket', add_label('le', '10.0'), 2.0), Sample('h_bucket', add_label('le', '+Inf'), 2.0), Sample('h_count', labels, 2.0), - Sample('h_sum', labels, 6.0), ] self.assertEqual(metrics['h'].samples, expected_histogram) + def test_collect_histogram_ordering(self): + pid = 0 + values.ValueClass = MultiProcessValue(lambda: pid) + labels = {i: i for i in 'abcd'} + + def add_label(key, value): + l = labels.copy() + l[key] = value + return l + + h = Histogram('h', 'help', labelnames=['view'], registry=None) + + h.labels(view='view1').observe(1) + + pid = 1 + + h.labels(view='view1').observe(5) + h.labels(view='view2').observe(1) + + metrics = {m.name: m for m in self.collector.collect()} + + expected_histogram = [ + Sample('h_sum', {'view': 'view1'}, 6.0), + Sample('h_bucket', {'view': 'view1', 'le': '0.005'}, 0.0), + Sample('h_bucket', {'view': 'view1', 'le': '0.01'}, 0.0), + Sample('h_bucket', {'view': 'view1', 'le': '0.025'}, 0.0), + Sample('h_bucket', {'view': 'view1', 'le': '0.05'}, 0.0), + Sample('h_bucket', {'view': 'view1', 'le': '0.075'}, 0.0), + Sample('h_bucket', {'view': 'view1', 'le': '0.1'}, 0.0), + Sample('h_bucket', {'view': 'view1', 'le': '0.25'}, 0.0), + Sample('h_bucket', {'view': 'view1', 'le': '0.5'}, 0.0), + Sample('h_bucket', {'view': 'view1', 'le': '0.75'}, 0.0), + Sample('h_bucket', {'view': 'view1', 'le': '1.0'}, 1.0), + Sample('h_bucket', {'view': 'view1', 'le': '2.5'}, 1.0), + Sample('h_bucket', {'view': 'view1', 'le': '5.0'}, 2.0), + Sample('h_bucket', {'view': 'view1', 'le': '7.5'}, 2.0), + Sample('h_bucket', {'view': 'view1', 'le': '10.0'}, 2.0), + Sample('h_bucket', {'view': 'view1', 'le': '+Inf'}, 2.0), + Sample('h_count', {'view': 'view1'}, 2.0), + Sample('h_sum', {'view': 'view2'}, 1.0), + Sample('h_bucket', {'view': 'view2', 'le': '0.005'}, 0.0), + Sample('h_bucket', {'view': 'view2', 'le': '0.01'}, 0.0), + Sample('h_bucket', {'view': 'view2', 'le': '0.025'}, 0.0), + Sample('h_bucket', {'view': 'view2', 'le': '0.05'}, 0.0), + Sample('h_bucket', {'view': 'view2', 'le': '0.075'}, 0.0), + Sample('h_bucket', {'view': 'view2', 'le': '0.1'}, 0.0), + Sample('h_bucket', {'view': 'view2', 'le': '0.25'}, 0.0), + Sample('h_bucket', {'view': 'view2', 'le': '0.5'}, 0.0), + Sample('h_bucket', {'view': 'view2', 'le': '0.75'}, 0.0), + Sample('h_bucket', {'view': 'view2', 'le': '1.0'}, 1.0), + Sample('h_bucket', {'view': 'view2', 'le': '2.5'}, 1.0), + Sample('h_bucket', {'view': 'view2', 'le': '5.0'}, 1.0), + Sample('h_bucket', {'view': 'view2', 'le': '7.5'}, 1.0), + Sample('h_bucket', {'view': 'view2', 'le': '10.0'}, 1.0), + Sample('h_bucket', {'view': 'view2', 'le': '+Inf'}, 1.0), + Sample('h_count', {'view': 'view2'}, 1.0), + ] + + self.assertEqual(metrics['h'].samples, expected_histogram) + + def test_restrict(self): + pid = 0 + values.ValueClass = MultiProcessValue(lambda: pid) + labels = {i: i for i in 'abcd'} + + def add_label(key, value): + l = labels.copy() + l[key] = value + return l + + c = Counter('c', 'help', labelnames=labels.keys(), registry=None) + g = Gauge('g', 'help', labelnames=labels.keys(), registry=None) + + c.labels(**labels).inc(1) + g.labels(**labels).set(1) + + pid = 1 + + c.labels(**labels).inc(1) + g.labels(**labels).set(1) + + metrics = {m.name: m for m in self.registry.restricted_registry(['c_total']).collect()} + + self.assertEqual(metrics.keys(), {'c'}) + + self.assertEqual( + metrics['c'].samples, [Sample('c_total', labels, 2.0)] + ) + def test_collect_preserves_help(self): pid = 0 values.ValueClass = MultiProcessValue(lambda: pid) @@ -347,10 +433,8 @@ def add_label(key, value): m.name: m for m in self.collector.merge(files, accumulate=False) } - metrics['h'].samples.sort( - key=lambda x: (x[0], float(x[1].get('le', 0))) - ) expected_histogram = [ + Sample('h_sum', labels, 6.0), Sample('h_bucket', add_label('le', '0.005'), 0.0), Sample('h_bucket', add_label('le', '0.01'), 0.0), Sample('h_bucket', add_label('le', '0.025'), 0.0), @@ -366,7 +450,6 @@ def add_label(key, value): Sample('h_bucket', add_label('le', '7.5'), 0.0), Sample('h_bucket', add_label('le', '10.0'), 0.0), Sample('h_bucket', add_label('le', '+Inf'), 0.0), - Sample('h_sum', labels, 6.0), ] self.assertEqual(metrics['h'].samples, expected_histogram) diff --git a/tox.ini b/tox.ini index ccf95cc2..992bd0a7 100644 --- a/tox.ini +++ b/tox.ini @@ -10,6 +10,8 @@ deps = attrs {py3.9,pypy3.9}: twisted {py3.9,pypy3.9}: aiohttp + {py3.9,pypy3.9}: django + {py3.9}: python-snappy commands = coverage run --parallel -m pytest {posargs} [testenv:py3.9-nooptionals]