From ddb23a1e324deede95c2c58cd128e170d1276166 Mon Sep 17 00:00:00 2001 From: Maykel Moya Date: Tue, 17 Mar 2015 19:11:55 +0100 Subject: [PATCH 1/6] Add support for exporting metrics to a pushgateway --- README.md | 16 ++++++++++++++++ prometheus_client/__init__.py | 30 ++++++++++++++++++++++++++++++ tests/test_client.py | 30 ++++++++++++++++++++++++++++++ 3 files changed, 76 insertions(+) diff --git a/README.md b/README.md index fedfd1e0..20b220f8 100644 --- a/README.md +++ b/README.md @@ -181,3 +181,19 @@ write_to_textfile('/configured/textfile/path/raid.prom', registry) ``` A separate registry is used, as the default registry may contain other metrics. + +## Exporting to a Pushgateway + +The [Pushgateway](https://github.com/prometheus/pushgateway) +allows ephemeral and batch jobs to expose their metrics to Prometheus. + +```python +from prometheus_client import CollectorRegistry,Gauge,build_pushgateway_url,push_to_gateway +registry = CollectorRegistry() +g = Gauge('raid_status', '1 if raid array is okay', registry=registry) +g.set(1) +url = build_pushgateway_url(job='cooljob', host='pushgateway.mydomain') +push_to_gateway(url, registry) +``` + +A separate registry is used, as the default registry may contain other metrics. diff --git a/prometheus_client/__init__.py b/prometheus_client/__init__.py index 04075779..0d3ea04b 100644 --- a/prometheus_client/__init__.py +++ b/prometheus_client/__init__.py @@ -5,8 +5,10 @@ import copy import re import os +import socket import time import threading +import urllib2 try: from BaseHTTPServer import BaseHTTPRequestHandler except ImportError: @@ -15,6 +17,7 @@ from functools import wraps from threading import Lock + __all__ = ['Counter', 'Gauge', 'Summary', 'Histogram'] _METRIC_NAME_RE = re.compile(r'^[a-zA-Z_:][a-zA-Z0-9_:]*$') @@ -434,6 +437,33 @@ def do_GET(self): self.wfile.write(generate_latest(REGISTRY)) +def build_pushgateway_url(job, instance=None, host='localhost', port=9091, + use_fqdn_as_instance=True): + ''' + Build a valid pushgateway url + ''' + + if instance is None and use_fqdn_as_instance: + instance = socket.getfqdn() + + if instance: + instancestr = '/instances/{}'.format(instance) + else: + instancestr = '' + + url = 'http://{}:{}/metrics/jobs/{}{}'.format(host, port, job, instancestr) + return url + + +def push_to_gateway(url, registry, timeout=None): + '''Push metrics to the given url''' + + resp = urllib2.urlopen(url, data=generate_latest(registry), timeout=timeout) + if resp.code >= 400: + raise IOError("error pushing to pushgateway: {0} {1}".format( + resp.code, resp.msg)) + + def write_to_textfile(path, registry): '''Write metrics to the given path. diff --git a/tests/test_client.py b/tests/test_client.py index a28f3f4a..b14fcc5a 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -1,8 +1,10 @@ from __future__ import unicode_literals +import socket import unittest from prometheus_client import Gauge, Counter, Summary, Histogram from prometheus_client import CollectorRegistry, generate_latest +from prometheus_client import build_pushgateway_url class TestCounter(unittest.TestCase): @@ -266,5 +268,33 @@ def test_escaping(self): self.assertEqual(b'# HELP cc A\\ncount\\\\er\n# TYPE cc counter\ncc{a="\\\\x\\n\\""} 1.0\n', generate_latest(self.registry)) +class TestBuildPushgatewayUrl(unittest.TestCase): + def test_job_instance(self): + expected = 'http://localhost:9091/metrics/jobs/foojob/instances/fooinstance' + + url = build_pushgateway_url('foojob', 'fooinstance') + self.assertEqual(url, expected) + + def test_host_port(self): + expected = 'http://foohost:9092/metrics/jobs/foojob' + + url = build_pushgateway_url('foojob', host='foohost', port=9092, + use_fqdn_as_instance=False) + self.assertEqual(url, expected) + + def test_no_fqdn(self): + expected = 'http://localhost:9091/metrics/jobs/foojob' + + url = build_pushgateway_url(job='foojob', use_fqdn_as_instance=False) + self.assertEqual(url, expected) + + def test_fqdn(self): + fqdn = socket.getfqdn() + expected = 'http://localhost:9091/metrics/jobs/foojob/instances/' + fqdn + + url = build_pushgateway_url(job='foojob') + self.assertEqual(url, expected) + + if __name__ == '__main__': unittest.main() From e1fb108f7d459ab1d1a64a8345a1bb950ba9c0de Mon Sep 17 00:00:00 2001 From: Maykel Moya Date: Mon, 23 Mar 2015 19:46:28 +0100 Subject: [PATCH 2/6] Remove logic for using FQDN as instance name --- prometheus_client/__init__.py | 7 +------ tests/test_client.py | 17 +---------------- 2 files changed, 2 insertions(+), 22 deletions(-) diff --git a/prometheus_client/__init__.py b/prometheus_client/__init__.py index 0d3ea04b..3181c784 100644 --- a/prometheus_client/__init__.py +++ b/prometheus_client/__init__.py @@ -5,7 +5,6 @@ import copy import re import os -import socket import time import threading import urllib2 @@ -437,15 +436,11 @@ def do_GET(self): self.wfile.write(generate_latest(REGISTRY)) -def build_pushgateway_url(job, instance=None, host='localhost', port=9091, - use_fqdn_as_instance=True): +def build_pushgateway_url(job, instance=None, host='localhost', port=9091): ''' Build a valid pushgateway url ''' - if instance is None and use_fqdn_as_instance: - instance = socket.getfqdn() - if instance: instancestr = '/instances/{}'.format(instance) else: diff --git a/tests/test_client.py b/tests/test_client.py index b14fcc5a..4478c8e8 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -1,5 +1,4 @@ from __future__ import unicode_literals -import socket import unittest from prometheus_client import Gauge, Counter, Summary, Histogram @@ -278,21 +277,7 @@ def test_job_instance(self): def test_host_port(self): expected = 'http://foohost:9092/metrics/jobs/foojob' - url = build_pushgateway_url('foojob', host='foohost', port=9092, - use_fqdn_as_instance=False) - self.assertEqual(url, expected) - - def test_no_fqdn(self): - expected = 'http://localhost:9091/metrics/jobs/foojob' - - url = build_pushgateway_url(job='foojob', use_fqdn_as_instance=False) - self.assertEqual(url, expected) - - def test_fqdn(self): - fqdn = socket.getfqdn() - expected = 'http://localhost:9091/metrics/jobs/foojob/instances/' + fqdn - - url = build_pushgateway_url(job='foojob') + url = build_pushgateway_url('foojob', host='foohost', port=9092) self.assertEqual(url, expected) From 7bfdf8b655c0a27e81da9350fe02cbfab1127363 Mon Sep 17 00:00:00 2001 From: Maykel Moya Date: Mon, 23 Mar 2015 19:56:18 +0100 Subject: [PATCH 3/6] Encapsulate url construction --- README.md | 5 ++--- prometheus_client/__init__.py | 9 ++++++++- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 20b220f8..644aea89 100644 --- a/README.md +++ b/README.md @@ -188,12 +188,11 @@ The [Pushgateway](https://github.com/prometheus/pushgateway) allows ephemeral and batch jobs to expose their metrics to Prometheus. ```python -from prometheus_client import CollectorRegistry,Gauge,build_pushgateway_url,push_to_gateway +from prometheus_client import CollectorRegistry,Gauge,push_to_gateway registry = CollectorRegistry() g = Gauge('raid_status', '1 if raid array is okay', registry=registry) g.set(1) -url = build_pushgateway_url(job='cooljob', host='pushgateway.mydomain') -push_to_gateway(url, registry) +push_to_gateway(registry, job='somejob', host='pushgateway.mydomain') ``` A separate registry is used, as the default registry may contain other metrics. diff --git a/prometheus_client/__init__.py b/prometheus_client/__init__.py index 3181c784..bdfc234f 100644 --- a/prometheus_client/__init__.py +++ b/prometheus_client/__init__.py @@ -450,7 +450,7 @@ def build_pushgateway_url(job, instance=None, host='localhost', port=9091): return url -def push_to_gateway(url, registry, timeout=None): +def push_to_gateway_url(url, registry, timeout=None): '''Push metrics to the given url''' resp = urllib2.urlopen(url, data=generate_latest(registry), timeout=timeout) @@ -459,6 +459,13 @@ def push_to_gateway(url, registry, timeout=None): resp.code, resp.msg)) +def push_to_gateway(registry, job, instance=None, host='localhost', port=9091, timeout=None): + '''Push metrics to a pushgateway''' + + url = build_pushgateway_url(job, instance, host, port) + push_to_gateway_url(url, registry, timeout) + + def write_to_textfile(path, registry): '''Write metrics to the given path. From b5542272b826d49a64c586bd288cd7f1ec330a7b Mon Sep 17 00:00:00 2001 From: Maykel Moya Date: Mon, 23 Mar 2015 20:09:42 +0100 Subject: [PATCH 4/6] Use set_to_current_time as better example --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 644aea89..434baa20 100644 --- a/README.md +++ b/README.md @@ -190,9 +190,9 @@ allows ephemeral and batch jobs to expose their metrics to Prometheus. ```python from prometheus_client import CollectorRegistry,Gauge,push_to_gateway registry = CollectorRegistry() -g = Gauge('raid_status', '1 if raid array is okay', registry=registry) -g.set(1) -push_to_gateway(registry, job='somejob', host='pushgateway.mydomain') +g = Gauge('job_finished_ok_at', 'last time a batch job successfully finished', registry=registry) +g.set_to_current_time() +push_to_gateway(registry, job='batchA', host='pushgateway.mydomain') ``` A separate registry is used, as the default registry may contain other metrics. From 4065c3b5291a97d4c6fc2f3080dd5e2e80bd1c76 Mon Sep 17 00:00:00 2001 From: Maykel Moya Date: Mon, 23 Mar 2015 21:32:43 +0100 Subject: [PATCH 5/6] Url escape job and instance values --- prometheus_client/__init__.py | 4 +++- tests/test_client.py | 7 +++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/prometheus_client/__init__.py b/prometheus_client/__init__.py index bdfc234f..c18e83b0 100644 --- a/prometheus_client/__init__.py +++ b/prometheus_client/__init__.py @@ -446,7 +446,9 @@ def build_pushgateway_url(job, instance=None, host='localhost', port=9091): else: instancestr = '' - url = 'http://{}:{}/metrics/jobs/{}{}'.format(host, port, job, instancestr) + url = 'http://{}:{}/metrics/jobs/{}{}'.format(host, port, + urllib2.quote(job), + urllib2.quote(instancestr)) return url diff --git a/tests/test_client.py b/tests/test_client.py index 4478c8e8..049729b8 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -280,6 +280,13 @@ def test_host_port(self): url = build_pushgateway_url('foojob', host='foohost', port=9092) self.assertEqual(url, expected) + def test_url_escaping(self): + expected = 'http://localhost:9091/metrics/jobs/foo%20job' + + url = build_pushgateway_url('foo job') + self.assertEqual(url, expected) + + if __name__ == '__main__': unittest.main() From ec4564fbbd1393b19af930cd9cbf90b77d390e07 Mon Sep 17 00:00:00 2001 From: Maykel Moya Date: Mon, 23 Mar 2015 21:33:37 +0100 Subject: [PATCH 6/6] py3 compat --- prometheus_client/__init__.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/prometheus_client/__init__.py b/prometheus_client/__init__.py index c18e83b0..65bb7324 100644 --- a/prometheus_client/__init__.py +++ b/prometheus_client/__init__.py @@ -7,7 +7,14 @@ import os import time import threading -import urllib2 + +try: + from urllib2 import urlopen, quote +except ImportError: + # Python 3 + from urllib.request import urlopen + from urllib.parse import quote + try: from BaseHTTPServer import BaseHTTPRequestHandler except ImportError: @@ -447,15 +454,15 @@ def build_pushgateway_url(job, instance=None, host='localhost', port=9091): instancestr = '' url = 'http://{}:{}/metrics/jobs/{}{}'.format(host, port, - urllib2.quote(job), - urllib2.quote(instancestr)) + quote(job), + quote(instancestr)) return url def push_to_gateway_url(url, registry, timeout=None): '''Push metrics to the given url''' - resp = urllib2.urlopen(url, data=generate_latest(registry), timeout=timeout) + resp = urlopen(url, data=generate_latest(registry), timeout=timeout) if resp.code >= 400: raise IOError("error pushing to pushgateway: {0} {1}".format( resp.code, resp.msg))