From 081bdc0fe835ebe6648aaf881faa5c96c598a7cd Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Besselat Date: Mon, 23 Sep 2019 09:36:47 +0200 Subject: [PATCH 1/4] [snapshot] --- consul/aio.py | 3 ++- consul/base.py | 32 +++++++++++++++++++++++++++++++- consul/std.py | 2 +- consul/tornado.py | 2 +- consul/twisted.py | 7 ++++--- 5 files changed, 39 insertions(+), 7 deletions(-) diff --git a/consul/aio.py b/consul/aio.py index 5e347f53..7274e7cf 100644 --- a/consul/aio.py +++ b/consul/aio.py @@ -25,9 +25,10 @@ def __init__(self, *args, loop=None, **kwargs): def _request(self, callback, method, uri, data=None): resp = yield from self._session.request(method, uri, data=data) body = yield from resp.text(encoding='utf-8') + content = yield from resp.read() if resp.status == 599: raise base.Timeout - r = base.Response(resp.status, resp.headers, body) + r = base.Response(resp.status, resp.headers, body, content) return callback(r) # python prior 3.4.1 does not play nice with __del__ method diff --git a/consul/base.py b/consul/base.py index ee6ab254..8b9e0064 100755 --- a/consul/base.py +++ b/consul/base.py @@ -162,7 +162,7 @@ def _compat( return ret -Response = collections.namedtuple('Response', ['code', 'headers', 'body']) +Response = collections.namedtuple('Response', ['code', 'headers', 'body', 'content']) # @@ -244,6 +244,16 @@ def cb(response): return data return cb + @classmethod + def binary(klass): + """ + This method simply returns response body, usefull for snapshot + """ + def cb(response): + CB._status(response) + return response.content + return cb + class HTTPClient(six.with_metaclass(abc.ABCMeta, object)): def __init__(self, host='127.0.0.1', port=8500, scheme='http', @@ -343,6 +353,7 @@ def __init__( self.query = Consul.Query(self) self.coordinate = Consul.Coordinate(self) self.operator = Consul.Operator(self) + self.snapshot = Consul.Snapshot(self) class Event(object): """ @@ -2434,3 +2445,22 @@ def raft_config(self): """ return self.agent.http.get( CB.json(), '/v1/operator/raft/configuration') + + class Snapshot(object): + def __init__(self,agent): + self.agent = agent + + def get(self): + """ + Returns gzipped snapshot of current consul cluster + """ + return self.agent.http.get( + CB.binary(),'/v1/snapshot') + + def save(self, file_path): + """ + Backup snapshot in a file + """ + backup_file = open(file_path, 'w+b') + backup_file.write(self.get()) + backup_file.close() diff --git a/consul/std.py b/consul/std.py index 96a5b9dc..d2dddbb4 100644 --- a/consul/std.py +++ b/consul/std.py @@ -14,7 +14,7 @@ def __init__(self, *args, **kwargs): def response(self, response): response.encoding = 'utf-8' return base.Response( - response.status_code, response.headers, response.text) + response.status_code, response.headers, response.text, response.content) def get(self, callback, path, params=None): uri = self.uri(path, params) diff --git a/consul/tornado.py b/consul/tornado.py index 50507748..48493de7 100644 --- a/consul/tornado.py +++ b/consul/tornado.py @@ -16,7 +16,7 @@ def __init__(self, *args, **kwargs): def response(self, response): return base.Response( - response.code, response.headers, response.body.decode('utf-8')) + response.code, response.headers, response.body.decode('utf-8'), response.body) @gen.coroutine def _request(self, callback, request): diff --git a/consul/twisted.py b/consul/twisted.py index 206105cb..3a7a20e4 100644 --- a/consul/twisted.py +++ b/consul/twisted.py @@ -47,8 +47,8 @@ def __init__(self, contextFactory, *args, **kwargs): self.client = TreqHTTPClient(Agent(**agent_kwargs)) @staticmethod - def response(code, headers, text): - return base.Response(code, headers, text) + def response(code, headers, text, content): + return base.Response(code, headers, text, content) @staticmethod def compat_string(value): @@ -70,7 +70,8 @@ def _get_resp(self, response): for k, v in dict(response.headers.getAllRawHeaders()).items() ]) body = yield response.text(encoding='utf-8') - returnValue((response.code, headers, body)) + content = yield response.content() + returnValue((response.code, headers, body, content)) @inlineCallbacks def request(self, callback, method, url, **kwargs): From b11b5e4ffc7b83ffd2245069395f97426cf8e0f6 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Besselat Date: Mon, 23 Sep 2019 10:22:59 +0200 Subject: [PATCH 2/4] [test] fix test --- tests/test_base.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/test_base.py b/tests/test_base.py index ef76a9fa..1183cbe4 100755 --- a/tests/test_base.py +++ b/tests/test_base.py @@ -131,39 +131,39 @@ def test_meta(self): class TestCB(object): def test_status_200_passes(self): - response = consul.base.Response(200, None, None) + response = consul.base.Response(200, None, None, None) CB._status(response) @pytest.mark.parametrize( 'response, expected_exception', [ - (Response(400, None, None), consul.base.BadRequest), - (Response(401, None, None), consul.base.ACLDisabled), - (Response(403, None, None), consul.base.ACLPermissionDenied), + (Response(400, None, None, None), consul.base.BadRequest), + (Response(401, None, None, None), consul.base.ACLDisabled), + (Response(403, None, None, None), consul.base.ACLPermissionDenied), ]) def test_status_4xx_raises_error(self, response, expected_exception): with pytest.raises(expected_exception): CB._status(response) def test_status_404_allow_404(self): - response = Response(404, None, None) + response = Response(404, None, None, None) CB._status(response, allow_404=True) def test_status_404_dont_allow_404(self): - response = Response(404, None, None) + response = Response(404, None, None, None) with pytest.raises(consul.base.NotFound): CB._status(response, allow_404=False) def test_status_405_raises_generic_ClientError(self): - response = Response(405, None, None) + response = Response(405, None, None, None) with pytest.raises(consul.base.ClientError): CB._status(response) @pytest.mark.parametrize( 'response', [ - Response(500, None, None), - Response(599, None, None), + Response(500, None, None, None), + Response(599, None, None, None), ]) def test_status_5xx_raises_error(self, response): with pytest.raises(consul.base.ConsulException): From 790df2850b54e2573261bacfb7c0b948d9150165 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Besselat Date: Mon, 23 Sep 2019 10:50:20 +0200 Subject: [PATCH 3/4] [fix] tox styleguide --- consul/base.py | 7 ++++--- consul/std.py | 3 ++- consul/tornado.py | 3 ++- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/consul/base.py b/consul/base.py index 8b9e0064..30916066 100755 --- a/consul/base.py +++ b/consul/base.py @@ -162,7 +162,8 @@ def _compat( return ret -Response = collections.namedtuple('Response', ['code', 'headers', 'body', 'content']) +Response = collections.namedtuple('Response', + ['code', 'headers', 'body', 'content']) # @@ -2447,7 +2448,7 @@ def raft_config(self): CB.json(), '/v1/operator/raft/configuration') class Snapshot(object): - def __init__(self,agent): + def __init__(self, agent): self.agent = agent def get(self): @@ -2455,7 +2456,7 @@ def get(self): Returns gzipped snapshot of current consul cluster """ return self.agent.http.get( - CB.binary(),'/v1/snapshot') + CB.binary(), '/v1/snapshot') def save(self, file_path): """ diff --git a/consul/std.py b/consul/std.py index d2dddbb4..f2a77a61 100644 --- a/consul/std.py +++ b/consul/std.py @@ -14,7 +14,8 @@ def __init__(self, *args, **kwargs): def response(self, response): response.encoding = 'utf-8' return base.Response( - response.status_code, response.headers, response.text, response.content) + response.status_code, response.headers, + response.text, response.content) def get(self, callback, path, params=None): uri = self.uri(path, params) diff --git a/consul/tornado.py b/consul/tornado.py index 48493de7..3ae4dd8e 100644 --- a/consul/tornado.py +++ b/consul/tornado.py @@ -16,7 +16,8 @@ def __init__(self, *args, **kwargs): def response(self, response): return base.Response( - response.code, response.headers, response.body.decode('utf-8'), response.body) + response.code, response.headers, + response.body.decode('utf-8'), response.body) @gen.coroutine def _request(self, callback, request): From 0b1118b0e470f55249b43c61f3cce1c2a43e5dff Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Besselat Date: Tue, 24 Sep 2019 09:13:18 +0200 Subject: [PATCH 4/4] [snapshot] add restore function --- consul/base.py | 36 ++++++++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/consul/base.py b/consul/base.py index 30916066..885bca8a 100755 --- a/consul/base.py +++ b/consul/base.py @@ -2451,17 +2451,45 @@ class Snapshot(object): def __init__(self, agent): self.agent = agent - def get(self): + def get(self, dc=None, token=None): """ Returns gzipped snapshot of current consul cluster """ + params = [] + token = token or self.agent.token + if token: + params.append(('token', token)) + if dc: + params.append(('dc', dc)) return self.agent.http.get( - CB.binary(), '/v1/snapshot') + CB.binary(), '/v1/snapshot', params=params) - def save(self, file_path): + def save(self, file_path, dc=None, token=None): """ Backup snapshot in a file """ backup_file = open(file_path, 'w+b') - backup_file.write(self.get()) + backup_file.write(self.get(dc, token)) backup_file.close() + + def restore(self, file_path=None, data=None, dc=None, token=None): + """ + Restore snapshot from a file or from data + """ + if file_path or data: + if file_path: + backup_file = open(file_path, 'rb') + data = backup_file.read() + backup_file.close() + params = [] + token = token or self.agent.token + if token: + params.append(('token', token)) + if dc: + params.append(('dc', dc)) + res = self.agent.http.put(CB.bool(), '/v1/snapshot', + params=params, + data=data) + return res + else: + return False