From 7350c6a1597f7cfe81309ea5d5bd7aa3c395f95c Mon Sep 17 00:00:00 2001 From: Anes Mukhametov Date: Thu, 2 Aug 2018 16:01:16 +0300 Subject: [PATCH 01/17] fix txn token,dc and consistency --- consul/base.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/consul/base.py b/consul/base.py index 0c86d406..f5706d8a 100755 --- a/consul/base.py +++ b/consul/base.py @@ -665,7 +665,11 @@ class Txn(object): def __init__(self, agent): self.agent = agent - def put(self, payload): + def put(self, + payload, + token=None, + consistency=None, + dc=None): """ Create a transaction by submitting a list of operations to apply to the KV store inside of a transaction. If any operation fails, the @@ -687,8 +691,21 @@ def put(self, payload): } } """ + + params = {} + token = token or self.agent.token + if token: + params['token'] = token + dc = dc or self.agent.dc + if dc: + params['dc'] = dc + consistency = consistency or self.agent.consistency + if consistency in ('consistent', 'stale'): + params[consistency] = '1' + return self.agent.http.put(CB.json(), "/v1/txn", - data=json.dumps(payload)) + data=json.dumps(payload), + params=params) class Agent(object): """ From ced8fcfac0645c799f60ecec7544d78df21cab06 Mon Sep 17 00:00:00 2001 From: Anes Mukhametov Date: Fri, 3 Aug 2018 12:15:59 +0300 Subject: [PATCH 02/17] add support to auto base64-decode values --- consul/base.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/consul/base.py b/consul/base.py index f5706d8a..61bf365b 100755 --- a/consul/base.py +++ b/consul/base.py @@ -203,7 +203,8 @@ def json( one=False, decode=False, is_id=False, - index=False): + index=False, + txn=False): """ *map* is a function to apply to the final result. @@ -227,9 +228,15 @@ def cb(response): data = json.loads(response.body) if decode: - for item in data: - if item.get(decode) is not None: - item[decode] = base64.b64decode(item[decode]) + if txn and data.get('Results') is not None: + for item in data.get('Results'): + if item.get('KV').get('Value') is not None: + item['KV']['Value'] = base64.b64decode(item['KV']['Value']) + elif not txn: + for item in data: + if item.get(decode) is not None: + item[decode] = base64.b64decode(item[decode]) + if is_id: data = data['ID'] if one: @@ -703,7 +710,7 @@ def put(self, if consistency in ('consistent', 'stale'): params[consistency] = '1' - return self.agent.http.put(CB.json(), "/v1/txn", + return self.agent.http.put(CB.json(txn=True, decode=True), "/v1/txn", data=json.dumps(payload), params=params) From 0a6e74e0e473166a622c29bec750b9103eba27de Mon Sep 17 00:00:00 2001 From: Anes Mukhametov Date: Mon, 6 Aug 2018 12:21:02 +0300 Subject: [PATCH 03/17] add decode txn result value option --- consul/base.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/consul/base.py b/consul/base.py index 61bf365b..8185186c 100755 --- a/consul/base.py +++ b/consul/base.py @@ -676,7 +676,8 @@ def put(self, payload, token=None, consistency=None, - dc=None): + dc=None, + decode=False): """ Create a transaction by submitting a list of operations to apply to the KV store inside of a transaction. If any operation fails, the @@ -710,7 +711,7 @@ def put(self, if consistency in ('consistent', 'stale'): params[consistency] = '1' - return self.agent.http.put(CB.json(txn=True, decode=True), "/v1/txn", + return self.agent.http.put(CB.json(txn=True, decode=decode), "/v1/txn", data=json.dumps(payload), params=params) From e2da0158f47d819a98d41df907182a0d8db0674d Mon Sep 17 00:00:00 2001 From: Anes Mukhametov Date: Mon, 6 Aug 2018 12:28:35 +0300 Subject: [PATCH 04/17] reduce str len --- consul/base.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/consul/base.py b/consul/base.py index 8185186c..f1e8a507 100755 --- a/consul/base.py +++ b/consul/base.py @@ -231,7 +231,8 @@ def cb(response): if txn and data.get('Results') is not None: for item in data.get('Results'): if item.get('KV').get('Value') is not None: - item['KV']['Value'] = base64.b64decode(item['KV']['Value']) + item['KV']['Value'] = base64.b64decode( + item['KV']['Value']) elif not txn: for item in data: if item.get(decode) is not None: @@ -711,7 +712,8 @@ def put(self, if consistency in ('consistent', 'stale'): params[consistency] = '1' - return self.agent.http.put(CB.json(txn=True, decode=decode), "/v1/txn", + return self.agent.http.put(CB.json(txn=True, decode=decode), + "/v1/txn", data=json.dumps(payload), params=params) From 2e4d1bc44332e1d6c60976df04c4b86683b35612 Mon Sep 17 00:00:00 2001 From: Anes Mukhametov Date: Mon, 6 Aug 2018 12:29:20 +0300 Subject: [PATCH 05/17] add decode txn test --- tests/test_std.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/test_std.py b/tests/test_std.py index 6aef1e72..67f375a8 100644 --- a/tests/test_std.py +++ b/tests/test_std.py @@ -170,6 +170,17 @@ def test_transaction(self, consul_port): r = c.txn.put([d]) assert r["Results"][0]["KV"]["Value"] == value + def test_transaction_decode(self, consul_port): + c = consul.Consul(port=consul_port) + value = '123 qweqwe' + d = {"KV": {"Verb": "set", "Key": "asdf", "Value": value}} + r = c.txn.put([d]) + assert r["Errors"] is None + + d = {"KV": {"Verb": "get", "Key": "asdf"}} + r = c.txn.put([d], decode=True) + assert r["Results"][0]["KV"]["Value"] == value + def test_event(self, consul_port): c = consul.Consul(port=consul_port) From fded80037115486294d242cf86883d558aae16d5 Mon Sep 17 00:00:00 2001 From: Anes Mukhametov Date: Mon, 6 Aug 2018 12:33:07 +0300 Subject: [PATCH 06/17] try to fix test --- tests/test_std.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_std.py b/tests/test_std.py index 67f375a8..bc68d2f3 100644 --- a/tests/test_std.py +++ b/tests/test_std.py @@ -173,7 +173,8 @@ def test_transaction(self, consul_port): def test_transaction_decode(self, consul_port): c = consul.Consul(port=consul_port) value = '123 qweqwe' - d = {"KV": {"Verb": "set", "Key": "asdf", "Value": value}} + enc_value = base64.b64encode(value).decode("utf8") + d = {"KV": {"Verb": "set", "Key": "asdf", "Value": enc_value}} r = c.txn.put([d]) assert r["Errors"] is None From d2c65a5c119664521b197b0ef6fb87f2bddd42ef Mon Sep 17 00:00:00 2001 From: Anes Mukhametov Date: Mon, 6 Aug 2018 12:38:57 +0300 Subject: [PATCH 07/17] str to byte --- tests/test_std.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_std.py b/tests/test_std.py index bc68d2f3..eab685dc 100644 --- a/tests/test_std.py +++ b/tests/test_std.py @@ -173,7 +173,7 @@ def test_transaction(self, consul_port): def test_transaction_decode(self, consul_port): c = consul.Consul(port=consul_port) value = '123 qweqwe' - enc_value = base64.b64encode(value).decode("utf8") + enc_value = base64.b64encode(value.encode()).decode("utf8") d = {"KV": {"Verb": "set", "Key": "asdf", "Value": enc_value}} r = c.txn.put([d]) assert r["Errors"] is None From cf2804c5ba7b3e7dae823f60cbed589623bebbf9 Mon Sep 17 00:00:00 2001 From: Anes Mukhametov Date: Mon, 6 Aug 2018 12:48:59 +0300 Subject: [PATCH 08/17] one more try to make the test successful --- tests/test_std.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_std.py b/tests/test_std.py index eab685dc..9557c437 100644 --- a/tests/test_std.py +++ b/tests/test_std.py @@ -172,8 +172,8 @@ def test_transaction(self, consul_port): def test_transaction_decode(self, consul_port): c = consul.Consul(port=consul_port) - value = '123 qweqwe' - enc_value = base64.b64encode(value.encode()).decode("utf8") + value = b'123 qweqwe' + enc_value = base64.b64encode(value).decode("utf8") d = {"KV": {"Verb": "set", "Key": "asdf", "Value": enc_value}} r = c.txn.put([d]) assert r["Errors"] is None From f13b8d36d1bfbea088455aaee5d59685047da41c Mon Sep 17 00:00:00 2001 From: Anes Mukhametov Date: Mon, 6 Aug 2018 12:56:24 +0300 Subject: [PATCH 09/17] try to fix indent --- consul/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consul/base.py b/consul/base.py index f1e8a507..1a8bedcd 100755 --- a/consul/base.py +++ b/consul/base.py @@ -232,7 +232,7 @@ def cb(response): for item in data.get('Results'): if item.get('KV').get('Value') is not None: item['KV']['Value'] = base64.b64decode( - item['KV']['Value']) + item['KV']['Value']) elif not txn: for item in data: if item.get(decode) is not None: From 701984175efc51d53110e0911b1619b86a147948 Mon Sep 17 00:00:00 2001 From: Anes Mukhametov Date: Mon, 6 Aug 2018 13:01:04 +0300 Subject: [PATCH 10/17] try to fix indent --- consul/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consul/base.py b/consul/base.py index 1a8bedcd..8e72fe7f 100755 --- a/consul/base.py +++ b/consul/base.py @@ -232,7 +232,7 @@ def cb(response): for item in data.get('Results'): if item.get('KV').get('Value') is not None: item['KV']['Value'] = base64.b64decode( - item['KV']['Value']) + item['KV']['Value']) elif not txn: for item in data: if item.get(decode) is not None: From ee901c07b8542025fb55e81db3074e45e268d1e5 Mon Sep 17 00:00:00 2001 From: Anes Mukhametov Date: Mon, 6 Aug 2018 13:10:45 +0300 Subject: [PATCH 11/17] try to fix indent --- consul/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consul/base.py b/consul/base.py index 8e72fe7f..f45c5774 100755 --- a/consul/base.py +++ b/consul/base.py @@ -232,7 +232,7 @@ def cb(response): for item in data.get('Results'): if item.get('KV').get('Value') is not None: item['KV']['Value'] = base64.b64decode( - item['KV']['Value']) + item['KV']['Value']) elif not txn: for item in data: if item.get(decode) is not None: From 462901aeba447257cd43604cfd20703e42b512cf Mon Sep 17 00:00:00 2001 From: Anes Mukhametov Date: Mon, 6 Aug 2018 13:16:53 +0300 Subject: [PATCH 12/17] try to fix indent --- consul/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consul/base.py b/consul/base.py index f45c5774..c9e16efe 100755 --- a/consul/base.py +++ b/consul/base.py @@ -232,7 +232,7 @@ def cb(response): for item in data.get('Results'): if item.get('KV').get('Value') is not None: item['KV']['Value'] = base64.b64decode( - item['KV']['Value']) + item['KV']['Value']) elif not txn: for item in data: if item.get(decode) is not None: From 9c56b2f6e959bb5301dc9973f1f2d813773e211b Mon Sep 17 00:00:00 2001 From: Anes Mukhametov Date: Mon, 6 Aug 2018 13:21:23 +0300 Subject: [PATCH 13/17] try to fix indent --- consul/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consul/base.py b/consul/base.py index c9e16efe..bf7efc31 100755 --- a/consul/base.py +++ b/consul/base.py @@ -232,7 +232,7 @@ def cb(response): for item in data.get('Results'): if item.get('KV').get('Value') is not None: item['KV']['Value'] = base64.b64decode( - item['KV']['Value']) + item['KV']['Value']) elif not txn: for item in data: if item.get(decode) is not None: From 5eb01daeafa42bc2b84d1709e111d661579bd38e Mon Sep 17 00:00:00 2001 From: Anes Mukhametov Date: Mon, 8 Oct 2018 21:21:14 +0300 Subject: [PATCH 14/17] Thanks, @matusvalo --- consul/base.py | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/consul/base.py b/consul/base.py index bf7efc31..08ca9ac6 100755 --- a/consul/base.py +++ b/consul/base.py @@ -219,6 +219,9 @@ def json( *decode* if specified this key will be base64 decoded. *is_id* only the 'ID' field of the json object will be returned. + + *txn* if specified along with *decode*, the answer will be treated as + transaction result and all values will be base64-decoded """ def cb(response): CB._status(response, allow_404=allow_404) @@ -673,12 +676,12 @@ class Txn(object): def __init__(self, agent): self.agent = agent - def put(self, + def put( + self, payload, token=None, consistency=None, - dc=None, - decode=False): + dc=None): """ Create a transaction by submitting a list of operations to apply to the KV store inside of a transaction. If any operation fails, the @@ -699,20 +702,28 @@ def put(self, "Session": "" } } + + *token* is an optional `ACL token`_ to apply to this request. If + the token's policy is not allowed to delete to this key an + *ACLPermissionDenied* exception will be raised. + + *consistency* can be either 'default', 'consistent' or 'stale'. if + not specified *consistency* will the consistency level this client + was configured with. """ - params = {} + params = [] token = token or self.agent.token if token: - params['token'] = token + params.append(('token', token)) dc = dc or self.agent.dc if dc: - params['dc'] = dc + params.append(('dc', dc)) consistency = consistency or self.agent.consistency if consistency in ('consistent', 'stale'): - params[consistency] = '1' + params.append(('consistency', 1)) - return self.agent.http.put(CB.json(txn=True, decode=decode), + return self.agent.http.put(CB.json(txn=True, decode=True), "/v1/txn", data=json.dumps(payload), params=params) From 75bb0ac12c8983611e38a0c1d6bbd59be604f436 Mon Sep 17 00:00:00 2001 From: Anes Mukhametov Date: Mon, 8 Oct 2018 21:25:59 +0300 Subject: [PATCH 15/17] update due to api change --- tests/test_std.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_std.py b/tests/test_std.py index 9557c437..5c06a9df 100644 --- a/tests/test_std.py +++ b/tests/test_std.py @@ -179,7 +179,7 @@ def test_transaction_decode(self, consul_port): assert r["Errors"] is None d = {"KV": {"Verb": "get", "Key": "asdf"}} - r = c.txn.put([d], decode=True) + r = c.txn.put([d]) assert r["Results"][0]["KV"]["Value"] == value def test_event(self, consul_port): From cd0a2c98a47457202d3901b01cee7930b6334747 Mon Sep 17 00:00:00 2001 From: Anes Mukhametov Date: Mon, 8 Oct 2018 21:41:07 +0300 Subject: [PATCH 16/17] Revert "update due to api change" This reverts commit 75bb0ac12c8983611e38a0c1d6bbd59be604f436. --- tests/test_std.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_std.py b/tests/test_std.py index 5c06a9df..9557c437 100644 --- a/tests/test_std.py +++ b/tests/test_std.py @@ -179,7 +179,7 @@ def test_transaction_decode(self, consul_port): assert r["Errors"] is None d = {"KV": {"Verb": "get", "Key": "asdf"}} - r = c.txn.put([d]) + r = c.txn.put([d], decode=True) assert r["Results"][0]["KV"]["Value"] == value def test_event(self, consul_port): From 6fe3a4f82f577644fde39756c2c3088a0c27731f Mon Sep 17 00:00:00 2001 From: Anes Mukhametov Date: Mon, 8 Oct 2018 21:44:37 +0300 Subject: [PATCH 17/17] revert auto-decode result, keep the API safe --- consul/base.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/consul/base.py b/consul/base.py index 08ca9ac6..6d4fd014 100755 --- a/consul/base.py +++ b/consul/base.py @@ -681,7 +681,8 @@ def put( payload, token=None, consistency=None, - dc=None): + dc=None, + decode=False): """ Create a transaction by submitting a list of operations to apply to the KV store inside of a transaction. If any operation fails, the @@ -710,6 +711,8 @@ def put( *consistency* can be either 'default', 'consistent' or 'stale'. if not specified *consistency* will the consistency level this client was configured with. + + *consistency* the result will be base64-decoded if set to True """ params = [] @@ -723,7 +726,7 @@ def put( if consistency in ('consistent', 'stale'): params.append(('consistency', 1)) - return self.agent.http.put(CB.json(txn=True, decode=True), + return self.agent.http.put(CB.json(txn=True, decode=decode), "/v1/txn", data=json.dumps(payload), params=params)