From 5ded4580f7c974d94e10481c65571ef0cc84ef8f Mon Sep 17 00:00:00 2001 From: moshe Date: Wed, 24 Oct 2018 12:15:18 +0300 Subject: [PATCH 001/209] basic injection of seeders --- tests/base_test_ci.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index 54b063e..fd67a69 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -46,10 +46,10 @@ def send_and_wait(self, data): self.send(data) self.wait_for_response() - def start_node_agent_pair(self): + def start_node_agent_pair(self, seeders = "NULL"): docker = Docker() docker.stop('agent_' + str(self.agents)) - docker.start('docker run --network=devnet --name agent_' + str(self.agents) + ' -v /root/spacemesh/devnet/tests:/opt/devnet -v /root/spacemesh/devnet/logs:/opt/logs -e SUBSCRIPTION_NAME_DOWNSTREAM=devnet_tests_agent_' + str(self.agents) + ' -e NODE=' + str(self.agents) + ' spacemesh/devnet_agent:latest python3 /opt/devnet/base_test_agent.py') + docker.start('docker run --network=devnet --name agent_' + str(self.agents) + ' -v /root/spacemesh/devnet/tests:/opt/devnet -v /root/spacemesh/devnet/logs:/opt/logs -e SUBSCRIPTION_NAME_DOWNSTREAM=devnet_tests_agent_' + str(self.agents) + ' -e NODE=' + str(self.agents) + ' -e SEEDERS=' + seeders + ' spacemesh/devnet_agent:latest python3 /opt/devnet/base_test_agent.py') self.agents += 1 if __name__ == '__main__': From b48b4c1a3218a26be21d4647784ea4a45d8f2bbf Mon Sep 17 00:00:00 2001 From: moshe Date: Wed, 24 Oct 2018 12:24:32 +0300 Subject: [PATCH 002/209] move default no seeders to config --- tests/base_test_ci.py | 2 +- tests/config.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index fd67a69..a3c5d65 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -46,7 +46,7 @@ def send_and_wait(self, data): self.send(data) self.wait_for_response() - def start_node_agent_pair(self, seeders = "NULL"): + def start_node_agent_pair(self, seeders = config.CONFIG['no_seeders']): docker = Docker() docker.stop('agent_' + str(self.agents)) docker.start('docker run --network=devnet --name agent_' + str(self.agents) + ' -v /root/spacemesh/devnet/tests:/opt/devnet -v /root/spacemesh/devnet/logs:/opt/logs -e SUBSCRIPTION_NAME_DOWNSTREAM=devnet_tests_agent_' + str(self.agents) + ' -e NODE=' + str(self.agents) + ' -e SEEDERS=' + seeders + ' spacemesh/devnet_agent:latest python3 /opt/devnet/base_test_agent.py') diff --git a/tests/config.py b/tests/config.py index 9127b8b..671e08b 100644 --- a/tests/config.py +++ b/tests/config.py @@ -6,5 +6,6 @@ 'subscription_name_downstream': 'devnet_tests_agent', 'host': '127.0.0.1', 'host_user': 'deploy', - 'host_password': 'deploy_password' + 'host_password': 'deploy_password', + 'no_seeders': 'NULL' } \ No newline at end of file From 6e13567cd510f5aea19ae011cc7a3b6e58d4e6eb Mon Sep 17 00:00:00 2001 From: moshe Date: Wed, 24 Oct 2018 13:07:28 +0300 Subject: [PATCH 003/209] Add a test config and use it --- agent/Dockerfile | 1 + tests/base_test_agent.py | 7 ++++++- tests/test.config.toml | 41 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 tests/test.config.toml diff --git a/agent/Dockerfile b/agent/Dockerfile index 779eb88..eb45791 100644 --- a/agent/Dockerfile +++ b/agent/Dockerfile @@ -4,4 +4,5 @@ RUN apt-get -y update && apt-get install -y python3 python3-pip RUN pip3 install --upgrade pip RUN pip3 install google-cloud google-cloud-pubsub spur RUN mkdir /opt/devnet +ADD ./tests/test.config.toml /opt/devnet/ WORKDIR /opt/devnet \ No newline at end of file diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index 1c79dcd..ce8c7d1 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -18,11 +18,16 @@ def __init__(self): self.node = os.environ['NODE'] self.docker = Docker() self.docker.stop('node_' + self.node) - self.docker.start('docker run --network=devnet --name node_' + self.node + ' -p ' + str(7513 + int(self.node)) + ':7513 -v /root/spacemesh/devnet/logs:/root/.spacemesh/nodes/ spacemesh/node:latest /go/src/github.com/spacemeshos/go-spacemesh/go-spacemesh') + self.modify_seeders() + self.docker.start('docker run --network=devnet --name node_' + self.node + ' -p ' + str(7513 + int(self.node)) + ':7513 -v /root/spacemesh/devnet/logs:/root/.spacemesh/nodes/ -v /root/spacemesh/devnet/tests/test.config.toml:/go/test.config.toml spacemesh/node:latest /go/src/github.com/spacemeshos/go-spacemesh/go-spacemesh -config /go/test.config.toml') subscription_name_downstream = os.environ['SUBSCRIPTION_NAME_DOWNSTREAM'] self.subscriber_downstream = pubsub_v1.SubscriberClient() self.subscription_path_downstream = self.subscriber_downstream.subscription_path(project, subscription_name_downstream) + def modify_seeders(config.CONFIG['no_seeders']): + #test.config.toml + pass + def callback(self, message): self.message = message.data message.ack() diff --git a/tests/test.config.toml b/tests/test.config.toml new file mode 100644 index 0000000..b6a516b --- /dev/null +++ b/tests/test.config.toml @@ -0,0 +1,41 @@ +# sample spacemesh config file +# use the config flag to start a node with a config file. +# e.g $./go-spacemash -config ./config.toml + +# Main Config +[main] +data-folder = "~/.spacemesh-data" + +# Node Config +[p2p] +security-param = 20 +fast-sync = true +tcp-port = 7513 +node-id = "" +new-node= false +dial-timeout = "1m" +conn-keepalive = "48h" +network-id = 1 # 0 - MainNet, 1 - TestNet +response-timeout = "2s" + +# Node Swarm Config +[p2p.swarm] +bootstrap = false +bucketsize = 20 # Routing table bucket size +rtalpha = 3 # Routing table alpha +randcon = 2 # Number of random connections +bootnodes = ["0.0.0.0:7517/j7qWfWaJRVp25ZsnCu9rJ4PmhigZBtesB4YmQHqqPvtR"] + +# API Config +[api] +grpc-server = false +json-server = false +grpc-port = 9091 +json-port = 9090 + +# Time sync NTP Config +[ntp] +max-allowed-time-drift = "10s" +ntp-queries = 5 +default-timeout-latency = "10s" +refresh-ntp-interval = "30m" From 031a3c37e2c408458a616fb655168aa3e98db2a7 Mon Sep 17 00:00:00 2001 From: moshe Date: Wed, 24 Oct 2018 13:11:43 +0300 Subject: [PATCH 004/209] deubgging --- tests/base_test_agent.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index ce8c7d1..703f4c8 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -24,8 +24,9 @@ def __init__(self): self.subscriber_downstream = pubsub_v1.SubscriberClient() self.subscription_path_downstream = self.subscriber_downstream.subscription_path(project, subscription_name_downstream) - def modify_seeders(config.CONFIG['no_seeders']): + def modify_seeders(self, seeders): #test.config.toml + #config.CONFIG['no_seeders'] pass def callback(self, message): From df45af14723b64f5d0574247841b0cf53d05a4a7 Mon Sep 17 00:00:00 2001 From: moshe Date: Wed, 24 Oct 2018 13:12:55 +0300 Subject: [PATCH 005/209] debug --- tests/base_test_agent.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index 703f4c8..4517f16 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -18,7 +18,7 @@ def __init__(self): self.node = os.environ['NODE'] self.docker = Docker() self.docker.stop('node_' + self.node) - self.modify_seeders() + self.modify_seeders(config.CONFIG['no_seeders']) self.docker.start('docker run --network=devnet --name node_' + self.node + ' -p ' + str(7513 + int(self.node)) + ':7513 -v /root/spacemesh/devnet/logs:/root/.spacemesh/nodes/ -v /root/spacemesh/devnet/tests/test.config.toml:/go/test.config.toml spacemesh/node:latest /go/src/github.com/spacemeshos/go-spacemesh/go-spacemesh -config /go/test.config.toml') subscription_name_downstream = os.environ['SUBSCRIPTION_NAME_DOWNSTREAM'] self.subscriber_downstream = pubsub_v1.SubscriberClient() @@ -26,7 +26,6 @@ def __init__(self): def modify_seeders(self, seeders): #test.config.toml - #config.CONFIG['no_seeders'] pass def callback(self, message): From e8206d0e33d6b9c2fa4f4c03590005628d4f4db8 Mon Sep 17 00:00:00 2001 From: moshe Date: Wed, 24 Oct 2018 13:15:16 +0300 Subject: [PATCH 006/209] use the seeders parameter --- tests/base_test_agent.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index 4517f16..7adde29 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -18,7 +18,7 @@ def __init__(self): self.node = os.environ['NODE'] self.docker = Docker() self.docker.stop('node_' + self.node) - self.modify_seeders(config.CONFIG['no_seeders']) + self.modify_seeders(os.environ['SEEDERS']) self.docker.start('docker run --network=devnet --name node_' + self.node + ' -p ' + str(7513 + int(self.node)) + ':7513 -v /root/spacemesh/devnet/logs:/root/.spacemesh/nodes/ -v /root/spacemesh/devnet/tests/test.config.toml:/go/test.config.toml spacemesh/node:latest /go/src/github.com/spacemeshos/go-spacemesh/go-spacemesh -config /go/test.config.toml') subscription_name_downstream = os.environ['SUBSCRIPTION_NAME_DOWNSTREAM'] self.subscriber_downstream = pubsub_v1.SubscriberClient() @@ -26,7 +26,8 @@ def __init__(self): def modify_seeders(self, seeders): #test.config.toml - pass + if (seeders == config.CONFIG['no_seeders']): + return def callback(self, message): self.message = message.data From e4d145f5cce33e78380fe95c4b5ca4b465f243f7 Mon Sep 17 00:00:00 2001 From: moshe Date: Wed, 24 Oct 2018 15:46:50 +0300 Subject: [PATCH 007/209] Replace bootnodes w/ default --- tests/base_test_agent.py | 10 +++++++--- tests/config.py | 2 +- tests/test.config.toml | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index 7adde29..0e32674 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -25,9 +25,13 @@ def __init__(self): self.subscription_path_downstream = self.subscriber_downstream.subscription_path(project, subscription_name_downstream) def modify_seeders(self, seeders): - #test.config.toml - if (seeders == config.CONFIG['no_seeders']): - return + file_name = "/opt/devnet/test.config.toml" + + with open(file_name) as f: + new_config = f.read().replace('BOOT_NODES', seeders) + + with open(file_name, "w") as f: + f.write(new_config) def callback(self, message): self.message = message.data diff --git a/tests/config.py b/tests/config.py index 671e08b..7e49e36 100644 --- a/tests/config.py +++ b/tests/config.py @@ -7,5 +7,5 @@ 'host': '127.0.0.1', 'host_user': 'deploy', 'host_password': 'deploy_password', - 'no_seeders': 'NULL' + 'no_seeders': '["0.0.0.0:7517/j7qWfWaJRVp25ZsnCu9rJ4PmhigZBtesB4YmQHqqPvtR"]' } \ No newline at end of file diff --git a/tests/test.config.toml b/tests/test.config.toml index b6a516b..972bc4e 100644 --- a/tests/test.config.toml +++ b/tests/test.config.toml @@ -24,7 +24,7 @@ bootstrap = false bucketsize = 20 # Routing table bucket size rtalpha = 3 # Routing table alpha randcon = 2 # Number of random connections -bootnodes = ["0.0.0.0:7517/j7qWfWaJRVp25ZsnCu9rJ4PmhigZBtesB4YmQHqqPvtR"] +BOOT_NODES # API Config [api] From f611fc67c18e9a31eedc6a8c6479ebb919662a6d Mon Sep 17 00:00:00 2001 From: moshe Date: Wed, 24 Oct 2018 15:55:41 +0300 Subject: [PATCH 008/209] add the initial --- tests/base_test_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index 0e32674..e248caf 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -28,7 +28,7 @@ def modify_seeders(self, seeders): file_name = "/opt/devnet/test.config.toml" with open(file_name) as f: - new_config = f.read().replace('BOOT_NODES', seeders) + new_config = f.read().replace('BOOT_NODES', 'bootnodes = ' + seeders) with open(file_name, "w") as f: f.write(new_config) From 0bc00c579ab0465425f25f02f3b0d0cf6a891706 Mon Sep 17 00:00:00 2001 From: moshe Date: Thu, 25 Oct 2018 11:57:19 +0300 Subject: [PATCH 009/209] Add the node id and ip to the address sent to the server --- tests/base_test_agent.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index e248caf..d8fb93e 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -58,15 +58,15 @@ def send(self, data): def get_node_id(self): time.sleep(15) - node_id = 'NULL' try: node_id = next(os.walk('/opt/logs'))[1][0] except Exception as e: print('Error finding log folder') print(e.__doc__ ) + return 'NULL' print('NodeId:' + node_id) - return node_id + return 'node_' + self.node + ':7513/' + node_id if __name__ == '__main__': t = BaseDevnetAgent() From 7d49a0e1ca0af893f4fd2b4bcebe86d6cc97b4b3 Mon Sep 17 00:00:00 2001 From: moshe Date: Thu, 25 Oct 2018 12:03:25 +0300 Subject: [PATCH 010/209] increase the minimum size --- tests/test_2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_2.py b/tests/test_2.py index 44a8b44..8c61759 100644 --- a/tests/test_2.py +++ b/tests/test_2.py @@ -12,7 +12,7 @@ def test_sendId(self): self.start_node_agent_pair() self.send_and_wait('GET_NODE_ID') self.assertNotEqual(b'NULL', self.message) - self.assertLess(5, len(self.message)) + self.assertLess(15, len(self.message)) seeds.append(self.message) self.assertEqual(3, len(seeds)) From 1b89a46d3ea054b2dcc1eb106b508176b1390d13 Mon Sep 17 00:00:00 2001 From: moshe Date: Thu, 25 Oct 2018 12:22:32 +0300 Subject: [PATCH 011/209] Create seeders and send to new nodes --- tests/test_2.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/test_2.py b/tests/test_2.py index 8c61759..db8ef28 100644 --- a/tests/test_2.py +++ b/tests/test_2.py @@ -17,5 +17,14 @@ def test_sendId(self): self.assertEqual(3, len(seeds)) + seeders_str = '["' + '","'.join(seeds) + ']"' + print(seeders_str) + for i in range(0, 3): + self.start_node_agent_pair('["0.0.0.0:7517/j7qWfWaJRVp25ZsnCu9rJ4PmhigZBtesB4YmQHqqPvtR"]') + self.send_and_wait('GET_NODE_ID') + self.assertNotEqual(b'NULL', self.message) + self.assertLess(15, len(self.message)) + seeds.append(self.message) + if __name__ == '__main__': unittest.main() \ No newline at end of file From 7de8bff3bde53084ade7aa641ba1367b49c40043 Mon Sep 17 00:00:00 2001 From: moshe Date: Thu, 25 Oct 2018 12:29:57 +0300 Subject: [PATCH 012/209] taking care of binaries --- tests/test_2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_2.py b/tests/test_2.py index db8ef28..e98cb2d 100644 --- a/tests/test_2.py +++ b/tests/test_2.py @@ -17,7 +17,7 @@ def test_sendId(self): self.assertEqual(3, len(seeds)) - seeders_str = '["' + '","'.join(seeds) + ']"' + seeders_str = '["' + str(b'","'.join(str(seeds))) + ']"' print(seeders_str) for i in range(0, 3): self.start_node_agent_pair('["0.0.0.0:7517/j7qWfWaJRVp25ZsnCu9rJ4PmhigZBtesB4YmQHqqPvtR"]') From 95f357341333143ee7e71d4184c1e975fe1ffce0 Mon Sep 17 00:00:00 2001 From: moshe Date: Thu, 25 Oct 2018 12:30:59 +0300 Subject: [PATCH 013/209] fix --- tests/test_2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_2.py b/tests/test_2.py index e98cb2d..1c83ba4 100644 --- a/tests/test_2.py +++ b/tests/test_2.py @@ -17,7 +17,7 @@ def test_sendId(self): self.assertEqual(3, len(seeds)) - seeders_str = '["' + str(b'","'.join(str(seeds))) + ']"' + seeders_str = '["' + str(b'","'.join(seeds)) + ']"' print(seeders_str) for i in range(0, 3): self.start_node_agent_pair('["0.0.0.0:7517/j7qWfWaJRVp25ZsnCu9rJ4PmhigZBtesB4YmQHqqPvtR"]') From ff035a7c35ff7caf8dce65f877d9497c4e69644d Mon Sep 17 00:00:00 2001 From: moshe Date: Thu, 25 Oct 2018 12:33:47 +0300 Subject: [PATCH 014/209] fix seeders --- tests/test_2.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_2.py b/tests/test_2.py index 1c83ba4..4a274a6 100644 --- a/tests/test_2.py +++ b/tests/test_2.py @@ -17,10 +17,11 @@ def test_sendId(self): self.assertEqual(3, len(seeds)) + #'["0.0.0.0:7517/j7qWfWaJRVp25ZsnCu9rJ4PmhigZBtesB4YmQHqqPvtR"]' like seeders_str = '["' + str(b'","'.join(seeds)) + ']"' print(seeders_str) for i in range(0, 3): - self.start_node_agent_pair('["0.0.0.0:7517/j7qWfWaJRVp25ZsnCu9rJ4PmhigZBtesB4YmQHqqPvtR"]') + self.start_node_agent_pair(seeders_str) self.send_and_wait('GET_NODE_ID') self.assertNotEqual(b'NULL', self.message) self.assertLess(15, len(self.message)) From 9c71b5be4b9642013778bf876b705074274689e4 Mon Sep 17 00:00:00 2001 From: moshe Date: Thu, 25 Oct 2018 13:01:30 +0300 Subject: [PATCH 015/209] handle the bytes issue --- tests/test_2.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_2.py b/tests/test_2.py index 4a274a6..5f47b20 100644 --- a/tests/test_2.py +++ b/tests/test_2.py @@ -13,12 +13,12 @@ def test_sendId(self): self.send_and_wait('GET_NODE_ID') self.assertNotEqual(b'NULL', self.message) self.assertLess(15, len(self.message)) - seeds.append(self.message) + seeds.append(str(self.message)) self.assertEqual(3, len(seeds)) #'["0.0.0.0:7517/j7qWfWaJRVp25ZsnCu9rJ4PmhigZBtesB4YmQHqqPvtR"]' like - seeders_str = '["' + str(b'","'.join(seeds)) + ']"' + seeders_str = '["' + '","'.join(seeds) + ']"' print(seeders_str) for i in range(0, 3): self.start_node_agent_pair(seeders_str) From 0882b459b2fa43bc4533c7e54c395115501d5875 Mon Sep 17 00:00:00 2001 From: moshe Date: Thu, 25 Oct 2018 13:20:16 +0300 Subject: [PATCH 016/209] debug --- tests/base_test_agent.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index d8fb93e..64cdd42 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -18,6 +18,7 @@ def __init__(self): self.node = os.environ['NODE'] self.docker = Docker() self.docker.stop('node_' + self.node) + print('seeders:' + os.environ['SEEDERS']) self.modify_seeders(os.environ['SEEDERS']) self.docker.start('docker run --network=devnet --name node_' + self.node + ' -p ' + str(7513 + int(self.node)) + ':7513 -v /root/spacemesh/devnet/logs:/root/.spacemesh/nodes/ -v /root/spacemesh/devnet/tests/test.config.toml:/go/test.config.toml spacemesh/node:latest /go/src/github.com/spacemeshos/go-spacemesh/go-spacemesh -config /go/test.config.toml') subscription_name_downstream = os.environ['SUBSCRIPTION_NAME_DOWNSTREAM'] From 20b42c5fa139c93de6b17f3526aa7d93a978fc21 Mon Sep 17 00:00:00 2001 From: moshe Date: Thu, 25 Oct 2018 13:30:13 +0300 Subject: [PATCH 017/209] no message --- tests/test_2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_2.py b/tests/test_2.py index 5f47b20..56000fc 100644 --- a/tests/test_2.py +++ b/tests/test_2.py @@ -13,7 +13,7 @@ def test_sendId(self): self.send_and_wait('GET_NODE_ID') self.assertNotEqual(b'NULL', self.message) self.assertLess(15, len(self.message)) - seeds.append(str(self.message)) + seeds.append(self.message.decode("utf-8")) self.assertEqual(3, len(seeds)) From c95fbc521f1ddbd4c0e95168a537615f5fbd654f Mon Sep 17 00:00:00 2001 From: moshe Date: Thu, 25 Oct 2018 14:10:20 +0300 Subject: [PATCH 018/209] Send the proper number of messages --- tests/test_2.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/test_2.py b/tests/test_2.py index 56000fc..d63c3aa 100644 --- a/tests/test_2.py +++ b/tests/test_2.py @@ -10,7 +10,9 @@ def test_sendId(self): seeds = [] for i in range(0, 3): self.start_node_agent_pair() - self.send_and_wait('GET_NODE_ID') + self.send('GET_NODE_ID') + for i in range(0, 3): + self.wait_for_response() self.assertNotEqual(b'NULL', self.message) self.assertLess(15, len(self.message)) seeds.append(self.message.decode("utf-8")) @@ -20,9 +22,14 @@ def test_sendId(self): #'["0.0.0.0:7517/j7qWfWaJRVp25ZsnCu9rJ4PmhigZBtesB4YmQHqqPvtR"]' like seeders_str = '["' + '","'.join(seeds) + ']"' print(seeders_str) + + seeds = [] for i in range(0, 3): self.start_node_agent_pair(seeders_str) - self.send_and_wait('GET_NODE_ID') + + self.send('GET_NODE_ID') + for i in range(0, 6): + self.wait_for_response() self.assertNotEqual(b'NULL', self.message) self.assertLess(15, len(self.message)) seeds.append(self.message) From 1f583d9b1d8400d53c5a5d91f523f5468e1be4a4 Mon Sep 17 00:00:00 2001 From: moshe Date: Thu, 25 Oct 2018 14:42:51 +0300 Subject: [PATCH 019/209] Move the cleaner to the tests --- Makefile | 4 ---- tests/base_cleaner.py | 10 ++++++---- tests/base_test_ci.py | 4 ++++ 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index e409e72..c40bfb9 100644 --- a/Makefile +++ b/Makefile @@ -29,9 +29,6 @@ install: @make build clean_containers: - @docker stop agent 2>/dev/null || true - @docker stop node 2>/dev/null || true - @docker stop cleaner 2>/dev/null || true @docker stop test_server 2>/dev/null || true run: @@ -39,7 +36,6 @@ run: @rm -rf $(shell pwd)/logs/*.log @make clean_containers @if [ ! "$(sudo docker network ls | grep devnet)" ]; then sudo docker network create devnet || true; fi - @docker run --rm --network=devnet --name cleaner -e PUBSUB_VERIFICATION_TOKEN='1234' -e PUBSUB_TOPIC='topic' -e GOOGLE_CLOUD_PROJECT='spacemesh-198810' -v $(shell pwd)/tests:/opt/devnet spacemesh/devnet_agent:latest python3 /opt/devnet/base_cleaner.py >> $(shell pwd)/logs/cleaner.log 2>&1 @docker run --rm --network=devnet --name test_server -e PUBSUB_VERIFICATION_TOKEN='1234' -e PUBSUB_TOPIC='topic' -e GOOGLE_CLOUD_PROJECT='spacemesh-198810' -v $(shell pwd)/tests:/opt/devnet spacemesh/devnet_agent:latest python3 /opt/devnet/tests.py >> $(shell pwd)/logs/test.log 2>&1 & build: diff --git a/tests/base_cleaner.py b/tests/base_cleaner.py index 5eae2f4..72282d8 100644 --- a/tests/base_cleaner.py +++ b/tests/base_cleaner.py @@ -27,9 +27,11 @@ def cleanup(self, subscriber, subscription_path): self.endFlag = True time.sleep(1) + def clean(self): + for i in range(0, 10): + self.cleanup(self.subscriber_downstream, self.get_downstream_subscription_path(i)) + self.cleanup(self.subscriber_upstream, self.subscription_path_upstream) + if __name__ == '__main__': t = BaseDevnetCleaner() - - for i in range(0, 10): - t.cleanup(t.subscriber_downstream, t.get_downstream_subscription_path(i)) - t.cleanup(t.subscriber_upstream, t.subscription_path_upstream) \ No newline at end of file + t.clean() \ No newline at end of file diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index a3c5d65..df6b009 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -1,4 +1,5 @@ from dockers import Docker +from base_cleaner import BaseDevnetCleaner import config import os import time @@ -8,6 +9,9 @@ class BaseTest(unittest.TestCase): def setUp(self): + t = BaseDevnetCleaner() + t.clean() + self.endFlag = False self.testLen = 20 self.message = b'NULL' From 4e0be940c47cd4084f33ceaba15cc5143458e003 Mon Sep 17 00:00:00 2001 From: moshe Date: Thu, 25 Oct 2018 14:52:39 +0300 Subject: [PATCH 020/209] dedicated folder for each pair --- tests/base_test_agent.py | 2 +- tests/base_test_ci.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index 64cdd42..c914cfb 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -20,7 +20,7 @@ def __init__(self): self.docker.stop('node_' + self.node) print('seeders:' + os.environ['SEEDERS']) self.modify_seeders(os.environ['SEEDERS']) - self.docker.start('docker run --network=devnet --name node_' + self.node + ' -p ' + str(7513 + int(self.node)) + ':7513 -v /root/spacemesh/devnet/logs:/root/.spacemesh/nodes/ -v /root/spacemesh/devnet/tests/test.config.toml:/go/test.config.toml spacemesh/node:latest /go/src/github.com/spacemeshos/go-spacemesh/go-spacemesh -config /go/test.config.toml') + self.docker.start('docker run --network=devnet --name node_' + self.node + ' -p ' + str(7513 + int(self.node)) + ':7513 -v /root/spacemesh/devnet/logs' + self.node + ':/root/.spacemesh/nodes/ -v /root/spacemesh/devnet/tests/test.config.toml:/go/test.config.toml spacemesh/node:latest /go/src/github.com/spacemeshos/go-spacemesh/go-spacemesh -config /go/test.config.toml') subscription_name_downstream = os.environ['SUBSCRIPTION_NAME_DOWNSTREAM'] self.subscriber_downstream = pubsub_v1.SubscriberClient() self.subscription_path_downstream = self.subscriber_downstream.subscription_path(project, subscription_name_downstream) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index df6b009..ab1f921 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -53,7 +53,7 @@ def send_and_wait(self, data): def start_node_agent_pair(self, seeders = config.CONFIG['no_seeders']): docker = Docker() docker.stop('agent_' + str(self.agents)) - docker.start('docker run --network=devnet --name agent_' + str(self.agents) + ' -v /root/spacemesh/devnet/tests:/opt/devnet -v /root/spacemesh/devnet/logs:/opt/logs -e SUBSCRIPTION_NAME_DOWNSTREAM=devnet_tests_agent_' + str(self.agents) + ' -e NODE=' + str(self.agents) + ' -e SEEDERS=' + seeders + ' spacemesh/devnet_agent:latest python3 /opt/devnet/base_test_agent.py') + docker.start('docker run --network=devnet --name agent_' + str(self.agents) + ' -v /root/spacemesh/devnet/tests:/opt/devnet -v /root/spacemesh/devnet/logs' + str(self.agents) + ':/opt/logs -e SUBSCRIPTION_NAME_DOWNSTREAM=devnet_tests_agent_' + str(self.agents) + ' -e NODE=' + str(self.agents) + ' -e SEEDERS=' + seeders + ' spacemesh/devnet_agent:latest python3 /opt/devnet/base_test_agent.py') self.agents += 1 if __name__ == '__main__': From 08e294f050536bf3f1cc33c8d08f72449ad876ef Mon Sep 17 00:00:00 2001 From: moshe Date: Thu, 25 Oct 2018 15:22:15 +0300 Subject: [PATCH 021/209] add debugging --- tests/base_test_ci.py | 1 + tests/test_2.py | 1 + 2 files changed, 2 insertions(+) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index ab1f921..6ea4384 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -36,6 +36,7 @@ def callback(self, message): self.endFlag = True def send(self, data): + print(data) data = data.encode('utf-8') self.publisher_downstream.publish(self.topic_path_downstream, data=data) diff --git a/tests/test_2.py b/tests/test_2.py index d63c3aa..c101875 100644 --- a/tests/test_2.py +++ b/tests/test_2.py @@ -9,6 +9,7 @@ class Test2(BaseTest): def test_sendId(self): seeds = [] for i in range(0, 3): + print(i) self.start_node_agent_pair() self.send('GET_NODE_ID') for i in range(0, 3): From dcdfd0f91bad44a3ed88757c72264d5b8d929979 Mon Sep 17 00:00:00 2001 From: moshe Date: Thu, 25 Oct 2018 15:30:54 +0300 Subject: [PATCH 022/209] debug stuff --- tests/base_cleaner.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/base_cleaner.py b/tests/base_cleaner.py index 72282d8..c9dbad4 100644 --- a/tests/base_cleaner.py +++ b/tests/base_cleaner.py @@ -18,6 +18,7 @@ def get_downstream_subscription_path(self, i): return self.subscriber_downstream.subscription_path(self.project, config.CONFIG['subscription_name_downstream'] + '_' + str(i)) def callback(self, message): + print('Cleaned: ' + message.data.decode("utf-8")) self.endFlag = False message.ack() From 8b7621a2e6fe366a1b25bf45420ec1081dc5c061 Mon Sep 17 00:00:00 2001 From: moshe Date: Thu, 25 Oct 2018 15:48:47 +0300 Subject: [PATCH 023/209] stops the threads --- tests/base_cleaner.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/base_cleaner.py b/tests/base_cleaner.py index c9dbad4..179ec0f 100644 --- a/tests/base_cleaner.py +++ b/tests/base_cleaner.py @@ -23,10 +23,11 @@ def callback(self, message): message.ack() def cleanup(self, subscriber, subscription_path): - subscriber.subscribe(subscription_path, callback=self.callback) + future = subscriber.subscribe(subscription_path, callback=self.callback) while not self.endFlag: self.endFlag = True time.sleep(1) + future.result(timeout=1) def clean(self): for i in range(0, 10): From 29e3599c1d40c5d1f8968c219d21cdc5d2c79145 Mon Sep 17 00:00:00 2001 From: moshe Date: Thu, 25 Oct 2018 15:54:33 +0300 Subject: [PATCH 024/209] add try catch --- tests/base_cleaner.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/base_cleaner.py b/tests/base_cleaner.py index 179ec0f..6cbfa3e 100644 --- a/tests/base_cleaner.py +++ b/tests/base_cleaner.py @@ -23,11 +23,14 @@ def callback(self, message): message.ack() def cleanup(self, subscriber, subscription_path): - future = subscriber.subscribe(subscription_path, callback=self.callback) - while not self.endFlag: - self.endFlag = True - time.sleep(1) - future.result(timeout=1) + try: + future = subscriber.subscribe(subscription_path, callback=self.callback) + while not self.endFlag: + self.endFlag = True + time.sleep(1) + future.result(timeout=1) + except: + pass def clean(self): for i in range(0, 10): From 5f085b863fd1b45988f7bb17494b6923883d3744 Mon Sep 17 00:00:00 2001 From: moshe Date: Thu, 25 Oct 2018 16:11:11 +0300 Subject: [PATCH 025/209] Return the cleaner to external process, and force it to end before we start the test --- Makefile | 4 +++- tests/base_cleaner.py | 3 +-- tests/base_test_ci.py | 3 --- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index c40bfb9..5a1df77 100644 --- a/Makefile +++ b/Makefile @@ -29,6 +29,7 @@ install: @make build clean_containers: + @docker stop cleaner 2>/dev/null || true @docker stop test_server 2>/dev/null || true run: @@ -36,7 +37,8 @@ run: @rm -rf $(shell pwd)/logs/*.log @make clean_containers @if [ ! "$(sudo docker network ls | grep devnet)" ]; then sudo docker network create devnet || true; fi - @docker run --rm --network=devnet --name test_server -e PUBSUB_VERIFICATION_TOKEN='1234' -e PUBSUB_TOPIC='topic' -e GOOGLE_CLOUD_PROJECT='spacemesh-198810' -v $(shell pwd)/tests:/opt/devnet spacemesh/devnet_agent:latest python3 /opt/devnet/tests.py >> $(shell pwd)/logs/test.log 2>&1 & + @docker run --rm --network=devnet --name cleaner -e PUBSUB_VERIFICATION_TOKEN='1234' -e PUBSUB_TOPIC='topic' -e GOOGLE_CLOUD_PROJECT='spacemesh-198810' -v $(shell pwd)/tests:/opt/devnet spacemesh/devnet_agent:latest python3 /opt/devnet/base_cleaner.py >> $(shell pwd)/logs/cleaner.log 2>&1 &&\ + docker run --rm --network=devnet --name test_server -e PUBSUB_VERIFICATION_TOKEN='1234' -e PUBSUB_TOPIC='topic' -e GOOGLE_CLOUD_PROJECT='spacemesh-198810' -v $(shell pwd)/tests:/opt/devnet spacemesh/devnet_agent:latest python3 /opt/devnet/tests.py >> $(shell pwd)/logs/test.log 2>&1 & build: @gcloud auth configure-docker diff --git a/tests/base_cleaner.py b/tests/base_cleaner.py index 6cbfa3e..316c4af 100644 --- a/tests/base_cleaner.py +++ b/tests/base_cleaner.py @@ -24,11 +24,10 @@ def callback(self, message): def cleanup(self, subscriber, subscription_path): try: - future = subscriber.subscribe(subscription_path, callback=self.callback) + subscriber.subscribe(subscription_path, callback=self.callback) while not self.endFlag: self.endFlag = True time.sleep(1) - future.result(timeout=1) except: pass diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index 6ea4384..c90b631 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -9,9 +9,6 @@ class BaseTest(unittest.TestCase): def setUp(self): - t = BaseDevnetCleaner() - t.clean() - self.endFlag = False self.testLen = 20 self.message = b'NULL' From 31b7fd6dc06e69ab887f2db20d0c547277903cdc Mon Sep 17 00:00:00 2001 From: moshe Date: Thu, 25 Oct 2018 16:40:48 +0300 Subject: [PATCH 026/209] Better the message consuming --- tests/base_test_ci.py | 12 ++++++------ tests/test_2.py | 24 ++++++++++++------------ 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index c90b631..a380417 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -23,25 +23,25 @@ def setUp(self): self.topic_path_downstream = self.publisher_downstream.topic_path(project, topic_name_downstream) self.agents = 0 + self.messages = [] def tearDown(self): self.send('END') def callback(self, message): - self.message = message.data + self.messages.append(message.data.decode("utf-8")) message.ack() - self.endFlag = True def send(self, data): print(data) data = data.encode('utf-8') self.publisher_downstream.publish(self.topic_path_downstream, data=data) - def wait_for_response(self): + def wait_for_response(self, num_messages = 1): for i in range(0, self.testLen): - if self.endFlag: - print(self.message) - break + if len(self.messages) == num_messages: + print(self.messages) + return time.sleep(1) def send_and_wait(self, data): diff --git a/tests/test_2.py b/tests/test_2.py index c101875..557e10c 100644 --- a/tests/test_2.py +++ b/tests/test_2.py @@ -12,28 +12,28 @@ def test_sendId(self): print(i) self.start_node_agent_pair() self.send('GET_NODE_ID') + self.wait_for_response(3) + + self.assertEqual(3, len(self.messages)) for i in range(0, 3): - self.wait_for_response() - self.assertNotEqual(b'NULL', self.message) - self.assertLess(15, len(self.message)) - seeds.append(self.message.decode("utf-8")) - - self.assertEqual(3, len(seeds)) + self.assertNotEqual(b'NULL', self.messages[i]) + self.assertLess(15, len(self.messages[i])) + seeds.append(self.messages[i]) #'["0.0.0.0:7517/j7qWfWaJRVp25ZsnCu9rJ4PmhigZBtesB4YmQHqqPvtR"]' like - seeders_str = '["' + '","'.join(seeds) + ']"' + seeders_str = '["' + '","'.join(self.messages) + ']"' print(seeders_str) + self.messages = [] - seeds = [] for i in range(0, 3): self.start_node_agent_pair(seeders_str) self.send('GET_NODE_ID') + self.wait_for_response(6) + self.assertEqual(6, len(self.messages)) for i in range(0, 6): - self.wait_for_response() - self.assertNotEqual(b'NULL', self.message) - self.assertLess(15, len(self.message)) - seeds.append(self.message) + self.assertNotEqual(b'NULL', self.messages[i]) + self.assertLess(15, len(self.messages[i])) if __name__ == '__main__': unittest.main() \ No newline at end of file From 95cd429d53ef65d570ff9654e7ce0b896d63acd9 Mon Sep 17 00:00:00 2001 From: moshe Date: Thu, 25 Oct 2018 17:22:12 +0300 Subject: [PATCH 027/209] make the test run --- tests/test_2.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/test_2.py b/tests/test_2.py index 557e10c..226ee99 100644 --- a/tests/test_2.py +++ b/tests/test_2.py @@ -9,7 +9,6 @@ class Test2(BaseTest): def test_sendId(self): seeds = [] for i in range(0, 3): - print(i) self.start_node_agent_pair() self.send('GET_NODE_ID') self.wait_for_response(3) @@ -27,10 +26,9 @@ def test_sendId(self): for i in range(0, 3): self.start_node_agent_pair(seeders_str) + self.wait_for_response(3) - self.send('GET_NODE_ID') - self.wait_for_response(6) - self.assertEqual(6, len(self.messages)) + self.assertEqual(3, len(self.messages)) for i in range(0, 6): self.assertNotEqual(b'NULL', self.messages[i]) self.assertLess(15, len(self.messages[i])) From c4b72397a1e723b6e71ea95a9732fbe92dd1bea9 Mon Sep 17 00:00:00 2001 From: moshe Date: Thu, 25 Oct 2018 17:42:07 +0300 Subject: [PATCH 028/209] debug stuff --- tests/test_2.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_2.py b/tests/test_2.py index 226ee99..7b8a9cd 100644 --- a/tests/test_2.py +++ b/tests/test_2.py @@ -22,12 +22,15 @@ def test_sendId(self): #'["0.0.0.0:7517/j7qWfWaJRVp25ZsnCu9rJ4PmhigZBtesB4YmQHqqPvtR"]' like seeders_str = '["' + '","'.join(self.messages) + ']"' print(seeders_str) + print(self.messages) self.messages = [] + print(self.messages) for i in range(0, 3): self.start_node_agent_pair(seeders_str) self.wait_for_response(3) + print(self.messages) self.assertEqual(3, len(self.messages)) for i in range(0, 6): self.assertNotEqual(b'NULL', self.messages[i]) From 7949e11b07d4c52aea943379612771c79381e805 Mon Sep 17 00:00:00 2001 From: moshe Date: Thu, 25 Oct 2018 17:46:32 +0300 Subject: [PATCH 029/209] increase timeouts --- tests/base_test_ci.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index a380417..a13dd2c 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -10,7 +10,7 @@ class BaseTest(unittest.TestCase): def setUp(self): self.endFlag = False - self.testLen = 20 + self.testLen = 60 self.message = b'NULL' project = config.CONFIG['project'] subscription_name_upstream = config.CONFIG['subscription_name_upstream'] From e5074418155e03846b9bcd541a42bdc1bb435487 Mon Sep 17 00:00:00 2001 From: moshe Date: Thu, 25 Oct 2018 17:51:24 +0300 Subject: [PATCH 030/209] add sleep to avoid getting out too fast --- tests/base_test_agent.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index c914cfb..e755f81 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -24,6 +24,7 @@ def __init__(self): subscription_name_downstream = os.environ['SUBSCRIPTION_NAME_DOWNSTREAM'] self.subscriber_downstream = pubsub_v1.SubscriberClient() self.subscription_path_downstream = self.subscriber_downstream.subscription_path(project, subscription_name_downstream) + time.sleep(5) def modify_seeders(self, seeders): file_name = "/opt/devnet/test.config.toml" From 89f63a7b3c38d35f3a2b1051325851438bfd0dc4 Mon Sep 17 00:00:00 2001 From: moshe Date: Thu, 25 Oct 2018 18:06:51 +0300 Subject: [PATCH 031/209] let's investigate the cleaner --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 5a1df77..47cf8e7 100644 --- a/Makefile +++ b/Makefile @@ -29,7 +29,7 @@ install: @make build clean_containers: - @docker stop cleaner 2>/dev/null || true + @(docker stop cleaner 2>/dev/null && docker rm cleaner 2>/dev/null) || true @docker stop test_server 2>/dev/null || true run: @@ -37,7 +37,7 @@ run: @rm -rf $(shell pwd)/logs/*.log @make clean_containers @if [ ! "$(sudo docker network ls | grep devnet)" ]; then sudo docker network create devnet || true; fi - @docker run --rm --network=devnet --name cleaner -e PUBSUB_VERIFICATION_TOKEN='1234' -e PUBSUB_TOPIC='topic' -e GOOGLE_CLOUD_PROJECT='spacemesh-198810' -v $(shell pwd)/tests:/opt/devnet spacemesh/devnet_agent:latest python3 /opt/devnet/base_cleaner.py >> $(shell pwd)/logs/cleaner.log 2>&1 &&\ + @docker run --network=devnet --name cleaner -e PUBSUB_VERIFICATION_TOKEN='1234' -e PUBSUB_TOPIC='topic' -e GOOGLE_CLOUD_PROJECT='spacemesh-198810' -v $(shell pwd)/tests:/opt/devnet spacemesh/devnet_agent:latest python3 /opt/devnet/base_cleaner.py >> $(shell pwd)/logs/cleaner.log 2>&1 &&\ docker run --rm --network=devnet --name test_server -e PUBSUB_VERIFICATION_TOKEN='1234' -e PUBSUB_TOPIC='topic' -e GOOGLE_CLOUD_PROJECT='spacemesh-198810' -v $(shell pwd)/tests:/opt/devnet spacemesh/devnet_agent:latest python3 /opt/devnet/tests.py >> $(shell pwd)/logs/test.log 2>&1 & build: From 073cc18f909b2ef75a20fcee278a5ff0c119942b Mon Sep 17 00:00:00 2001 From: moshe Date: Thu, 25 Oct 2018 18:16:39 +0300 Subject: [PATCH 032/209] testing publishTime attribute --- tests/base_test_ci.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index a13dd2c..d0d664c 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -29,6 +29,7 @@ def tearDown(self): self.send('END') def callback(self, message): + print(message.publishTime)) self.messages.append(message.data.decode("utf-8")) message.ack() From 2bdedd26b2366a9e2204515833484f068162f97b Mon Sep 17 00:00:00 2001 From: moshe Date: Thu, 25 Oct 2018 18:21:22 +0300 Subject: [PATCH 033/209] break the cleaner --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 47cf8e7..69d98a2 100644 --- a/Makefile +++ b/Makefile @@ -37,8 +37,8 @@ run: @rm -rf $(shell pwd)/logs/*.log @make clean_containers @if [ ! "$(sudo docker network ls | grep devnet)" ]; then sudo docker network create devnet || true; fi - @docker run --network=devnet --name cleaner -e PUBSUB_VERIFICATION_TOKEN='1234' -e PUBSUB_TOPIC='topic' -e GOOGLE_CLOUD_PROJECT='spacemesh-198810' -v $(shell pwd)/tests:/opt/devnet spacemesh/devnet_agent:latest python3 /opt/devnet/base_cleaner.py >> $(shell pwd)/logs/cleaner.log 2>&1 &&\ - docker run --rm --network=devnet --name test_server -e PUBSUB_VERIFICATION_TOKEN='1234' -e PUBSUB_TOPIC='topic' -e GOOGLE_CLOUD_PROJECT='spacemesh-198810' -v $(shell pwd)/tests:/opt/devnet spacemesh/devnet_agent:latest python3 /opt/devnet/tests.py >> $(shell pwd)/logs/test.log 2>&1 & + #@docker run --network=devnet --name cleaner -e PUBSUB_VERIFICATION_TOKEN='1234' -e PUBSUB_TOPIC='topic' -e GOOGLE_CLOUD_PROJECT='spacemesh-198810' -v $(shell pwd)/tests:/opt/devnet spacemesh/devnet_agent:latest python3 /opt/devnet/base_cleaner.py >> $(shell pwd)/logs/cleaner.log 2>&1 + @docker run --rm --network=devnet --name test_server -e PUBSUB_VERIFICATION_TOKEN='1234' -e PUBSUB_TOPIC='topic' -e GOOGLE_CLOUD_PROJECT='spacemesh-198810' -v $(shell pwd)/tests:/opt/devnet spacemesh/devnet_agent:latest python3 /opt/devnet/tests.py >> $(shell pwd)/logs/test.log 2>&1 & build: @gcloud auth configure-docker From 2a5040a35ae6ecf11fd5699d4db177ff11883436 Mon Sep 17 00:00:00 2001 From: moshe Date: Thu, 25 Oct 2018 18:23:49 +0300 Subject: [PATCH 034/209] debug --- Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/Makefile b/Makefile index 69d98a2..2c24a30 100644 --- a/Makefile +++ b/Makefile @@ -37,7 +37,6 @@ run: @rm -rf $(shell pwd)/logs/*.log @make clean_containers @if [ ! "$(sudo docker network ls | grep devnet)" ]; then sudo docker network create devnet || true; fi - #@docker run --network=devnet --name cleaner -e PUBSUB_VERIFICATION_TOKEN='1234' -e PUBSUB_TOPIC='topic' -e GOOGLE_CLOUD_PROJECT='spacemesh-198810' -v $(shell pwd)/tests:/opt/devnet spacemesh/devnet_agent:latest python3 /opt/devnet/base_cleaner.py >> $(shell pwd)/logs/cleaner.log 2>&1 @docker run --rm --network=devnet --name test_server -e PUBSUB_VERIFICATION_TOKEN='1234' -e PUBSUB_TOPIC='topic' -e GOOGLE_CLOUD_PROJECT='spacemesh-198810' -v $(shell pwd)/tests:/opt/devnet spacemesh/devnet_agent:latest python3 /opt/devnet/tests.py >> $(shell pwd)/logs/test.log 2>&1 & build: From 9cf096c4be7516a617f8a215c3729ba8e1ef6298 Mon Sep 17 00:00:00 2001 From: moshe Date: Thu, 25 Oct 2018 18:24:55 +0300 Subject: [PATCH 035/209] fix --- tests/base_test_ci.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index d0d664c..87d8ef4 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -29,7 +29,7 @@ def tearDown(self): self.send('END') def callback(self, message): - print(message.publishTime)) + print(message.publishTime) self.messages.append(message.data.decode("utf-8")) message.ack() From fbd85f7591c4dd74d519a724d429258475ceb0a6 Mon Sep 17 00:00:00 2001 From: moshe Date: Thu, 25 Oct 2018 18:29:57 +0300 Subject: [PATCH 036/209] continue investigation --- tests/base_test_ci.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index 87d8ef4..82f45c7 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -29,7 +29,7 @@ def tearDown(self): self.send('END') def callback(self, message): - print(message.publishTime) + print(message) self.messages.append(message.data.decode("utf-8")) message.ack() From 59406894bb2397a8ee08235aa7ee842044c59622 Mon Sep 17 00:00:00 2001 From: moshe Date: Thu, 25 Oct 2018 18:33:52 +0300 Subject: [PATCH 037/209] no message --- tests/base_test_ci.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index 82f45c7..80636c9 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -36,7 +36,7 @@ def callback(self, message): def send(self, data): print(data) data = data.encode('utf-8') - self.publisher_downstream.publish(self.topic_path_downstream, data=data) + self.publisher_downstream.publish(self.topic_path_downstream, data=data, attributes = {'time' : mktime()}) def wait_for_response(self, num_messages = 1): for i in range(0, self.testLen): From c2a2cf0def1689f48a149c8320b781fa3c5616a5 Mon Sep 17 00:00:00 2001 From: moshe Date: Thu, 25 Oct 2018 18:36:03 +0300 Subject: [PATCH 038/209] no message --- tests/base_test_agent.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index e755f81..85600ad 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -36,6 +36,7 @@ def modify_seeders(self, seeders): f.write(new_config) def callback(self, message): + print(message) self.message = message.data message.ack() print(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + " GOT_DOWN_MSG " + "".join(map(chr, self.message))) From 95abe500742299114534f1dc778f07985944a1a0 Mon Sep 17 00:00:00 2001 From: moshe Date: Thu, 25 Oct 2018 18:41:58 +0300 Subject: [PATCH 039/209] fix --- tests/base_test_ci.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index 80636c9..fb99bea 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -36,7 +36,7 @@ def callback(self, message): def send(self, data): print(data) data = data.encode('utf-8') - self.publisher_downstream.publish(self.topic_path_downstream, data=data, attributes = {'time' : mktime()}) + self.publisher_downstream.publish(self.topic_path_downstream, data=data, attributes = {'time' : time.mktime()}) def wait_for_response(self, num_messages = 1): for i in range(0, self.testLen): From 7dd9a45892a1157b7a6d5b6a1d20d8364184eab1 Mon Sep 17 00:00:00 2001 From: moshe Date: Thu, 25 Oct 2018 18:52:08 +0300 Subject: [PATCH 040/209] fix --- tests/base_test_ci.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index fb99bea..2c23389 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -36,7 +36,7 @@ def callback(self, message): def send(self, data): print(data) data = data.encode('utf-8') - self.publisher_downstream.publish(self.topic_path_downstream, data=data, attributes = {'time' : time.mktime()}) + self.publisher_downstream.publish(self.topic_path_downstream, data=data, attributes = {'time' : calendar.timegm(time.gmtime())}) def wait_for_response(self, num_messages = 1): for i in range(0, self.testLen): From a84ed9fb5820c8a0baa35022289344f983cc4813 Mon Sep 17 00:00:00 2001 From: moshe Date: Thu, 25 Oct 2018 18:57:34 +0300 Subject: [PATCH 041/209] fix --- tests/base_test_ci.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index 2c23389..83ec275 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -3,6 +3,7 @@ import config import os import time +import calendar from google.cloud import pubsub_v1 import unittest import spur From a9a7f433532a855feed6ffb322bf7d58ebce490a Mon Sep 17 00:00:00 2001 From: moshe Date: Thu, 25 Oct 2018 19:00:35 +0300 Subject: [PATCH 042/209] make it string --- tests/base_test_ci.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index 83ec275..a6f5837 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -37,7 +37,7 @@ def callback(self, message): def send(self, data): print(data) data = data.encode('utf-8') - self.publisher_downstream.publish(self.topic_path_downstream, data=data, attributes = {'time' : calendar.timegm(time.gmtime())}) + self.publisher_downstream.publish(self.topic_path_downstream, data=data, attributes = {'time' : str(calendar.timegm(time.gmtime()))}) def wait_for_response(self, num_messages = 1): for i in range(0, self.testLen): From bb71815fafc795c0d9b3697371f8f315aa376d42 Mon Sep 17 00:00:00 2001 From: moshe Date: Thu, 25 Oct 2018 19:03:10 +0300 Subject: [PATCH 043/209] another syntax --- tests/base_test_ci.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index a6f5837..2486ed2 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -37,7 +37,7 @@ def callback(self, message): def send(self, data): print(data) data = data.encode('utf-8') - self.publisher_downstream.publish(self.topic_path_downstream, data=data, attributes = {'time' : str(calendar.timegm(time.gmtime()))}) + self.publisher_downstream.publish(self.topic_path_downstream, data=data, time=str(calendar.timegm(time.gmtime()))}) def wait_for_response(self, num_messages = 1): for i in range(0, self.testLen): From 8cd5140edbe37e6fe07a33ce08b6f1292e08f639 Mon Sep 17 00:00:00 2001 From: moshe Date: Thu, 25 Oct 2018 19:03:55 +0300 Subject: [PATCH 044/209] fix --- tests/base_test_ci.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index 2486ed2..3eea624 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -37,7 +37,7 @@ def callback(self, message): def send(self, data): print(data) data = data.encode('utf-8') - self.publisher_downstream.publish(self.topic_path_downstream, data=data, time=str(calendar.timegm(time.gmtime()))}) + self.publisher_downstream.publish(self.topic_path_downstream, data=data, time=str(calendar.timegm(time.gmtime()))) def wait_for_response(self, num_messages = 1): for i in range(0, self.testLen): From dae566fec0526dfd2fc8e476a9e3b7c8787937c5 Mon Sep 17 00:00:00 2001 From: moshe Date: Fri, 26 Oct 2018 00:34:07 +0300 Subject: [PATCH 045/209] Add phase based messages to cleanup to avoid old messages --- tests/base_test_agent.py | 1 + tests/base_test_ci.py | 16 ++++++++++------ tests/test_2.py | 7 +++++-- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index 85600ad..d548a85 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -20,6 +20,7 @@ def __init__(self): self.docker.stop('node_' + self.node) print('seeders:' + os.environ['SEEDERS']) self.modify_seeders(os.environ['SEEDERS']) + self.phase = os.environ['PHASE'] self.docker.start('docker run --network=devnet --name node_' + self.node + ' -p ' + str(7513 + int(self.node)) + ':7513 -v /root/spacemesh/devnet/logs' + self.node + ':/root/.spacemesh/nodes/ -v /root/spacemesh/devnet/tests/test.config.toml:/go/test.config.toml spacemesh/node:latest /go/src/github.com/spacemeshos/go-spacemesh/go-spacemesh -config /go/test.config.toml') subscription_name_downstream = os.environ['SUBSCRIPTION_NAME_DOWNSTREAM'] self.subscriber_downstream = pubsub_v1.SubscriberClient() diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index 3eea624..ce942cc 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -10,6 +10,7 @@ class BaseTest(unittest.TestCase): def setUp(self): + self.create_phase(0, 0) self.endFlag = False self.testLen = 60 self.message = b'NULL' @@ -34,10 +35,13 @@ def callback(self, message): self.messages.append(message.data.decode("utf-8")) message.ack() - def send(self, data): + def create_phase(self, test, phase) + return self.phase = 'test_' + str(test) + '_phase_' + str(phase) + '_' + str(calendar.timegm(time.gmtime())) + + def send(self, data, phase=self.phase): print(data) data = data.encode('utf-8') - self.publisher_downstream.publish(self.topic_path_downstream, data=data, time=str(calendar.timegm(time.gmtime()))) + self.publisher_downstream.publish(self.topic_path_downstream, data=data, phase=phase) def wait_for_response(self, num_messages = 1): for i in range(0, self.testLen): @@ -46,14 +50,14 @@ def wait_for_response(self, num_messages = 1): return time.sleep(1) - def send_and_wait(self, data): - self.send(data) + def send_and_wait(self, data, phase=self.phase): + self.send(data, phase) self.wait_for_response() - def start_node_agent_pair(self, seeders = config.CONFIG['no_seeders']): + def start_node_agent_pair(self, phase=self.phase, seeders=config.CONFIG['no_seeders']): docker = Docker() docker.stop('agent_' + str(self.agents)) - docker.start('docker run --network=devnet --name agent_' + str(self.agents) + ' -v /root/spacemesh/devnet/tests:/opt/devnet -v /root/spacemesh/devnet/logs' + str(self.agents) + ':/opt/logs -e SUBSCRIPTION_NAME_DOWNSTREAM=devnet_tests_agent_' + str(self.agents) + ' -e NODE=' + str(self.agents) + ' -e SEEDERS=' + seeders + ' spacemesh/devnet_agent:latest python3 /opt/devnet/base_test_agent.py') + docker.start('docker run --network=devnet --name agent_' + str(self.agents) + ' -v /root/spacemesh/devnet/tests:/opt/devnet -v /root/spacemesh/devnet/logs' + str(self.agents) + ':/opt/logs -e SUBSCRIPTION_NAME_DOWNSTREAM=devnet_tests_agent_' + str(self.agents) + ' -e PHASE=' + phase + '-e NODE=' + str(self.agents) + ' -e SEEDERS=' + seeders + ' spacemesh/devnet_agent:latest python3 /opt/devnet/base_test_agent.py') self.agents += 1 if __name__ == '__main__': diff --git a/tests/test_2.py b/tests/test_2.py index 7b8a9cd..751c8cf 100644 --- a/tests/test_2.py +++ b/tests/test_2.py @@ -2,15 +2,17 @@ import config import os import time +import calendar from google.cloud import pubsub_v1 import unittest class Test2(BaseTest): def test_sendId(self): + phase_0 = self.create_phase(2, 0) seeds = [] for i in range(0, 3): self.start_node_agent_pair() - self.send('GET_NODE_ID') + self.send('GET_NODE_ID', phase) self.wait_for_response(3) self.assertEqual(3, len(self.messages)) @@ -20,6 +22,7 @@ def test_sendId(self): seeds.append(self.messages[i]) #'["0.0.0.0:7517/j7qWfWaJRVp25ZsnCu9rJ4PmhigZBtesB4YmQHqqPvtR"]' like + phase_1 = self.create_phase(2, 1) seeders_str = '["' + '","'.join(self.messages) + ']"' print(seeders_str) print(self.messages) @@ -32,7 +35,7 @@ def test_sendId(self): print(self.messages) self.assertEqual(3, len(self.messages)) - for i in range(0, 6): + for i in range(0, 3): self.assertNotEqual(b'NULL', self.messages[i]) self.assertLess(15, len(self.messages[i])) From 136b2a1fd93f486e842250a63c717367132a06d9 Mon Sep 17 00:00:00 2001 From: moshe Date: Fri, 26 Oct 2018 00:45:36 +0300 Subject: [PATCH 046/209] typo --- tests/base_test_ci.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index ce942cc..d1d0f38 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -35,7 +35,7 @@ def callback(self, message): self.messages.append(message.data.decode("utf-8")) message.ack() - def create_phase(self, test, phase) + def create_phase(self, test, phase): return self.phase = 'test_' + str(test) + '_phase_' + str(phase) + '_' + str(calendar.timegm(time.gmtime())) def send(self, data, phase=self.phase): From 08242c31d8e0ee2a0422684771e939d782f5048e Mon Sep 17 00:00:00 2001 From: moshe Date: Fri, 26 Oct 2018 00:46:29 +0300 Subject: [PATCH 047/209] typo --- tests/base_test_ci.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index d1d0f38..644470d 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -36,7 +36,7 @@ def callback(self, message): message.ack() def create_phase(self, test, phase): - return self.phase = 'test_' + str(test) + '_phase_' + str(phase) + '_' + str(calendar.timegm(time.gmtime())) + return (self.phase = 'test_' + str(test) + '_phase_' + str(phase) + '_' + str(calendar.timegm(time.gmtime()))) def send(self, data, phase=self.phase): print(data) From 632f1e6ca66d5906fb75614f56ada0672b48db4c Mon Sep 17 00:00:00 2001 From: moshe Date: Fri, 26 Oct 2018 00:47:20 +0300 Subject: [PATCH 048/209] typo --- tests/base_test_ci.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index 644470d..95f4e20 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -36,7 +36,8 @@ def callback(self, message): message.ack() def create_phase(self, test, phase): - return (self.phase = 'test_' + str(test) + '_phase_' + str(phase) + '_' + str(calendar.timegm(time.gmtime()))) + self.phase = 'test_' + str(test) + '_phase_' + str(phase) + '_' + str(calendar.timegm(time.gmtime())) + return self.phase def send(self, data, phase=self.phase): print(data) From bedec856f5de7dfcebfb73646541a008b82ce9ef Mon Sep 17 00:00:00 2001 From: moshe Date: Fri, 26 Oct 2018 00:53:35 +0300 Subject: [PATCH 049/209] fixing the default phases --- tests/base_test_ci.py | 10 +++++++--- tests/test_2.py | 4 +++- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index 95f4e20..145083c 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -39,7 +39,9 @@ def create_phase(self, test, phase): self.phase = 'test_' + str(test) + '_phase_' + str(phase) + '_' + str(calendar.timegm(time.gmtime())) return self.phase - def send(self, data, phase=self.phase): + def send(self, data, phase=null): + if phase == null: + phase = self.phase print(data) data = data.encode('utf-8') self.publisher_downstream.publish(self.topic_path_downstream, data=data, phase=phase) @@ -51,11 +53,13 @@ def wait_for_response(self, num_messages = 1): return time.sleep(1) - def send_and_wait(self, data, phase=self.phase): + def send_and_wait(self, data, phase=null): self.send(data, phase) self.wait_for_response() - def start_node_agent_pair(self, phase=self.phase, seeders=config.CONFIG['no_seeders']): + def start_node_agent_pair(self, phase=null, seeders=config.CONFIG['no_seeders']): + if phase == null: + phase = self.phase docker = Docker() docker.stop('agent_' + str(self.agents)) docker.start('docker run --network=devnet --name agent_' + str(self.agents) + ' -v /root/spacemesh/devnet/tests:/opt/devnet -v /root/spacemesh/devnet/logs' + str(self.agents) + ':/opt/logs -e SUBSCRIPTION_NAME_DOWNSTREAM=devnet_tests_agent_' + str(self.agents) + ' -e PHASE=' + phase + '-e NODE=' + str(self.agents) + ' -e SEEDERS=' + seeders + ' spacemesh/devnet_agent:latest python3 /opt/devnet/base_test_agent.py') diff --git a/tests/test_2.py b/tests/test_2.py index 751c8cf..901f3e2 100644 --- a/tests/test_2.py +++ b/tests/test_2.py @@ -12,7 +12,7 @@ def test_sendId(self): seeds = [] for i in range(0, 3): self.start_node_agent_pair() - self.send('GET_NODE_ID', phase) + self.send('GET_NODE_ID') self.wait_for_response(3) self.assertEqual(3, len(self.messages)) @@ -39,5 +39,7 @@ def test_sendId(self): self.assertNotEqual(b'NULL', self.messages[i]) self.assertLess(15, len(self.messages[i])) + self.send("END", phase_0) + self.send("END", phase_1) if __name__ == '__main__': unittest.main() \ No newline at end of file From 8652e6feba0c66b3530d1860f9c90cd38758e712 Mon Sep 17 00:00:00 2001 From: moshe Date: Fri, 26 Oct 2018 00:55:59 +0300 Subject: [PATCH 050/209] Null => None --- tests/base_test_ci.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index 145083c..42694f4 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -39,8 +39,8 @@ def create_phase(self, test, phase): self.phase = 'test_' + str(test) + '_phase_' + str(phase) + '_' + str(calendar.timegm(time.gmtime())) return self.phase - def send(self, data, phase=null): - if phase == null: + def send(self, data, phase=None): + if phase == None: phase = self.phase print(data) data = data.encode('utf-8') @@ -53,12 +53,12 @@ def wait_for_response(self, num_messages = 1): return time.sleep(1) - def send_and_wait(self, data, phase=null): + def send_and_wait(self, data, phase=None): self.send(data, phase) self.wait_for_response() - def start_node_agent_pair(self, phase=null, seeders=config.CONFIG['no_seeders']): - if phase == null: + def start_node_agent_pair(self, phase=None, seeders=config.CONFIG['no_seeders']): + if phase == None: phase = self.phase docker = Docker() docker.stop('agent_' + str(self.agents)) From ec8f04020914630a05f8b308b732c918a7db8510 Mon Sep 17 00:00:00 2001 From: moshe Date: Fri, 26 Oct 2018 00:59:23 +0300 Subject: [PATCH 051/209] add space --- tests/base_test_ci.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index 42694f4..5138b76 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -62,7 +62,7 @@ def start_node_agent_pair(self, phase=None, seeders=config.CONFIG['no_seeders']) phase = self.phase docker = Docker() docker.stop('agent_' + str(self.agents)) - docker.start('docker run --network=devnet --name agent_' + str(self.agents) + ' -v /root/spacemesh/devnet/tests:/opt/devnet -v /root/spacemesh/devnet/logs' + str(self.agents) + ':/opt/logs -e SUBSCRIPTION_NAME_DOWNSTREAM=devnet_tests_agent_' + str(self.agents) + ' -e PHASE=' + phase + '-e NODE=' + str(self.agents) + ' -e SEEDERS=' + seeders + ' spacemesh/devnet_agent:latest python3 /opt/devnet/base_test_agent.py') + docker.start('docker run --network=devnet --name agent_' + str(self.agents) + ' -v /root/spacemesh/devnet/tests:/opt/devnet -v /root/spacemesh/devnet/logs' + str(self.agents) + ':/opt/logs -e SUBSCRIPTION_NAME_DOWNSTREAM=devnet_tests_agent_' + str(self.agents) + ' -e PHASE=' + phase + ' -e NODE=' + str(self.agents) + ' -e SEEDERS=' + seeders + ' spacemesh/devnet_agent:latest python3 /opt/devnet/base_test_agent.py') self.agents += 1 if __name__ == '__main__': From 9087ecec76fbb45d642ec0a974e22843fd97c420 Mon Sep 17 00:00:00 2001 From: moshe Date: Fri, 26 Oct 2018 01:14:32 +0300 Subject: [PATCH 052/209] avoid other phases messages at client --- tests/base_test_agent.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index d548a85..8b0f9eb 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -40,7 +40,10 @@ def callback(self, message): print(message) self.message = message.data message.ack() + if self.phase != message.attributes.phase: + return; print(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + " GOT_DOWN_MSG " + "".join(map(chr, self.message))) + if b'END' == self.message: self.docker.stop('node_' + self.node) self.endFlag = True From d1d2ced783ce2f7d1142d8e3ae4f2cc47d2db6ed Mon Sep 17 00:00:00 2001 From: moshe Date: Fri, 26 Oct 2018 01:17:16 +0300 Subject: [PATCH 053/209] fix --- tests/base_test_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index 8b0f9eb..0fda65c 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -40,7 +40,7 @@ def callback(self, message): print(message) self.message = message.data message.ack() - if self.phase != message.attributes.phase: + if self.phase != message.attributes['phase']: return; print(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + " GOT_DOWN_MSG " + "".join(map(chr, self.message))) From 9cf5ecab3aed5432c2b7ec2a84bd81f3383b498b Mon Sep 17 00:00:00 2001 From: moshe Date: Fri, 26 Oct 2018 01:27:19 +0300 Subject: [PATCH 054/209] debugging --- tests/base_test_agent.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index 0fda65c..14767fd 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -40,8 +40,10 @@ def callback(self, message): print(message) self.message = message.data message.ack() + print(message.attributes['phase']) if self.phase != message.attributes['phase']: - return; + #return; + print("NO_MESSAGE") print(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + " GOT_DOWN_MSG " + "".join(map(chr, self.message))) if b'END' == self.message: From 47946d711aa083107bdb9fbeddc9e91f32ec6d0b Mon Sep 17 00:00:00 2001 From: moshe Date: Fri, 26 Oct 2018 01:30:09 +0300 Subject: [PATCH 055/209] reduce time --- tests/base_test_ci.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index 5138b76..b41d34d 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -12,7 +12,7 @@ class BaseTest(unittest.TestCase): def setUp(self): self.create_phase(0, 0) self.endFlag = False - self.testLen = 60 + self.testLen = 15 self.message = b'NULL' project = config.CONFIG['project'] subscription_name_upstream = config.CONFIG['subscription_name_upstream'] From 39f7bf8f078909aa6f8d4bfb35b404df5f1d9c58 Mon Sep 17 00:00:00 2001 From: moshe Date: Fri, 26 Oct 2018 12:12:14 +0300 Subject: [PATCH 056/209] Avoid nono relevant messages --- tests/base_test_agent.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index 14767fd..a9dde50 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -42,9 +42,9 @@ def callback(self, message): message.ack() print(message.attributes['phase']) if self.phase != message.attributes['phase']: - #return; print("NO_MESSAGE") - print(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + " GOT_DOWN_MSG " + "".join(map(chr, self.message))) + return + print(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + " GOT_DOWN_MSG " + "".join(map(chr, self.message))) if b'END' == self.message: self.docker.stop('node_' + self.node) From f22593c7d079c8af78a47fae687d67a60e828c60 Mon Sep 17 00:00:00 2001 From: moshe Date: Fri, 26 Oct 2018 21:12:11 +0300 Subject: [PATCH 057/209] fix the order to params --- tests/base_test_ci.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index b41d34d..0da2014 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -57,7 +57,7 @@ def send_and_wait(self, data, phase=None): self.send(data, phase) self.wait_for_response() - def start_node_agent_pair(self, phase=None, seeders=config.CONFIG['no_seeders']): + def start_node_agent_pair(self, seeders=config.CONFIG['no_seeders'], phase=None): if phase == None: phase = self.phase docker = Docker() From 519ec8ceabf64ff7292ad2c4712945da30c31f27 Mon Sep 17 00:00:00 2001 From: moshe Date: Fri, 26 Oct 2018 21:48:36 +0300 Subject: [PATCH 058/209] Remove the cleaner as we move to labeling the messages w/ phase --- Makefile | 1 - tests/base_cleaner.py | 41 ----------------------------------------- 2 files changed, 42 deletions(-) delete mode 100644 tests/base_cleaner.py diff --git a/Makefile b/Makefile index 2c24a30..c40bfb9 100644 --- a/Makefile +++ b/Makefile @@ -29,7 +29,6 @@ install: @make build clean_containers: - @(docker stop cleaner 2>/dev/null && docker rm cleaner 2>/dev/null) || true @docker stop test_server 2>/dev/null || true run: diff --git a/tests/base_cleaner.py b/tests/base_cleaner.py deleted file mode 100644 index 316c4af..0000000 --- a/tests/base_cleaner.py +++ /dev/null @@ -1,41 +0,0 @@ -import config -from google.cloud import pubsub_v1 -import time - -class BaseDevnetCleaner: - def __init__(self): - self.endFlag = False - - self.project = config.CONFIG['project'] - - subscription_name_upstream = config.CONFIG['subscription_name_upstream'] - self.subscriber_upstream = pubsub_v1.SubscriberClient() - self.subscription_path_upstream = self.subscriber_upstream.subscription_path(self.project, subscription_name_upstream) - - self.subscriber_downstream = pubsub_v1.SubscriberClient() - - def get_downstream_subscription_path(self, i): - return self.subscriber_downstream.subscription_path(self.project, config.CONFIG['subscription_name_downstream'] + '_' + str(i)) - - def callback(self, message): - print('Cleaned: ' + message.data.decode("utf-8")) - self.endFlag = False - message.ack() - - def cleanup(self, subscriber, subscription_path): - try: - subscriber.subscribe(subscription_path, callback=self.callback) - while not self.endFlag: - self.endFlag = True - time.sleep(1) - except: - pass - - def clean(self): - for i in range(0, 10): - self.cleanup(self.subscriber_downstream, self.get_downstream_subscription_path(i)) - self.cleanup(self.subscriber_upstream, self.subscription_path_upstream) - -if __name__ == '__main__': - t = BaseDevnetCleaner() - t.clean() \ No newline at end of file From b71ca273a0f9011822fd48d51b2d72bc67511214 Mon Sep 17 00:00:00 2001 From: moshe Date: Fri, 26 Oct 2018 21:51:44 +0300 Subject: [PATCH 059/209] Add the phase to the upstream as well --- tests/base_test_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index a9dde50..dccccc8 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -63,7 +63,7 @@ def act_on_request(self): def send(self, data): data = data.encode('utf-8') - self.publisher_upstream.publish(self.topic_path_upstream, data=data) + self.publisher_upstream.publish(self.topic_path_upstream, data=data, phase=self.phase) def get_node_id(self): time.sleep(15) From 8c4b3e4802da271611559fa0673a15259fe9e6b7 Mon Sep 17 00:00:00 2001 From: moshe Date: Fri, 26 Oct 2018 21:53:41 +0300 Subject: [PATCH 060/209] cleanup --- tests/base_test_ci.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index 0da2014..15ec9ad 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -1,5 +1,4 @@ from dockers import Docker -from base_cleaner import BaseDevnetCleaner import config import os import time From d3143f85a3c5071dc6e6fe315dab9afbebada830 Mon Sep 17 00:00:00 2001 From: moshe Date: Fri, 26 Oct 2018 21:57:27 +0300 Subject: [PATCH 061/209] typo --- tests/test_2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_2.py b/tests/test_2.py index 901f3e2..54477eb 100644 --- a/tests/test_2.py +++ b/tests/test_2.py @@ -23,7 +23,7 @@ def test_sendId(self): #'["0.0.0.0:7517/j7qWfWaJRVp25ZsnCu9rJ4PmhigZBtesB4YmQHqqPvtR"]' like phase_1 = self.create_phase(2, 1) - seeders_str = '["' + '","'.join(self.messages) + ']"' + seeders_str = '["' + '","'.join(self.messages) + '"]' print(seeders_str) print(self.messages) self.messages = [] From 2dc74fbd48e8d0270852186d532570258ef4fc15 Mon Sep 17 00:00:00 2001 From: moshe Date: Fri, 26 Oct 2018 22:05:29 +0300 Subject: [PATCH 062/209] add tags to avoid string processing issues --- tests/test_2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_2.py b/tests/test_2.py index 54477eb..882c25d 100644 --- a/tests/test_2.py +++ b/tests/test_2.py @@ -23,7 +23,7 @@ def test_sendId(self): #'["0.0.0.0:7517/j7qWfWaJRVp25ZsnCu9rJ4PmhigZBtesB4YmQHqqPvtR"]' like phase_1 = self.create_phase(2, 1) - seeders_str = '["' + '","'.join(self.messages) + '"]' + seeders_str = '\'["' + '","'.join(self.messages) + '"]\'' print(seeders_str) print(self.messages) self.messages = [] From 3543d3ffaf62056123749843ba1525fc5fcb4427 Mon Sep 17 00:00:00 2001 From: moshe Date: Mon, 29 Oct 2018 03:24:27 +0200 Subject: [PATCH 063/209] Move prints to logging and enable printing time --- tests/base_test_agent.py | 18 ++++++++++-------- tests/base_test_ci.py | 9 ++++++--- tests/test_2.py | 8 ++++---- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index dccccc8..ab0df4c 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -5,9 +5,11 @@ import datetime from subprocess import call import os +import logging class BaseDevnetAgent: def __init__(self): + logging.basicConfig(format='%(asctime)s %(message)s') self.endFlag = False project = config.CONFIG['project'] @@ -18,7 +20,7 @@ def __init__(self): self.node = os.environ['NODE'] self.docker = Docker() self.docker.stop('node_' + self.node) - print('seeders:' + os.environ['SEEDERS']) + logging.debug('seeders:' + os.environ['SEEDERS']) self.modify_seeders(os.environ['SEEDERS']) self.phase = os.environ['PHASE'] self.docker.start('docker run --network=devnet --name node_' + self.node + ' -p ' + str(7513 + int(self.node)) + ':7513 -v /root/spacemesh/devnet/logs' + self.node + ':/root/.spacemesh/nodes/ -v /root/spacemesh/devnet/tests/test.config.toml:/go/test.config.toml spacemesh/node:latest /go/src/github.com/spacemeshos/go-spacemesh/go-spacemesh -config /go/test.config.toml') @@ -37,14 +39,14 @@ def modify_seeders(self, seeders): f.write(new_config) def callback(self, message): - print(message) + logging.debug(message) self.message = message.data message.ack() - print(message.attributes['phase']) + logging.debug(message.attributes['phase']) if self.phase != message.attributes['phase']: - print("NO_MESSAGE") + logging.debug("NO_MESSAGE") return - print(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + " GOT_DOWN_MSG " + "".join(map(chr, self.message))) + logging.debug(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + " GOT_DOWN_MSG " + "".join(map(chr, self.message))) if b'END' == self.message: self.docker.stop('node_' + self.node) @@ -70,11 +72,11 @@ def get_node_id(self): try: node_id = next(os.walk('/opt/logs'))[1][0] except Exception as e: - print('Error finding log folder') - print(e.__doc__ ) + logging.warning('Error finding log folder') + logging.warning(e.__doc__ ) return 'NULL' - print('NodeId:' + node_id) + logging.debug('NodeId:' + node_id) return 'node_' + self.node + ':7513/' + node_id if __name__ == '__main__': diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index 15ec9ad..3ad91d2 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -6,9 +6,12 @@ from google.cloud import pubsub_v1 import unittest import spur +import logging class BaseTest(unittest.TestCase): def setUp(self): + logging.basicConfig(format='%(asctime)s %(message)s') + self.create_phase(0, 0) self.endFlag = False self.testLen = 15 @@ -30,7 +33,7 @@ def tearDown(self): self.send('END') def callback(self, message): - print(message) + logging.debug(message) self.messages.append(message.data.decode("utf-8")) message.ack() @@ -41,14 +44,14 @@ def create_phase(self, test, phase): def send(self, data, phase=None): if phase == None: phase = self.phase - print(data) + logging.debug(data) data = data.encode('utf-8') self.publisher_downstream.publish(self.topic_path_downstream, data=data, phase=phase) def wait_for_response(self, num_messages = 1): for i in range(0, self.testLen): if len(self.messages) == num_messages: - print(self.messages) + logging.debug(self.messages) return time.sleep(1) diff --git a/tests/test_2.py b/tests/test_2.py index 882c25d..4ba7d9a 100644 --- a/tests/test_2.py +++ b/tests/test_2.py @@ -24,16 +24,16 @@ def test_sendId(self): #'["0.0.0.0:7517/j7qWfWaJRVp25ZsnCu9rJ4PmhigZBtesB4YmQHqqPvtR"]' like phase_1 = self.create_phase(2, 1) seeders_str = '\'["' + '","'.join(self.messages) + '"]\'' - print(seeders_str) - print(self.messages) + logging.debug(seeders_str) + logging.debug(self.messages) self.messages = [] - print(self.messages) + logging.debug(self.messages) for i in range(0, 3): self.start_node_agent_pair(seeders_str) self.wait_for_response(3) - print(self.messages) + logging.debug(self.messages) self.assertEqual(3, len(self.messages)) for i in range(0, 3): self.assertNotEqual(b'NULL', self.messages[i]) From 3c63e216dcffaf0a65fda1b097c0cf164276cc2e Mon Sep 17 00:00:00 2001 From: moshe Date: Mon, 29 Oct 2018 04:11:57 +0200 Subject: [PATCH 064/209] migrate to logging in dockers as well + remove the main section --- tests/dockers.py | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/tests/dockers.py b/tests/dockers.py index 2b051bc..094fe62 100644 --- a/tests/dockers.py +++ b/tests/dockers.py @@ -1,13 +1,14 @@ import config import spur +import logging class Docker(): def start(self, cmd): try: - print(cmd) - print(config.CONFIG['host']) - print(config.CONFIG['host_user']) - print(config.CONFIG['host_password']) + logging.debug(cmd) + logging.debug(config.CONFIG['host']) + logging.debug(config.CONFIG['host_user']) + logging.debug(config.CONFIG['host_password']) shell = spur.SshShell( hostname=config.CONFIG['host'], username=config.CONFIG['host_user'], @@ -16,10 +17,10 @@ def start(self, cmd): ) with shell: result = shell.spawn(cmd.split(' ')) - print('Docker started') + logging.debug('Docker started') except Exception as e: - print('Docker start failed') - print(e.__doc__ ) + logging.warning('Docker start failed') + logging.warning(e.__doc__ ) def run_cmd_interactive(self, cmd): try: @@ -31,14 +32,11 @@ def run_cmd_interactive(self, cmd): ) with shell: result = shell.run(cmd.split(' ')) - print('Run interactive: ' + cmd) + logging.debug('Run interactive: ' + cmd) except Exception as e: - print('Run interactive failed: ' + cmd) - print(e.__doc__ ) + logging.warning('Run interactive failed: ' + cmd) + logging.warning(e.__doc__ ) def stop(self, name): self.run_cmd_interactive("docker stop " + name) - self.run_cmd_interactive("docker rm " + name) - -if __name__ == '__main__': - unittest.main() \ No newline at end of file + self.run_cmd_interactive("docker rm " + name) \ No newline at end of file From f09d033cd6aef8612b312b33ec3922c2916c552f Mon Sep 17 00:00:00 2001 From: moshe Date: Mon, 29 Oct 2018 04:14:35 +0200 Subject: [PATCH 065/209] add to test2 --- tests/test_2.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_2.py b/tests/test_2.py index 4ba7d9a..5ceaeff 100644 --- a/tests/test_2.py +++ b/tests/test_2.py @@ -5,6 +5,7 @@ import calendar from google.cloud import pubsub_v1 import unittest +import logging class Test2(BaseTest): def test_sendId(self): From 401d49dc6b45831d6568a36ded32baee283998e8 Mon Sep 17 00:00:00 2001 From: moshe Date: Mon, 29 Oct 2018 14:37:15 +0200 Subject: [PATCH 066/209] set logging level to debug --- tests/base_test_agent.py | 1 + tests/base_test_ci.py | 1 + 2 files changed, 2 insertions(+) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index ab0df4c..06d9be4 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -10,6 +10,7 @@ class BaseDevnetAgent: def __init__(self): logging.basicConfig(format='%(asctime)s %(message)s') + logging.setLevel(DEBUG) self.endFlag = False project = config.CONFIG['project'] diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index 3ad91d2..2f79b0d 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -11,6 +11,7 @@ class BaseTest(unittest.TestCase): def setUp(self): logging.basicConfig(format='%(asctime)s %(message)s') + logging.setLevel(DEBUG) self.create_phase(0, 0) self.endFlag = False From a1206f936306c46981da9b929b41cc85f79ea248 Mon Sep 17 00:00:00 2001 From: moshe Date: Mon, 29 Oct 2018 14:48:25 +0200 Subject: [PATCH 067/209] typo --- tests/base_test_agent.py | 3 ++- tests/base_test_ci.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index 06d9be4..23225d7 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -6,11 +6,12 @@ from subprocess import call import os import logging +from logging import Logger class BaseDevnetAgent: def __init__(self): logging.basicConfig(format='%(asctime)s %(message)s') - logging.setLevel(DEBUG) + Logger.setLevel(DEBUG) self.endFlag = False project = config.CONFIG['project'] diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index 2f79b0d..f692eb6 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -7,11 +7,12 @@ import unittest import spur import logging +from logging import Logger class BaseTest(unittest.TestCase): def setUp(self): logging.basicConfig(format='%(asctime)s %(message)s') - logging.setLevel(DEBUG) + Logger.setLevel(DEBUG) self.create_phase(0, 0) self.endFlag = False From 51deaef1b2cb1dcb79c580b868f8bfe0dbeb3611 Mon Sep 17 00:00:00 2001 From: moshe Date: Mon, 29 Oct 2018 14:55:07 +0200 Subject: [PATCH 068/209] typo --- tests/base_test_agent.py | 2 +- tests/base_test_ci.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index 23225d7..ce3ff9f 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -11,7 +11,7 @@ class BaseDevnetAgent: def __init__(self): logging.basicConfig(format='%(asctime)s %(message)s') - Logger.setLevel(DEBUG) + Logger.setLevel(logging.DEBUG) self.endFlag = False project = config.CONFIG['project'] diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index f692eb6..6e2f2aa 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -12,7 +12,7 @@ class BaseTest(unittest.TestCase): def setUp(self): logging.basicConfig(format='%(asctime)s %(message)s') - Logger.setLevel(DEBUG) + Logger.setLevel(logging.DEBUG) self.create_phase(0, 0) self.endFlag = False From c695d77d72d8031522f27d69aca8eee6026f3629 Mon Sep 17 00:00:00 2001 From: moshe Date: Mon, 29 Oct 2018 15:14:54 +0200 Subject: [PATCH 069/209] Refactor --- tests/base_test_agent.py | 3 +-- tests/base_test_ci.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index ce3ff9f..0589a9f 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -10,8 +10,7 @@ class BaseDevnetAgent: def __init__(self): - logging.basicConfig(format='%(asctime)s %(message)s') - Logger.setLevel(logging.DEBUG) + logging.basicConfig(format='%(asctime)s %(message)s', level=logging.DEBUG) self.endFlag = False project = config.CONFIG['project'] diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index 6e2f2aa..36ab9c7 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -11,8 +11,7 @@ class BaseTest(unittest.TestCase): def setUp(self): - logging.basicConfig(format='%(asctime)s %(message)s') - Logger.setLevel(logging.DEBUG) + logging.basicConfig(format='%(asctime)s %(message)s', level=logging.DEBUG) self.create_phase(0, 0) self.endFlag = False From e9a0e6cf15a0cc934f5b3d45327d4d76c4b9dc58 Mon Sep 17 00:00:00 2001 From: moshe Date: Mon, 29 Oct 2018 15:38:51 +0200 Subject: [PATCH 070/209] move to info --- tests/base_test_agent.py | 14 +++++++------- tests/base_test_ci.py | 8 ++++---- tests/test_2.py | 8 ++++---- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index 0589a9f..efb3975 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -10,7 +10,7 @@ class BaseDevnetAgent: def __init__(self): - logging.basicConfig(format='%(asctime)s %(message)s', level=logging.DEBUG) + logging.basicConfig(format='%(asctime)s %(message)s', level=logging.INFO) self.endFlag = False project = config.CONFIG['project'] @@ -21,7 +21,7 @@ def __init__(self): self.node = os.environ['NODE'] self.docker = Docker() self.docker.stop('node_' + self.node) - logging.debug('seeders:' + os.environ['SEEDERS']) + logging.info('seeders:' + os.environ['SEEDERS']) self.modify_seeders(os.environ['SEEDERS']) self.phase = os.environ['PHASE'] self.docker.start('docker run --network=devnet --name node_' + self.node + ' -p ' + str(7513 + int(self.node)) + ':7513 -v /root/spacemesh/devnet/logs' + self.node + ':/root/.spacemesh/nodes/ -v /root/spacemesh/devnet/tests/test.config.toml:/go/test.config.toml spacemesh/node:latest /go/src/github.com/spacemeshos/go-spacemesh/go-spacemesh -config /go/test.config.toml') @@ -40,14 +40,14 @@ def modify_seeders(self, seeders): f.write(new_config) def callback(self, message): - logging.debug(message) + logging.info(message) self.message = message.data message.ack() - logging.debug(message.attributes['phase']) + logging.info(message.attributes['phase']) if self.phase != message.attributes['phase']: - logging.debug("NO_MESSAGE") + logging.info("NO_MESSAGE") return - logging.debug(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + " GOT_DOWN_MSG " + "".join(map(chr, self.message))) + logging.info(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + " GOT_DOWN_MSG " + "".join(map(chr, self.message))) if b'END' == self.message: self.docker.stop('node_' + self.node) @@ -77,7 +77,7 @@ def get_node_id(self): logging.warning(e.__doc__ ) return 'NULL' - logging.debug('NodeId:' + node_id) + logging.info('NodeId:' + node_id) return 'node_' + self.node + ':7513/' + node_id if __name__ == '__main__': diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index 36ab9c7..7ae9a70 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -11,7 +11,7 @@ class BaseTest(unittest.TestCase): def setUp(self): - logging.basicConfig(format='%(asctime)s %(message)s', level=logging.DEBUG) + logging.basicConfig(format='%(asctime)s %(message)s', level=logging.INFO) self.create_phase(0, 0) self.endFlag = False @@ -34,7 +34,7 @@ def tearDown(self): self.send('END') def callback(self, message): - logging.debug(message) + logging.info(message) self.messages.append(message.data.decode("utf-8")) message.ack() @@ -45,14 +45,14 @@ def create_phase(self, test, phase): def send(self, data, phase=None): if phase == None: phase = self.phase - logging.debug(data) + logging.info(data) data = data.encode('utf-8') self.publisher_downstream.publish(self.topic_path_downstream, data=data, phase=phase) def wait_for_response(self, num_messages = 1): for i in range(0, self.testLen): if len(self.messages) == num_messages: - logging.debug(self.messages) + logging.info(self.messages) return time.sleep(1) diff --git a/tests/test_2.py b/tests/test_2.py index 5ceaeff..2b9d9fe 100644 --- a/tests/test_2.py +++ b/tests/test_2.py @@ -25,16 +25,16 @@ def test_sendId(self): #'["0.0.0.0:7517/j7qWfWaJRVp25ZsnCu9rJ4PmhigZBtesB4YmQHqqPvtR"]' like phase_1 = self.create_phase(2, 1) seeders_str = '\'["' + '","'.join(self.messages) + '"]\'' - logging.debug(seeders_str) - logging.debug(self.messages) + logging.info(seeders_str) + logging.info(self.messages) self.messages = [] - logging.debug(self.messages) + logging.info(self.messages) for i in range(0, 3): self.start_node_agent_pair(seeders_str) self.wait_for_response(3) - logging.debug(self.messages) + logging.info(self.messages) self.assertEqual(3, len(self.messages)) for i in range(0, 3): self.assertNotEqual(b'NULL', self.messages[i]) From 12baab5e95e7ba7765129303ba4ae845ca9dc552 Mon Sep 17 00:00:00 2001 From: moshe Date: Mon, 29 Oct 2018 15:46:58 +0200 Subject: [PATCH 071/209] block the fast exit for debugging --- tests/base_test_ci.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index 7ae9a70..b96742b 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -51,9 +51,9 @@ def send(self, data, phase=None): def wait_for_response(self, num_messages = 1): for i in range(0, self.testLen): - if len(self.messages) == num_messages: - logging.info(self.messages) - return +# if len(self.messages) == num_messages: +# logging.info(self.messages) +# return time.sleep(1) def send_and_wait(self, data, phase=None): From 468e665c67cf8fef21a452aba9ef0ed9e5fe8d4f Mon Sep 17 00:00:00 2001 From: moshe Date: Mon, 29 Oct 2018 15:48:08 +0200 Subject: [PATCH 072/209] some debugging --- tests/base_test_agent.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index efb3975..a6afbc9 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -67,6 +67,7 @@ def act_on_request(self): def send(self, data): data = data.encode('utf-8') self.publisher_upstream.publish(self.topic_path_upstream, data=data, phase=self.phase) + logging.INFO("sent " + data) def get_node_id(self): time.sleep(15) From c92542fff013e739b6cd8bbc52bee855f2cee138 Mon Sep 17 00:00:00 2001 From: moshe Date: Mon, 29 Oct 2018 16:05:37 +0200 Subject: [PATCH 073/209] no use --- tests/test_2.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_2.py b/tests/test_2.py index 2b9d9fe..bd592d8 100644 --- a/tests/test_2.py +++ b/tests/test_2.py @@ -10,7 +10,6 @@ class Test2(BaseTest): def test_sendId(self): phase_0 = self.create_phase(2, 0) - seeds = [] for i in range(0, 3): self.start_node_agent_pair() self.send('GET_NODE_ID') @@ -20,7 +19,6 @@ def test_sendId(self): for i in range(0, 3): self.assertNotEqual(b'NULL', self.messages[i]) self.assertLess(15, len(self.messages[i])) - seeds.append(self.messages[i]) #'["0.0.0.0:7517/j7qWfWaJRVp25ZsnCu9rJ4PmhigZBtesB4YmQHqqPvtR"]' like phase_1 = self.create_phase(2, 1) From 81d8e95df62369a2a14943e72085833a9c4e926c Mon Sep 17 00:00:00 2001 From: moshe Date: Mon, 29 Oct 2018 23:10:27 +0200 Subject: [PATCH 074/209] add filter based phase on ci side --- tests/base_test_ci.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index b96742b..2949b04 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -34,6 +34,8 @@ def tearDown(self): self.send('END') def callback(self, message): + if self.phase != message.attributes['phase']: + return logging.info(message) self.messages.append(message.data.decode("utf-8")) message.ack() From b3464cf035068b966b70cf688897794593661fc1 Mon Sep 17 00:00:00 2001 From: moshe Date: Mon, 29 Oct 2018 23:20:12 +0200 Subject: [PATCH 075/209] change order --- tests/base_test_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index a6afbc9..d77d050 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -65,9 +65,9 @@ def act_on_request(self): time.sleep(1) def send(self, data): + logging.INFO("sent " + data) data = data.encode('utf-8') self.publisher_upstream.publish(self.topic_path_upstream, data=data, phase=self.phase) - logging.INFO("sent " + data) def get_node_id(self): time.sleep(15) From d3ad6f9dd1b9c964f5fc59933b489b8638c8ee72 Mon Sep 17 00:00:00 2001 From: moshe Date: Mon, 29 Oct 2018 23:22:01 +0200 Subject: [PATCH 076/209] casting --- tests/base_test_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index d77d050..e1793f9 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -65,7 +65,7 @@ def act_on_request(self): time.sleep(1) def send(self, data): - logging.INFO("sent " + data) + logging.INFO("sent " + str(data)) data = data.encode('utf-8') self.publisher_upstream.publish(self.topic_path_upstream, data=data, phase=self.phase) From 865caa1527a5068d4a267338715bf3247922b5a7 Mon Sep 17 00:00:00 2001 From: moshe Date: Mon, 29 Oct 2018 23:24:28 +0200 Subject: [PATCH 077/209] send --- tests/base_test_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index e1793f9..92da935 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -65,7 +65,7 @@ def act_on_request(self): time.sleep(1) def send(self, data): - logging.INFO("sent " + str(data)) + logging.INFO("sent") data = data.encode('utf-8') self.publisher_upstream.publish(self.topic_path_upstream, data=data, phase=self.phase) From 761df68c4a815663be6d3a1a67b17868a6636b30 Mon Sep 17 00:00:00 2001 From: moshe Date: Mon, 29 Oct 2018 23:32:25 +0200 Subject: [PATCH 078/209] fix typo --- tests/base_test_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index 92da935..01d202d 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -65,7 +65,7 @@ def act_on_request(self): time.sleep(1) def send(self, data): - logging.INFO("sent") + logging.info("sent: " + data) data = data.encode('utf-8') self.publisher_upstream.publish(self.topic_path_upstream, data=data, phase=self.phase) From 9dafe0fafeb9ff1f23b9bc8b377d4823b2478ecb Mon Sep 17 00:00:00 2001 From: moshe Date: Mon, 29 Oct 2018 23:36:16 +0200 Subject: [PATCH 079/209] avoid binary --- tests/base_test_agent.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index 01d202d..0900d19 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -41,22 +41,22 @@ def modify_seeders(self, seeders): def callback(self, message): logging.info(message) - self.message = message.data + self.message = "".join(map(chr, message.data)) message.ack() logging.info(message.attributes['phase']) if self.phase != message.attributes['phase']: logging.info("NO_MESSAGE") return - logging.info(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + " GOT_DOWN_MSG " + "".join(map(chr, self.message))) + logging.info(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + " GOT_DOWN_MSG " + self.message) - if b'END' == self.message: + if 'END' == self.message: self.docker.stop('node_' + self.node) self.endFlag = True - elif b'SEND_UP' == self.message: + elif 'SEND_UP' == self.message: self.send('UP') - elif b'GET_NODE_ID' == self.message: + elif 'GET_NODE_ID' == self.message: self.send(self.get_node_id()) - elif b'SHUTDOWN_NODE' == self.message: + elif 'SHUTDOWN_NODE' == self.message: self.docker.stop('node_' + self.node) def act_on_request(self): From 44c9c23edde331f478787fdcb889a70d4d8f1869 Mon Sep 17 00:00:00 2001 From: moshe Date: Mon, 29 Oct 2018 23:42:15 +0200 Subject: [PATCH 080/209] wait a little bit more --- tests/base_test_ci.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index 2949b04..94f823c 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -15,7 +15,7 @@ def setUp(self): self.create_phase(0, 0) self.endFlag = False - self.testLen = 15 + self.testLen = 30 self.message = b'NULL' project = config.CONFIG['project'] subscription_name_upstream = config.CONFIG['subscription_name_upstream'] From 5fe0dbfa9f339e2ac6ab84ec63e74945a0df68dd Mon Sep 17 00:00:00 2001 From: moshe Date: Mon, 29 Oct 2018 23:52:36 +0200 Subject: [PATCH 081/209] Reduce time to get node --- tests/base_test_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index 0900d19..3b063e4 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -70,7 +70,7 @@ def send(self, data): self.publisher_upstream.publish(self.topic_path_upstream, data=data, phase=self.phase) def get_node_id(self): - time.sleep(15) + time.sleep(10) try: node_id = next(os.walk('/opt/logs'))[1][0] except Exception as e: From 70b33c17cc8d821e93f66b3f6a02a28f5ba9472e Mon Sep 17 00:00:00 2001 From: moshe Date: Mon, 29 Oct 2018 23:57:23 +0200 Subject: [PATCH 082/209] add check for node_id --- tests/test_2.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_2.py b/tests/test_2.py index bd592d8..62ce702 100644 --- a/tests/test_2.py +++ b/tests/test_2.py @@ -30,6 +30,7 @@ def test_sendId(self): for i in range(0, 3): self.start_node_agent_pair(seeders_str) + self.send('GET_NODE_ID') self.wait_for_response(3) logging.info(self.messages) From 73f445388a1086b870a8e3202dd956b8f7269963 Mon Sep 17 00:00:00 2001 From: moshe Date: Tue, 30 Oct 2018 00:24:32 +0200 Subject: [PATCH 083/209] place conf in dedicated file --- tests/base_test_agent.py | 2 +- tests/base_test_ci.py | 2 +- tests/test_2.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index 3b063e4..4ad92ae 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -24,7 +24,7 @@ def __init__(self): logging.info('seeders:' + os.environ['SEEDERS']) self.modify_seeders(os.environ['SEEDERS']) self.phase = os.environ['PHASE'] - self.docker.start('docker run --network=devnet --name node_' + self.node + ' -p ' + str(7513 + int(self.node)) + ':7513 -v /root/spacemesh/devnet/logs' + self.node + ':/root/.spacemesh/nodes/ -v /root/spacemesh/devnet/tests/test.config.toml:/go/test.config.toml spacemesh/node:latest /go/src/github.com/spacemeshos/go-spacemesh/go-spacemesh -config /go/test.config.toml') + self.docker.start('docker run --network=devnet --name node_' + self.node + ' -p ' + str(7513 + int(self.node)) + ':7513 -v /root/spacemesh/devnet/logs' + self.node + ':/root/.spacemesh/nodes/ -v /root/spacemesh/devnet/cnf' + str(self.agents) + '/test.config.toml:/go/test.config.toml spacemesh/node:latest /go/src/github.com/spacemeshos/go-spacemesh/go-spacemesh -config /go/test.config.toml') subscription_name_downstream = os.environ['SUBSCRIPTION_NAME_DOWNSTREAM'] self.subscriber_downstream = pubsub_v1.SubscriberClient() self.subscription_path_downstream = self.subscriber_downstream.subscription_path(project, subscription_name_downstream) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index 94f823c..621a538 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -67,7 +67,7 @@ def start_node_agent_pair(self, seeders=config.CONFIG['no_seeders'], phase=None) phase = self.phase docker = Docker() docker.stop('agent_' + str(self.agents)) - docker.start('docker run --network=devnet --name agent_' + str(self.agents) + ' -v /root/spacemesh/devnet/tests:/opt/devnet -v /root/spacemesh/devnet/logs' + str(self.agents) + ':/opt/logs -e SUBSCRIPTION_NAME_DOWNSTREAM=devnet_tests_agent_' + str(self.agents) + ' -e PHASE=' + phase + ' -e NODE=' + str(self.agents) + ' -e SEEDERS=' + seeders + ' spacemesh/devnet_agent:latest python3 /opt/devnet/base_test_agent.py') + docker.start('docker run --network=devnet --name agent_' + str(self.agents) + ' -v /root/spacemesh/devnet/tests:/opt/devnet -v /root/spacemesh/devnet/logs' + str(self.agents) + ':/opt/logs -v /root/spacemesh/devnet/cnf' + str(self.agents) + '/test.config.toml:/opt/devnet/test.config.toml -e SUBSCRIPTION_NAME_DOWNSTREAM=devnet_tests_agent_' + str(self.agents) + ' -e PHASE=' + phase + ' -e NODE=' + str(self.agents) + ' -e SEEDERS=' + seeders + ' spacemesh/devnet_agent:latest python3 /opt/devnet/base_test_agent.py') self.agents += 1 if __name__ == '__main__': diff --git a/tests/test_2.py b/tests/test_2.py index 62ce702..dbd158d 100644 --- a/tests/test_2.py +++ b/tests/test_2.py @@ -29,7 +29,7 @@ def test_sendId(self): logging.info(self.messages) for i in range(0, 3): - self.start_node_agent_pair(seeders_str) + self.start_node_agent_pair(seeders=seeders_str) self.send('GET_NODE_ID') self.wait_for_response(3) From fe024eb0193fc3bd0be08cce2c8d157002187308 Mon Sep 17 00:00:00 2001 From: moshe Date: Tue, 30 Oct 2018 17:25:24 +0200 Subject: [PATCH 084/209] Turn the config file to be test independent --- agent/Dockerfile | 2 +- tests/base_test_agent.py | 5 +++-- tests/base_test_ci.py | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/agent/Dockerfile b/agent/Dockerfile index eb45791..06124df 100644 --- a/agent/Dockerfile +++ b/agent/Dockerfile @@ -4,5 +4,5 @@ RUN apt-get -y update && apt-get install -y python3 python3-pip RUN pip3 install --upgrade pip RUN pip3 install google-cloud google-cloud-pubsub spur RUN mkdir /opt/devnet -ADD ./tests/test.config.toml /opt/devnet/ +ADD ./tests/test.config.toml /opt/basecnf/ WORKDIR /opt/devnet \ No newline at end of file diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index 4ad92ae..01f5b22 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -31,12 +31,13 @@ def __init__(self): time.sleep(5) def modify_seeders(self, seeders): - file_name = "/opt/devnet/test.config.toml" + file_name = "/opt/basecnf/test.config.toml" + out_file_name = "/opt/cnf/test.config.toml" with open(file_name) as f: new_config = f.read().replace('BOOT_NODES', 'bootnodes = ' + seeders) - with open(file_name, "w") as f: + with open(out_file_name, "w") as f: f.write(new_config) def callback(self, message): diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index 621a538..09f6f3a 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -67,7 +67,7 @@ def start_node_agent_pair(self, seeders=config.CONFIG['no_seeders'], phase=None) phase = self.phase docker = Docker() docker.stop('agent_' + str(self.agents)) - docker.start('docker run --network=devnet --name agent_' + str(self.agents) + ' -v /root/spacemesh/devnet/tests:/opt/devnet -v /root/spacemesh/devnet/logs' + str(self.agents) + ':/opt/logs -v /root/spacemesh/devnet/cnf' + str(self.agents) + '/test.config.toml:/opt/devnet/test.config.toml -e SUBSCRIPTION_NAME_DOWNSTREAM=devnet_tests_agent_' + str(self.agents) + ' -e PHASE=' + phase + ' -e NODE=' + str(self.agents) + ' -e SEEDERS=' + seeders + ' spacemesh/devnet_agent:latest python3 /opt/devnet/base_test_agent.py') + docker.start('docker run --network=devnet --name agent_' + str(self.agents) + ' -v /root/spacemesh/devnet/tests:/opt/devnet -v /root/spacemesh/devnet/logs' + str(self.agents) + ':/opt/logs -v /root/spacemesh/devnet/cnf' + str(self.agents) + ':/opt/cnf/ -e SUBSCRIPTION_NAME_DOWNSTREAM=devnet_tests_agent_' + str(self.agents) + ' -e PHASE=' + phase + ' -e NODE=' + str(self.agents) + ' -e SEEDERS=' + seeders + ' spacemesh/devnet_agent:latest python3 /opt/devnet/base_test_agent.py') self.agents += 1 if __name__ == '__main__': From e8e1d9e23f4813c57d3412804fae1655181aeaf3 Mon Sep 17 00:00:00 2001 From: moshe Date: Tue, 30 Oct 2018 17:36:36 +0200 Subject: [PATCH 085/209] improve the logging --- tests/base_test_ci.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index 09f6f3a..d1b4746 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -67,7 +67,9 @@ def start_node_agent_pair(self, seeders=config.CONFIG['no_seeders'], phase=None) phase = self.phase docker = Docker() docker.stop('agent_' + str(self.agents)) - docker.start('docker run --network=devnet --name agent_' + str(self.agents) + ' -v /root/spacemesh/devnet/tests:/opt/devnet -v /root/spacemesh/devnet/logs' + str(self.agents) + ':/opt/logs -v /root/spacemesh/devnet/cnf' + str(self.agents) + ':/opt/cnf/ -e SUBSCRIPTION_NAME_DOWNSTREAM=devnet_tests_agent_' + str(self.agents) + ' -e PHASE=' + phase + ' -e NODE=' + str(self.agents) + ' -e SEEDERS=' + seeders + ' spacemesh/devnet_agent:latest python3 /opt/devnet/base_test_agent.py') + cmd = 'docker run --network=devnet --name agent_' + str(self.agents) + ' -v /root/spacemesh/devnet/tests:/opt/devnet -v /root/spacemesh/devnet/logs' + str(self.agents) + ':/opt/logs -v /root/spacemesh/devnet/cnf' + str(self.agents) + ':/opt/cnf/ -e SUBSCRIPTION_NAME_DOWNSTREAM=devnet_tests_agent_' + str(self.agents) + ' -e PHASE=' + phase + ' -e NODE=' + str(self.agents) + ' -e SEEDERS=' + seeders + ' spacemesh/devnet_agent:latest python3 /opt/devnet/base_test_agent.py' + logging.info(cmd) + docker.start(cmd) self.agents += 1 if __name__ == '__main__': From 35d34e1b01d196b04aed912f43cc3fe6db971391 Mon Sep 17 00:00:00 2001 From: moshe Date: Tue, 30 Oct 2018 17:40:37 +0200 Subject: [PATCH 086/209] fix the cnf node id --- tests/base_test_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index 01f5b22..e395a13 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -24,7 +24,7 @@ def __init__(self): logging.info('seeders:' + os.environ['SEEDERS']) self.modify_seeders(os.environ['SEEDERS']) self.phase = os.environ['PHASE'] - self.docker.start('docker run --network=devnet --name node_' + self.node + ' -p ' + str(7513 + int(self.node)) + ':7513 -v /root/spacemesh/devnet/logs' + self.node + ':/root/.spacemesh/nodes/ -v /root/spacemesh/devnet/cnf' + str(self.agents) + '/test.config.toml:/go/test.config.toml spacemesh/node:latest /go/src/github.com/spacemeshos/go-spacemesh/go-spacemesh -config /go/test.config.toml') + self.docker.start('docker run --network=devnet --name node_' + self.node + ' -p ' + str(7513 + int(self.node)) + ':7513 -v /root/spacemesh/devnet/logs' + self.node + ':/root/.spacemesh/nodes/ -v /root/spacemesh/devnet/cnf' + self.node + '/test.config.toml:/go/test.config.toml spacemesh/node:latest /go/src/github.com/spacemeshos/go-spacemesh/go-spacemesh -config /go/test.config.toml') subscription_name_downstream = os.environ['SUBSCRIPTION_NAME_DOWNSTREAM'] self.subscriber_downstream = pubsub_v1.SubscriberClient() self.subscription_path_downstream = self.subscriber_downstream.subscription_path(project, subscription_name_downstream) From f8a02a9b06ebc36a066cc8253291b02078ea48b4 Mon Sep 17 00:00:00 2001 From: moshe Date: Tue, 30 Oct 2018 18:04:32 +0200 Subject: [PATCH 087/209] init logging in the docker as well --- tests/dockers.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/dockers.py b/tests/dockers.py index 094fe62..1c5afc3 100644 --- a/tests/dockers.py +++ b/tests/dockers.py @@ -1,8 +1,12 @@ import config import spur import logging +from logging import Logger class Docker(): + def __init__: + logging.basicConfig(format='%(asctime)s %(message)s', level=logging.INFO) + def start(self, cmd): try: logging.debug(cmd) From 8caf570da5dbf84243be4d241b2c0f4f1cea3d46 Mon Sep 17 00:00:00 2001 From: moshe Date: Tue, 30 Oct 2018 18:07:07 +0200 Subject: [PATCH 088/209] typo --- tests/dockers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/dockers.py b/tests/dockers.py index 1c5afc3..c97f37b 100644 --- a/tests/dockers.py +++ b/tests/dockers.py @@ -4,7 +4,7 @@ from logging import Logger class Docker(): - def __init__: + def __init__(self): logging.basicConfig(format='%(asctime)s %(message)s', level=logging.INFO) def start(self, cmd): From d5ed5e07acc6ec87f550c4e4d9c1e3f27e9db2af Mon Sep 17 00:00:00 2001 From: moshe Date: Tue, 30 Oct 2018 18:11:45 +0200 Subject: [PATCH 089/209] fix the logging --- tests/dockers.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/dockers.py b/tests/dockers.py index c97f37b..e404e02 100644 --- a/tests/dockers.py +++ b/tests/dockers.py @@ -9,10 +9,10 @@ def __init__(self): def start(self, cmd): try: - logging.debug(cmd) - logging.debug(config.CONFIG['host']) - logging.debug(config.CONFIG['host_user']) - logging.debug(config.CONFIG['host_password']) + logging.info(cmd) + logging.info(config.CONFIG['host']) + logging.info(config.CONFIG['host_user']) + logging.info(config.CONFIG['host_password']) shell = spur.SshShell( hostname=config.CONFIG['host'], username=config.CONFIG['host_user'], @@ -21,7 +21,7 @@ def start(self, cmd): ) with shell: result = shell.spawn(cmd.split(' ')) - logging.debug('Docker started') + logging.info('Docker started') except Exception as e: logging.warning('Docker start failed') logging.warning(e.__doc__ ) @@ -36,7 +36,7 @@ def run_cmd_interactive(self, cmd): ) with shell: result = shell.run(cmd.split(' ')) - logging.debug('Run interactive: ' + cmd) + logging.info('Run interactive: ' + cmd) except Exception as e: logging.warning('Run interactive failed: ' + cmd) logging.warning(e.__doc__ ) From 5810b7d5597413ca1e4d98703eac12b095f10021 Mon Sep 17 00:00:00 2001 From: moshe Date: Tue, 30 Oct 2018 18:29:23 +0200 Subject: [PATCH 090/209] debug w/ interactive --- tests/base_test_ci.py | 1 - tests/dockers.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index d1b4746..c891a2d 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -68,7 +68,6 @@ def start_node_agent_pair(self, seeders=config.CONFIG['no_seeders'], phase=None) docker = Docker() docker.stop('agent_' + str(self.agents)) cmd = 'docker run --network=devnet --name agent_' + str(self.agents) + ' -v /root/spacemesh/devnet/tests:/opt/devnet -v /root/spacemesh/devnet/logs' + str(self.agents) + ':/opt/logs -v /root/spacemesh/devnet/cnf' + str(self.agents) + ':/opt/cnf/ -e SUBSCRIPTION_NAME_DOWNSTREAM=devnet_tests_agent_' + str(self.agents) + ' -e PHASE=' + phase + ' -e NODE=' + str(self.agents) + ' -e SEEDERS=' + seeders + ' spacemesh/devnet_agent:latest python3 /opt/devnet/base_test_agent.py' - logging.info(cmd) docker.start(cmd) self.agents += 1 diff --git a/tests/dockers.py b/tests/dockers.py index e404e02..ce5a125 100644 --- a/tests/dockers.py +++ b/tests/dockers.py @@ -20,7 +20,7 @@ def start(self, cmd): missing_host_key=spur.ssh.MissingHostKey.accept ) with shell: - result = shell.spawn(cmd.split(' ')) + result = shell.run(cmd.split(' ')) logging.info('Docker started') except Exception as e: logging.warning('Docker start failed') From 44288d36e2262644daa2c44e4426834e6c178367 Mon Sep 17 00:00:00 2001 From: moshe Date: Tue, 30 Oct 2018 18:48:28 +0200 Subject: [PATCH 091/209] bask --- tests/dockers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/dockers.py b/tests/dockers.py index ce5a125..e404e02 100644 --- a/tests/dockers.py +++ b/tests/dockers.py @@ -20,7 +20,7 @@ def start(self, cmd): missing_host_key=spur.ssh.MissingHostKey.accept ) with shell: - result = shell.run(cmd.split(' ')) + result = shell.spawn(cmd.split(' ')) logging.info('Docker started') except Exception as e: logging.warning('Docker start failed') From 53745764e6261f9b712fd150d08bddf98abda6a2 Mon Sep 17 00:00:00 2001 From: moshe Date: Tue, 30 Oct 2018 18:59:13 +0200 Subject: [PATCH 092/209] refactor --- tests/dockers.py | 31 +++++++++++-------------------- 1 file changed, 11 insertions(+), 20 deletions(-) diff --git a/tests/dockers.py b/tests/dockers.py index e404e02..a519a09 100644 --- a/tests/dockers.py +++ b/tests/dockers.py @@ -7,7 +7,7 @@ class Docker(): def __init__(self): logging.basicConfig(format='%(asctime)s %(message)s', level=logging.INFO) - def start(self, cmd): + def run_cmd(self, cmd, interactive = True): try: logging.info(cmd) logging.info(config.CONFIG['host']) @@ -20,27 +20,18 @@ def start(self, cmd): missing_host_key=spur.ssh.MissingHostKey.accept ) with shell: - result = shell.spawn(cmd.split(' ')) - logging.info('Docker started') + if interactive: + result = shell.run(cmd.split(' ')) + else: + result = shell.spawn(cmd.split(' ')) + logging.info('Run: ' + cmd) except Exception as e: - logging.warning('Docker start failed') + logging.warning('Run failed: ' + cmd) logging.warning(e.__doc__ ) - def run_cmd_interactive(self, cmd): - try: - shell = spur.SshShell( - hostname=config.CONFIG['host'], - username=config.CONFIG['host_user'], - password=config.CONFIG['host_password'], - missing_host_key=spur.ssh.MissingHostKey.accept - ) - with shell: - result = shell.run(cmd.split(' ')) - logging.info('Run interactive: ' + cmd) - except Exception as e: - logging.warning('Run interactive failed: ' + cmd) - logging.warning(e.__doc__ ) + def start(self, cmd): + run_cmd(self, cmd, interactive = False): def stop(self, name): - self.run_cmd_interactive("docker stop " + name) - self.run_cmd_interactive("docker rm " + name) \ No newline at end of file + self.run_cmd("docker stop " + name) + self.run_cmd("docker rm " + name) \ No newline at end of file From 1723e4338c305a610ea0ce477d37ce8ecbb6a5b4 Mon Sep 17 00:00:00 2001 From: moshe Date: Tue, 30 Oct 2018 19:00:35 +0200 Subject: [PATCH 093/209] typo --- tests/dockers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/dockers.py b/tests/dockers.py index a519a09..3cacbec 100644 --- a/tests/dockers.py +++ b/tests/dockers.py @@ -30,7 +30,7 @@ def run_cmd(self, cmd, interactive = True): logging.warning(e.__doc__ ) def start(self, cmd): - run_cmd(self, cmd, interactive = False): + run_cmd(self, cmd, interactive = False) def stop(self, name): self.run_cmd("docker stop " + name) From da96c5a62c83d38e3b79f6b16f624b736cc853e5 Mon Sep 17 00:00:00 2001 From: moshe Date: Tue, 30 Oct 2018 19:04:52 +0200 Subject: [PATCH 094/209] typo --- tests/dockers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/dockers.py b/tests/dockers.py index 3cacbec..8c57d8c 100644 --- a/tests/dockers.py +++ b/tests/dockers.py @@ -30,7 +30,7 @@ def run_cmd(self, cmd, interactive = True): logging.warning(e.__doc__ ) def start(self, cmd): - run_cmd(self, cmd, interactive = False) + self.run_cmd(self, cmd, interactive = False) def stop(self, name): self.run_cmd("docker stop " + name) From 15e04aa4297797e2c4620d81ec33d98eae756a72 Mon Sep 17 00:00:00 2001 From: moshe Date: Tue, 30 Oct 2018 19:06:35 +0200 Subject: [PATCH 095/209] typo? --- tests/dockers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/dockers.py b/tests/dockers.py index 8c57d8c..db9fe6b 100644 --- a/tests/dockers.py +++ b/tests/dockers.py @@ -30,7 +30,7 @@ def run_cmd(self, cmd, interactive = True): logging.warning(e.__doc__ ) def start(self, cmd): - self.run_cmd(self, cmd, interactive = False) + self.run_cmd(self, cmd, False) def stop(self, name): self.run_cmd("docker stop " + name) From 5ee0d7471a6501d1494e73d6736098c9516a8e6d Mon Sep 17 00:00:00 2001 From: moshe Date: Tue, 30 Oct 2018 19:07:41 +0200 Subject: [PATCH 096/209] typo --- tests/dockers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/dockers.py b/tests/dockers.py index db9fe6b..70976f6 100644 --- a/tests/dockers.py +++ b/tests/dockers.py @@ -30,7 +30,7 @@ def run_cmd(self, cmd, interactive = True): logging.warning(e.__doc__ ) def start(self, cmd): - self.run_cmd(self, cmd, False) + self.run_cmd(cmd, interactive = False) def stop(self, name): self.run_cmd("docker stop " + name) From 407b73a483c427e7001ecd089f1ad0ada310b1e9 Mon Sep 17 00:00:00 2001 From: moshe Date: Tue, 30 Oct 2018 19:27:31 +0200 Subject: [PATCH 097/209] make the relevant dirs --- agent/Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/agent/Dockerfile b/agent/Dockerfile index 06124df..f75eadd 100644 --- a/agent/Dockerfile +++ b/agent/Dockerfile @@ -3,6 +3,8 @@ FROM ubuntu:16.04 RUN apt-get -y update && apt-get install -y python3 python3-pip RUN pip3 install --upgrade pip RUN pip3 install google-cloud google-cloud-pubsub spur -RUN mkdir /opt/devnet +RUN mkdir -p /opt/devnet +RUN mkdir -p /opt/cnf/ +RUN mkdir -p /opt/basecnf/ ADD ./tests/test.config.toml /opt/basecnf/ WORKDIR /opt/devnet \ No newline at end of file From 8bff0225d600fbb09f056b7561fb342db7d92ebf Mon Sep 17 00:00:00 2001 From: moshe Date: Tue, 30 Oct 2018 22:28:07 +0200 Subject: [PATCH 098/209] adding another --- agent/Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/agent/Dockerfile b/agent/Dockerfile index f75eadd..903dc82 100644 --- a/agent/Dockerfile +++ b/agent/Dockerfile @@ -4,7 +4,8 @@ RUN apt-get -y update && apt-get install -y python3 python3-pip RUN pip3 install --upgrade pip RUN pip3 install google-cloud google-cloud-pubsub spur RUN mkdir -p /opt/devnet -RUN mkdir -p /opt/cnf/ -RUN mkdir -p /opt/basecnf/ +RUN mkdir -p /opt/cnf +RUN mkdir -p /opt/basecnf +RUN mkdir -p /opt/logs ADD ./tests/test.config.toml /opt/basecnf/ WORKDIR /opt/devnet \ No newline at end of file From d6ffcb0afc57604934bd7a5e2b8c1114e414a529 Mon Sep 17 00:00:00 2001 From: moshe Date: Tue, 30 Oct 2018 23:25:30 +0200 Subject: [PATCH 099/209] bug due to space? --- tests/base_test_ci.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index c891a2d..ee6aec0 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -67,7 +67,7 @@ def start_node_agent_pair(self, seeders=config.CONFIG['no_seeders'], phase=None) phase = self.phase docker = Docker() docker.stop('agent_' + str(self.agents)) - cmd = 'docker run --network=devnet --name agent_' + str(self.agents) + ' -v /root/spacemesh/devnet/tests:/opt/devnet -v /root/spacemesh/devnet/logs' + str(self.agents) + ':/opt/logs -v /root/spacemesh/devnet/cnf' + str(self.agents) + ':/opt/cnf/ -e SUBSCRIPTION_NAME_DOWNSTREAM=devnet_tests_agent_' + str(self.agents) + ' -e PHASE=' + phase + ' -e NODE=' + str(self.agents) + ' -e SEEDERS=' + seeders + ' spacemesh/devnet_agent:latest python3 /opt/devnet/base_test_agent.py' + cmd = 'docker run --network=devnet --name agent_' + str(self.agents) + ' -v /root/spacemesh/devnet/tests:/opt/devnet -v /root/spacemesh/devnet/logs' + str(self.agents) + ':/opt/logs -v /root/spacemesh/devnet/cnf' + str(self.agents) + ':/opt/cnf/ -e SUBSCRIPTION_NAME_DOWNSTREAM=devnet_tests_agent_' + str(self.agents) + ' -e PHASE=' + phase + ' -e NODE=' + str(self.agents) + ' -e SEEDERS=' + seeders + ' spacemesh/devnet_agent:latest python3 /opt/devnet/base_test_agent.py' docker.start(cmd) self.agents += 1 From 389d0d13854cc857bbab71946d055fbf51ef4918 Mon Sep 17 00:00:00 2001 From: moshe Date: Wed, 31 Oct 2018 01:05:52 +0200 Subject: [PATCH 100/209] moving node port to const --- tests/base_test_agent.py | 5 +++-- tests/config.py | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index e395a13..42c8606 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -24,7 +24,8 @@ def __init__(self): logging.info('seeders:' + os.environ['SEEDERS']) self.modify_seeders(os.environ['SEEDERS']) self.phase = os.environ['PHASE'] - self.docker.start('docker run --network=devnet --name node_' + self.node + ' -p ' + str(7513 + int(self.node)) + ':7513 -v /root/spacemesh/devnet/logs' + self.node + ':/root/.spacemesh/nodes/ -v /root/spacemesh/devnet/cnf' + self.node + '/test.config.toml:/go/test.config.toml spacemesh/node:latest /go/src/github.com/spacemeshos/go-spacemesh/go-spacemesh -config /go/test.config.toml') + self.node_port = config.CONFIG['node_port'] + self.docker.start('docker run --network=devnet --name node_' + self.node + ' -p ' + str(self.node_port + int(self.node)) + ':' + str(self.node_port) + ' -v /root/spacemesh/devnet/logs' + self.node + ':/root/.spacemesh/nodes/ -v /root/spacemesh/devnet/cnf' + self.node + '/test.config.toml:/go/test.config.toml spacemesh/node:latest /go/src/github.com/spacemeshos/go-spacemesh/go-spacemesh -config /go/test.config.toml') subscription_name_downstream = os.environ['SUBSCRIPTION_NAME_DOWNSTREAM'] self.subscriber_downstream = pubsub_v1.SubscriberClient() self.subscription_path_downstream = self.subscriber_downstream.subscription_path(project, subscription_name_downstream) @@ -80,7 +81,7 @@ def get_node_id(self): return 'NULL' logging.info('NodeId:' + node_id) - return 'node_' + self.node + ':7513/' + node_id + return 'node_' + self.node + ':' + str(self.node_port) + '/' + node_id if __name__ == '__main__': t = BaseDevnetAgent() diff --git a/tests/config.py b/tests/config.py index 7e49e36..de427b4 100644 --- a/tests/config.py +++ b/tests/config.py @@ -7,5 +7,6 @@ 'host': '127.0.0.1', 'host_user': 'deploy', 'host_password': 'deploy_password', - 'no_seeders': '["0.0.0.0:7517/j7qWfWaJRVp25ZsnCu9rJ4PmhigZBtesB4YmQHqqPvtR"]' + 'no_seeders': '["0.0.0.0:7517/j7qWfWaJRVp25ZsnCu9rJ4PmhigZBtesB4YmQHqqPvtR"]', + 'node_port': 7513 } \ No newline at end of file From ed17d371b579f5713e0cc5b1e2161ea895e8125a Mon Sep 17 00:00:00 2001 From: moshe Date: Wed, 31 Oct 2018 01:15:44 +0200 Subject: [PATCH 101/209] duplication --- tests/base_test_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index 42c8606..df213cf 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -49,7 +49,7 @@ def callback(self, message): if self.phase != message.attributes['phase']: logging.info("NO_MESSAGE") return - logging.info(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + " GOT_DOWN_MSG " + self.message) + logging.info("GOT_DOWN_MSG " + self.message) if 'END' == self.message: self.docker.stop('node_' + self.node) From 6c6f07a715b658837e47c5f7ed0e6bb76b203556 Mon Sep 17 00:00:00 2001 From: moshe Date: Mon, 5 Nov 2018 14:25:05 +0200 Subject: [PATCH 102/209] 1. Refactor the agent init to create node and links in seperated functions 2. Wait to node to complete start instead of sleeping for defined time 3. Use defailt phase for now --- tests/base_test_agent.py | 29 ++++++++++++++++++----------- tests/base_test_ci.py | 18 +++++++----------- tests/test_2.py | 4 ++-- 3 files changed, 27 insertions(+), 24 deletions(-) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index df213cf..ec08745 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -12,24 +12,32 @@ class BaseDevnetAgent: def __init__(self): logging.basicConfig(format='%(asctime)s %(message)s', level=logging.INFO) self.endFlag = False - + + self.node = os.environ['NODE'] + self.phase = os.environ['PHASE'] + self.node_port = config.CONFIG['node_port'] + self.docker = Docker() + self.start_node() + self.establish_links() + + def establish_links(): project = config.CONFIG['project'] topic_name_upstream = config.CONFIG['topic_name_upstream'] self.publisher_upstream = pubsub_v1.PublisherClient() self.topic_path_upstream = self.publisher_upstream.topic_path(project, topic_name_upstream) - self.node = os.environ['NODE'] - self.docker = Docker() - self.docker.stop('node_' + self.node) - logging.info('seeders:' + os.environ['SEEDERS']) - self.modify_seeders(os.environ['SEEDERS']) - self.phase = os.environ['PHASE'] - self.node_port = config.CONFIG['node_port'] - self.docker.start('docker run --network=devnet --name node_' + self.node + ' -p ' + str(self.node_port + int(self.node)) + ':' + str(self.node_port) + ' -v /root/spacemesh/devnet/logs' + self.node + ':/root/.spacemesh/nodes/ -v /root/spacemesh/devnet/cnf' + self.node + '/test.config.toml:/go/test.config.toml spacemesh/node:latest /go/src/github.com/spacemeshos/go-spacemesh/go-spacemesh -config /go/test.config.toml') subscription_name_downstream = os.environ['SUBSCRIPTION_NAME_DOWNSTREAM'] self.subscriber_downstream = pubsub_v1.SubscriberClient() self.subscription_path_downstream = self.subscriber_downstream.subscription_path(project, subscription_name_downstream) - time.sleep(5) + + def start_node(self): + logging.info('seeders:' + os.environ['SEEDERS']) + self.modify_seeders(os.environ['SEEDERS']) + self.docker.stop('node_' + self.node) + self.docker.start('docker run --network=devnet --name node_' + self.node + ' -p ' + str(self.node_port + int(self.node)) + ':' + str(self.node_port) + ' -v /root/spacemesh/devnet/logs' + self.node + ':/root/.spacemesh/nodes/ -v /root/spacemesh/devnet/cnf' + self.node + '/test.config.toml:/go/test.config.toml spacemesh/node:latest /go/src/github.com/spacemeshos/go-spacemesh/go-spacemesh -config /go/test.config.toml') + + while self.get_node_id() != "NULL": + time.sleep(1) def modify_seeders(self, seeders): file_name = "/opt/basecnf/test.config.toml" @@ -72,7 +80,6 @@ def send(self, data): self.publisher_upstream.publish(self.topic_path_upstream, data=data, phase=self.phase) def get_node_id(self): - time.sleep(10) try: node_id = next(os.walk('/opt/logs'))[1][0] except Exception as e: diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index ee6aec0..dc4989f 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -13,7 +13,7 @@ class BaseTest(unittest.TestCase): def setUp(self): logging.basicConfig(format='%(asctime)s %(message)s', level=logging.INFO) - self.create_phase(0, 0) + self.create_phase() self.endFlag = False self.testLen = 30 self.message = b'NULL' @@ -40,16 +40,14 @@ def callback(self, message): self.messages.append(message.data.decode("utf-8")) message.ack() - def create_phase(self, test, phase): - self.phase = 'test_' + str(test) + '_phase_' + str(phase) + '_' + str(calendar.timegm(time.gmtime())) + def create_phase(self, phase = ""): + self.phase = 'phase_' + str(phase) + '_' + str(calendar.timegm(time.gmtime())) return self.phase - def send(self, data, phase=None): - if phase == None: - phase = self.phase + def send(self, data): logging.info(data) data = data.encode('utf-8') - self.publisher_downstream.publish(self.topic_path_downstream, data=data, phase=phase) + self.publisher_downstream.publish(self.topic_path_downstream, data=data, phase=self.phase) def wait_for_response(self, num_messages = 1): for i in range(0, self.testLen): @@ -62,12 +60,10 @@ def send_and_wait(self, data, phase=None): self.send(data, phase) self.wait_for_response() - def start_node_agent_pair(self, seeders=config.CONFIG['no_seeders'], phase=None): - if phase == None: - phase = self.phase + def start_node_agent_pair(self, seeders=config.CONFIG['no_seeders']): docker = Docker() docker.stop('agent_' + str(self.agents)) - cmd = 'docker run --network=devnet --name agent_' + str(self.agents) + ' -v /root/spacemesh/devnet/tests:/opt/devnet -v /root/spacemesh/devnet/logs' + str(self.agents) + ':/opt/logs -v /root/spacemesh/devnet/cnf' + str(self.agents) + ':/opt/cnf/ -e SUBSCRIPTION_NAME_DOWNSTREAM=devnet_tests_agent_' + str(self.agents) + ' -e PHASE=' + phase + ' -e NODE=' + str(self.agents) + ' -e SEEDERS=' + seeders + ' spacemesh/devnet_agent:latest python3 /opt/devnet/base_test_agent.py' + cmd = 'docker run --network=devnet --name agent_' + str(self.agents) + ' -v /root/spacemesh/devnet/tests:/opt/devnet -v /root/spacemesh/devnet/logs' + str(self.agents) + ':/opt/logs -v /root/spacemesh/devnet/cnf' + str(self.agents) + ':/opt/cnf/ -e SUBSCRIPTION_NAME_DOWNSTREAM=devnet_tests_agent_' + str(self.agents) + ' -e PHASE=' + self.phase + ' -e NODE=' + str(self.agents) + ' -e SEEDERS=' + seeders + ' spacemesh/devnet_agent:latest python3 /opt/devnet/base_test_agent.py' docker.start(cmd) self.agents += 1 diff --git a/tests/test_2.py b/tests/test_2.py index dbd158d..92f5d00 100644 --- a/tests/test_2.py +++ b/tests/test_2.py @@ -9,7 +9,7 @@ class Test2(BaseTest): def test_sendId(self): - phase_0 = self.create_phase(2, 0) + phase_0 = self.create_phase("2.0") for i in range(0, 3): self.start_node_agent_pair() self.send('GET_NODE_ID') @@ -21,7 +21,7 @@ def test_sendId(self): self.assertLess(15, len(self.messages[i])) #'["0.0.0.0:7517/j7qWfWaJRVp25ZsnCu9rJ4PmhigZBtesB4YmQHqqPvtR"]' like - phase_1 = self.create_phase(2, 1) + phase_1 = self.create_phase("2.1") seeders_str = '\'["' + '","'.join(self.messages) + '"]\'' logging.info(seeders_str) logging.info(self.messages) From b070c045639954d0be6237abc02d18bf7f3a29b0 Mon Sep 17 00:00:00 2001 From: moshe Date: Mon, 5 Nov 2018 14:43:18 +0200 Subject: [PATCH 103/209] typo --- tests/base_test_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index ec08745..ab79b28 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -36,7 +36,7 @@ def start_node(self): self.docker.stop('node_' + self.node) self.docker.start('docker run --network=devnet --name node_' + self.node + ' -p ' + str(self.node_port + int(self.node)) + ':' + str(self.node_port) + ' -v /root/spacemesh/devnet/logs' + self.node + ':/root/.spacemesh/nodes/ -v /root/spacemesh/devnet/cnf' + self.node + '/test.config.toml:/go/test.config.toml spacemesh/node:latest /go/src/github.com/spacemeshos/go-spacemesh/go-spacemesh -config /go/test.config.toml') - while self.get_node_id() != "NULL": + while self.get_node_id() == 'NULL': time.sleep(1) def modify_seeders(self, seeders): From a1ff1949b22976d927db4d831cd9652ab828f365 Mon Sep 17 00:00:00 2001 From: moshe Date: Mon, 5 Nov 2018 14:44:54 +0200 Subject: [PATCH 104/209] typo --- tests/base_test_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index ab79b28..22903e5 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -20,7 +20,7 @@ def __init__(self): self.start_node() self.establish_links() - def establish_links(): + def establish_links(self): project = config.CONFIG['project'] topic_name_upstream = config.CONFIG['topic_name_upstream'] self.publisher_upstream = pubsub_v1.PublisherClient() From 473519733827154e5e8eae546d1af8bd7d3defdc Mon Sep 17 00:00:00 2001 From: moshe Date: Mon, 5 Nov 2018 14:51:16 +0200 Subject: [PATCH 105/209] 1. fix send 2. Return get out of test as soon as possible --- tests/base_test_ci.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index dc4989f..4ba2aef 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -51,13 +51,15 @@ def send(self, data): def wait_for_response(self, num_messages = 1): for i in range(0, self.testLen): -# if len(self.messages) == num_messages: -# logging.info(self.messages) -# return + logging.info(num_messages) + logging.info(self.messages) + if len(self.messages) == num_messages: + logging.info(self.messages) + return time.sleep(1) def send_and_wait(self, data, phase=None): - self.send(data, phase) + self.send(data) self.wait_for_response() def start_node_agent_pair(self, seeders=config.CONFIG['no_seeders']): From 2d01326745ab7161d1014a491bc80db9f0e9c5ea Mon Sep 17 00:00:00 2001 From: moshe Date: Mon, 5 Nov 2018 15:00:43 +0200 Subject: [PATCH 106/209] fix and debug slepp --- tests/base_test_ci.py | 4 ++-- tests/test_2.py | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index 4ba2aef..da5eedd 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -51,11 +51,11 @@ def send(self, data): def wait_for_response(self, num_messages = 1): for i in range(0, self.testLen): - logging.info(num_messages) + logging.info(len(self.messages)) logging.info(self.messages) if len(self.messages) == num_messages: logging.info(self.messages) - return + break time.sleep(1) def send_and_wait(self, data, phase=None): diff --git a/tests/test_2.py b/tests/test_2.py index 92f5d00..cbb2fe1 100644 --- a/tests/test_2.py +++ b/tests/test_2.py @@ -39,7 +39,9 @@ def test_sendId(self): self.assertNotEqual(b'NULL', self.messages[i]) self.assertLess(15, len(self.messages[i])) - self.send("END", phase_0) - self.send("END", phase_1) + self.phase = phase_0 + self.send("END") + self.phase = phase_1 + self.send("END") if __name__ == '__main__': unittest.main() \ No newline at end of file From 4c9337bfa27e45ddc040da8cd4f81cf406cc048f Mon Sep 17 00:00:00 2001 From: moshe Date: Mon, 5 Nov 2018 15:04:32 +0200 Subject: [PATCH 107/209] cleanup --- tests/base_test_ci.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index da5eedd..195dedf 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -51,8 +51,6 @@ def send(self, data): def wait_for_response(self, num_messages = 1): for i in range(0, self.testLen): - logging.info(len(self.messages)) - logging.info(self.messages) if len(self.messages) == num_messages: logging.info(self.messages) break From c9f0582075975832f8eb060c32b059c0a79611a5 Mon Sep 17 00:00:00 2001 From: moshe Date: Mon, 5 Nov 2018 15:34:15 +0200 Subject: [PATCH 108/209] Add the node specific branch --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index c40bfb9..fb231bd 100644 --- a/Makefile +++ b/Makefile @@ -55,7 +55,7 @@ build_agent_packed: @docker push gcr.io/spacemesh-198810/devnet_agent_packed build_node: - @wget https://raw.githubusercontent.com/spacemeshos/go-spacemesh/develop/Dockerfile -O Dockerfile.spacemesh.node - @docker build -f $(shell pwd)/Dockerfile.spacemesh.node -t spacemesh/node:latest . + @wget https://raw.githubusercontent.com/spacemeshos/go-spacemesh/$(NODE)/Dockerfile -O Dockerfile.spacemesh.node + @docker build -f $(shell pwd)/Dockerfile.spacemesh.node --build-arg BRANCH=$(NODE) -t spacemesh/node:latest . @docker tag spacemesh/devnet_agent gcr.io/spacemesh-198810/devnet_agent @docker push gcr.io/spacemesh-198810/node \ No newline at end of file From dfb010a3cc68e81bc08c45ba1f5f92b98a36e8cc Mon Sep 17 00:00:00 2001 From: moshe Date: Mon, 5 Nov 2018 15:46:48 +0200 Subject: [PATCH 109/209] fix --- tests/base_test_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index 22903e5..ee584d0 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -34,7 +34,7 @@ def start_node(self): logging.info('seeders:' + os.environ['SEEDERS']) self.modify_seeders(os.environ['SEEDERS']) self.docker.stop('node_' + self.node) - self.docker.start('docker run --network=devnet --name node_' + self.node + ' -p ' + str(self.node_port + int(self.node)) + ':' + str(self.node_port) + ' -v /root/spacemesh/devnet/logs' + self.node + ':/root/.spacemesh/nodes/ -v /root/spacemesh/devnet/cnf' + self.node + '/test.config.toml:/go/test.config.toml spacemesh/node:latest /go/src/github.com/spacemeshos/go-spacemesh/go-spacemesh -config /go/test.config.toml') + self.docker.start('docker run --network=devnet --name node_' + self.node + ' -p ' + str(self.node_port + int(self.node)) + ':' + str(self.node_port) + ' -v /root/spacemesh/devnet/logs' + self.node + ':/root/.spacemesh/nodes/ -v /root/spacemesh/devnet/cnf' + self.node + '/test.config.toml:/go/test.config.toml spacemesh/node:latest /go/src/github.com/spacemeshos/go-spacemesh/go-spacemesh --config /go/test.config.toml') while self.get_node_id() == 'NULL': time.sleep(1) From 9c01292b6dc41e0848d881791cdd5386adfa33af Mon Sep 17 00:00:00 2001 From: moshe Date: Mon, 5 Nov 2018 17:24:45 +0200 Subject: [PATCH 110/209] fix the config parameter --- tests/base_test_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index ee584d0..c45ef83 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -34,7 +34,7 @@ def start_node(self): logging.info('seeders:' + os.environ['SEEDERS']) self.modify_seeders(os.environ['SEEDERS']) self.docker.stop('node_' + self.node) - self.docker.start('docker run --network=devnet --name node_' + self.node + ' -p ' + str(self.node_port + int(self.node)) + ':' + str(self.node_port) + ' -v /root/spacemesh/devnet/logs' + self.node + ':/root/.spacemesh/nodes/ -v /root/spacemesh/devnet/cnf' + self.node + '/test.config.toml:/go/test.config.toml spacemesh/node:latest /go/src/github.com/spacemeshos/go-spacemesh/go-spacemesh --config /go/test.config.toml') + self.docker.start('docker run --network=devnet --name node_' + self.node + ' -p ' + str(self.node_port + int(self.node)) + ':' + str(self.node_port) + ' -v /root/spacemesh/devnet/logs' + self.node + ':/root/.spacemesh/nodes/ -v /root/spacemesh/devnet/cnf' + self.node + '/test.config.toml:/go/test.config.toml spacemesh/node:latest /go/src/github.com/spacemeshos/go-spacemesh/go-spacemesh --config=/go/test.config.toml') while self.get_node_id() == 'NULL': time.sleep(1) From a89779689f5e1d5c79d24ae1eb69d048853a9e7a Mon Sep 17 00:00:00 2001 From: moshe Date: Mon, 5 Nov 2018 18:10:15 +0200 Subject: [PATCH 111/209] use the default config location --- tests/base_test_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index c45ef83..096c02f 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -34,7 +34,7 @@ def start_node(self): logging.info('seeders:' + os.environ['SEEDERS']) self.modify_seeders(os.environ['SEEDERS']) self.docker.stop('node_' + self.node) - self.docker.start('docker run --network=devnet --name node_' + self.node + ' -p ' + str(self.node_port + int(self.node)) + ':' + str(self.node_port) + ' -v /root/spacemesh/devnet/logs' + self.node + ':/root/.spacemesh/nodes/ -v /root/spacemesh/devnet/cnf' + self.node + '/test.config.toml:/go/test.config.toml spacemesh/node:latest /go/src/github.com/spacemeshos/go-spacemesh/go-spacemesh --config=/go/test.config.toml') + self.docker.start('docker run --network=devnet --name node_' + self.node + ' -p ' + str(self.node_port + int(self.node)) + ':' + str(self.node_port) + ' -v /root/spacemesh/devnet/logs' + self.node + ':/root/.spacemesh/nodes/ -v /root/spacemesh/devnet/cnf' + self.node + '/test.config.toml:/root/config.toml spacemesh/node:latest /go/src/github.com/spacemeshos/go-spacemesh/go-spacemesh --config=/root/config.toml') while self.get_node_id() == 'NULL': time.sleep(1) From b4f46fbc6265918789c2c4905555fa7e8774807e Mon Sep 17 00:00:00 2001 From: moshe Date: Mon, 5 Nov 2018 18:28:11 +0200 Subject: [PATCH 112/209] support bootstrap parameter --- tests/base_test_agent.py | 6 +++--- tests/base_test_ci.py | 6 +++--- tests/test.config.toml | 2 +- tests/test_2.py | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index 096c02f..24ac833 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -32,19 +32,19 @@ def establish_links(self): def start_node(self): logging.info('seeders:' + os.environ['SEEDERS']) - self.modify_seeders(os.environ['SEEDERS']) + self.modify_seeders(os.environ['SEEDERS'], os.environ['BOOTSTRAP']) self.docker.stop('node_' + self.node) self.docker.start('docker run --network=devnet --name node_' + self.node + ' -p ' + str(self.node_port + int(self.node)) + ':' + str(self.node_port) + ' -v /root/spacemesh/devnet/logs' + self.node + ':/root/.spacemesh/nodes/ -v /root/spacemesh/devnet/cnf' + self.node + '/test.config.toml:/root/config.toml spacemesh/node:latest /go/src/github.com/spacemeshos/go-spacemesh/go-spacemesh --config=/root/config.toml') while self.get_node_id() == 'NULL': time.sleep(1) - def modify_seeders(self, seeders): + def modify_seeders(self, seeders, bootstrap): file_name = "/opt/basecnf/test.config.toml" out_file_name = "/opt/cnf/test.config.toml" with open(file_name) as f: - new_config = f.read().replace('BOOT_NODES', 'bootnodes = ' + seeders) + new_config = f.read().replace('BOOT_NODES', 'bootnodes = ' + seeders).replace('BOOTSTRAP_VALUE', bootstrap) with open(out_file_name, "w") as f: f.write(new_config) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index 195dedf..f69adc3 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -56,14 +56,14 @@ def wait_for_response(self, num_messages = 1): break time.sleep(1) - def send_and_wait(self, data, phase=None): + def send_and_wait(self, data): self.send(data) self.wait_for_response() - def start_node_agent_pair(self, seeders=config.CONFIG['no_seeders']): + def start_node_agent_pair(self, seeders=config.CONFIG['no_seeders'], bootstrap = 'false'): docker = Docker() docker.stop('agent_' + str(self.agents)) - cmd = 'docker run --network=devnet --name agent_' + str(self.agents) + ' -v /root/spacemesh/devnet/tests:/opt/devnet -v /root/spacemesh/devnet/logs' + str(self.agents) + ':/opt/logs -v /root/spacemesh/devnet/cnf' + str(self.agents) + ':/opt/cnf/ -e SUBSCRIPTION_NAME_DOWNSTREAM=devnet_tests_agent_' + str(self.agents) + ' -e PHASE=' + self.phase + ' -e NODE=' + str(self.agents) + ' -e SEEDERS=' + seeders + ' spacemesh/devnet_agent:latest python3 /opt/devnet/base_test_agent.py' + cmd = 'docker run --network=devnet --name agent_' + str(self.agents) + ' -v /root/spacemesh/devnet/tests:/opt/devnet -v /root/spacemesh/devnet/logs' + str(self.agents) + ':/opt/logs -v /root/spacemesh/devnet/cnf' + str(self.agents) + ':/opt/cnf/ -e SUBSCRIPTION_NAME_DOWNSTREAM=devnet_tests_agent_' + str(self.agents) + ' -e PHASE=' + self.phase + ' -e BOOTSTRAP=' + bootstrap +' -e NODE=' + str(self.agents) + ' -e SEEDERS=' + seeders + ' spacemesh/devnet_agent:latest python3 /opt/devnet/base_test_agent.py' docker.start(cmd) self.agents += 1 diff --git a/tests/test.config.toml b/tests/test.config.toml index 972bc4e..54e4b6c 100644 --- a/tests/test.config.toml +++ b/tests/test.config.toml @@ -20,7 +20,7 @@ response-timeout = "2s" # Node Swarm Config [p2p.swarm] -bootstrap = false +bootstrap = BOOTSTRAP_VALUE bucketsize = 20 # Routing table bucket size rtalpha = 3 # Routing table alpha randcon = 2 # Number of random connections diff --git a/tests/test_2.py b/tests/test_2.py index cbb2fe1..d72ad3e 100644 --- a/tests/test_2.py +++ b/tests/test_2.py @@ -11,7 +11,7 @@ class Test2(BaseTest): def test_sendId(self): phase_0 = self.create_phase("2.0") for i in range(0, 3): - self.start_node_agent_pair() + self.start_node_agent_pair(bootstrap='true') self.send('GET_NODE_ID') self.wait_for_response(3) @@ -29,7 +29,7 @@ def test_sendId(self): logging.info(self.messages) for i in range(0, 3): - self.start_node_agent_pair(seeders=seeders_str) + self.start_node_agent_pair(seeders=seeders_str, bootstrap='true') self.send('GET_NODE_ID') self.wait_for_response(3) From 9d8ad6641df757faa917ab0fa2b4e4975bbdb691 Mon Sep 17 00:00:00 2001 From: moshe Date: Mon, 5 Nov 2018 18:46:15 +0200 Subject: [PATCH 113/209] Remove commas --- tests/base_test_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index 24ac833..39344b5 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -44,7 +44,7 @@ def modify_seeders(self, seeders, bootstrap): out_file_name = "/opt/cnf/test.config.toml" with open(file_name) as f: - new_config = f.read().replace('BOOT_NODES', 'bootnodes = ' + seeders).replace('BOOTSTRAP_VALUE', bootstrap) + new_config = f.read().replace('BOOT_NODES', 'bootnodes = ' + seeders[1:-1]).replace('BOOTSTRAP_VALUE', bootstrap) with open(out_file_name, "w") as f: f.write(new_config) From bba2c05ab36ba1e41879fe2a8e3651acc86f5482 Mon Sep 17 00:00:00 2001 From: moshe Date: Mon, 5 Nov 2018 19:02:39 +0200 Subject: [PATCH 114/209] for debug stuff --- tests/base_test_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index 39344b5..a6b11c3 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -34,7 +34,7 @@ def start_node(self): logging.info('seeders:' + os.environ['SEEDERS']) self.modify_seeders(os.environ['SEEDERS'], os.environ['BOOTSTRAP']) self.docker.stop('node_' + self.node) - self.docker.start('docker run --network=devnet --name node_' + self.node + ' -p ' + str(self.node_port + int(self.node)) + ':' + str(self.node_port) + ' -v /root/spacemesh/devnet/logs' + self.node + ':/root/.spacemesh/nodes/ -v /root/spacemesh/devnet/cnf' + self.node + '/test.config.toml:/root/config.toml spacemesh/node:latest /go/src/github.com/spacemeshos/go-spacemesh/go-spacemesh --config=/root/config.toml') + self.docker.start('docker run --network=devnet --name node_' + self.node + ' -p ' + str(self.node_port + int(self.node)) + ':' + str(self.node_port) + ' -v /root/spacemesh/devnet/logs' + self.node + ':/root/.spacemesh/nodes/ -v /root/spacemesh/devnet/cnf' + self.node + '/test.config.toml:/root/config1.toml spacemesh/node:latest /go/src/github.com/spacemeshos/go-spacemesh/go-spacemesh --config=/root/config.toml') while self.get_node_id() == 'NULL': time.sleep(1) From de28c269a34fd3bdd0d686ce3ab733af27e6bb75 Mon Sep 17 00:00:00 2001 From: moshe Date: Mon, 5 Nov 2018 19:17:50 +0200 Subject: [PATCH 115/209] debug --- tests/base_test_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index a6b11c3..365f00a 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -34,7 +34,7 @@ def start_node(self): logging.info('seeders:' + os.environ['SEEDERS']) self.modify_seeders(os.environ['SEEDERS'], os.environ['BOOTSTRAP']) self.docker.stop('node_' + self.node) - self.docker.start('docker run --network=devnet --name node_' + self.node + ' -p ' + str(self.node_port + int(self.node)) + ':' + str(self.node_port) + ' -v /root/spacemesh/devnet/logs' + self.node + ':/root/.spacemesh/nodes/ -v /root/spacemesh/devnet/cnf' + self.node + '/test.config.toml:/root/config1.toml spacemesh/node:latest /go/src/github.com/spacemeshos/go-spacemesh/go-spacemesh --config=/root/config.toml') + self.docker.start('docker run --network=devnet --name node_' + self.node + ' -p ' + str(self.node_port + int(self.node)) + ':' + str(self.node_port) + ' -v /root/spacemesh/devnet/logs' + self.node + ':/root/.spacemesh/nodes/ -v /root/spacemesh/devnet/cnf' + self.node + '/test.config.toml:/config1.toml spacemesh/node:latest /go/src/github.com/spacemeshos/go-spacemesh/go-spacemesh --config=/root/config.toml') while self.get_node_id() == 'NULL': time.sleep(1) From d37499f4dd9c860c4bb230d8e7861c8585b27e50 Mon Sep 17 00:00:00 2001 From: moshe Date: Mon, 5 Nov 2018 19:19:52 +0200 Subject: [PATCH 116/209] revert --- tests/base_test_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index 365f00a..39344b5 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -34,7 +34,7 @@ def start_node(self): logging.info('seeders:' + os.environ['SEEDERS']) self.modify_seeders(os.environ['SEEDERS'], os.environ['BOOTSTRAP']) self.docker.stop('node_' + self.node) - self.docker.start('docker run --network=devnet --name node_' + self.node + ' -p ' + str(self.node_port + int(self.node)) + ':' + str(self.node_port) + ' -v /root/spacemesh/devnet/logs' + self.node + ':/root/.spacemesh/nodes/ -v /root/spacemesh/devnet/cnf' + self.node + '/test.config.toml:/config1.toml spacemesh/node:latest /go/src/github.com/spacemeshos/go-spacemesh/go-spacemesh --config=/root/config.toml') + self.docker.start('docker run --network=devnet --name node_' + self.node + ' -p ' + str(self.node_port + int(self.node)) + ':' + str(self.node_port) + ' -v /root/spacemesh/devnet/logs' + self.node + ':/root/.spacemesh/nodes/ -v /root/spacemesh/devnet/cnf' + self.node + '/test.config.toml:/root/config.toml spacemesh/node:latest /go/src/github.com/spacemeshos/go-spacemesh/go-spacemesh --config=/root/config.toml') while self.get_node_id() == 'NULL': time.sleep(1) From 2d9a71c890557f53cecf5e4a07fbbe3416aa87a1 Mon Sep 17 00:00:00 2001 From: moshe Date: Tue, 6 Nov 2018 11:27:57 +0200 Subject: [PATCH 117/209] Update the config file --- tests/test.config.toml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/tests/test.config.toml b/tests/test.config.toml index 54e4b6c..490cd7f 100644 --- a/tests/test.config.toml +++ b/tests/test.config.toml @@ -4,7 +4,7 @@ # Main Config [main] -data-folder = "~/.spacemesh-data" +data-folder = "~/.spacemesh" # Node Config [p2p] @@ -12,7 +12,7 @@ security-param = 20 fast-sync = true tcp-port = 7513 node-id = "" -new-node= false +new-node= false # will try to load from file if this is false dial-timeout = "1m" conn-keepalive = "48h" network-id = 1 # 0 - MainNet, 1 - TestNet @@ -20,16 +20,17 @@ response-timeout = "2s" # Node Swarm Config [p2p.swarm] +gossip = true bootstrap = BOOTSTRAP_VALUE -bucketsize = 20 # Routing table bucket size -rtalpha = 3 # Routing table alpha -randcon = 2 # Number of random connections +bucketsize = 20 # Routing table bucket size. recommended higher for bootstrap node +rtalpha = 4 # Routing table alpha +randcon = 5 # Number of random connections (neighbors for gossip). increase on bigger networks decrease for small BOOT_NODES # API Config [api] -grpc-server = false -json-server = false +grpc-server = true +json-server = true grpc-port = 9091 json-port = 9090 @@ -38,4 +39,4 @@ json-port = 9090 max-allowed-time-drift = "10s" ntp-queries = 5 default-timeout-latency = "10s" -refresh-ntp-interval = "30m" +refresh-ntp-interval = "30m" \ No newline at end of file From 94c8c0caec0337a6518f8bd8aafda19364306c6d Mon Sep 17 00:00:00 2001 From: moshe Date: Tue, 6 Nov 2018 12:23:18 +0200 Subject: [PATCH 118/209] do some sleeping to define the bootstrap --- tests/test_2.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_2.py b/tests/test_2.py index d72ad3e..ad6b81d 100644 --- a/tests/test_2.py +++ b/tests/test_2.py @@ -39,6 +39,7 @@ def test_sendId(self): self.assertNotEqual(b'NULL', self.messages[i]) self.assertLess(15, len(self.messages[i])) + time.sleep(20) self.phase = phase_0 self.send("END") self.phase = phase_1 From 67e65ec1b2daf5e6a628457e22451319bbf535e7 Mon Sep 17 00:00:00 2001 From: moshe Date: Tue, 6 Nov 2018 14:12:06 +0200 Subject: [PATCH 119/209] Add DHT verification in the test side --- tests/test_2.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_2.py b/tests/test_2.py index ad6b81d..33c26cf 100644 --- a/tests/test_2.py +++ b/tests/test_2.py @@ -39,7 +39,12 @@ def test_sendId(self): self.assertNotEqual(b'NULL', self.messages[i]) self.assertLess(15, len(self.messages[i])) - time.sleep(20) + self.messages = [] + self.send('GET_DHT') + self.assertEqual(3, len(self.messages)) + for i in range(0, 3): + self.assertEqual(6, self.messages[i]) + self.phase = phase_0 self.send("END") self.phase = phase_1 From 1ae981f2bc318d3cef7864643048ef21a89b2aac Mon Sep 17 00:00:00 2001 From: moshe Date: Tue, 6 Nov 2018 14:34:57 +0200 Subject: [PATCH 120/209] get the number of peers from the node log --- tests/base_test_agent.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index 39344b5..43fb54f 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -66,6 +66,8 @@ def callback(self, message): self.send('UP') elif 'GET_NODE_ID' == self.message: self.send(self.get_node_id()) + elif 'GET_DHT' == self.message: + self.send(self.get_dht()) elif 'SHUTDOWN_NODE' == self.message: self.docker.stop('node_' + self.node) @@ -90,6 +92,15 @@ def get_node_id(self): logging.info('NodeId:' + node_id) return 'node_' + self.node + ':' + str(self.node_port) + '/' + node_id + def get_dht(self): + pattern = re.compile(u'.*DHT State with (\d+)') + for i in range(1..60): + for line in open('/opt/logs/node.log', "r", encoding="utf-8"): + results = pattern.match(line) + if results != None: + return results.group(1) + return 0 + if __name__ == '__main__': t = BaseDevnetAgent() t.act_on_request() \ No newline at end of file From 9f6cee4908414708a6814f90a8d42b86febc6f0a Mon Sep 17 00:00:00 2001 From: moshe Date: Tue, 6 Nov 2018 14:57:31 +0200 Subject: [PATCH 121/209] debugging --- tests/test_2.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_2.py b/tests/test_2.py index 33c26cf..30e0ae0 100644 --- a/tests/test_2.py +++ b/tests/test_2.py @@ -40,11 +40,13 @@ def test_sendId(self): self.assertLess(15, len(self.messages[i])) self.messages = [] + logging.info('Xx') self.send('GET_DHT') + logging.info('Xy') self.assertEqual(3, len(self.messages)) for i in range(0, 3): self.assertEqual(6, self.messages[i]) - + logging.info('Xz') self.phase = phase_0 self.send("END") self.phase = phase_1 From 49c8823a846d3255aeede6fcf92c623596c82d69 Mon Sep 17 00:00:00 2001 From: moshe Date: Tue, 6 Nov 2018 15:02:32 +0200 Subject: [PATCH 122/209] debugging --- tests/test_2.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_2.py b/tests/test_2.py index 30e0ae0..5294d72 100644 --- a/tests/test_2.py +++ b/tests/test_2.py @@ -13,12 +13,15 @@ def test_sendId(self): for i in range(0, 3): self.start_node_agent_pair(bootstrap='true') self.send('GET_NODE_ID') + logging.info('Xa') self.wait_for_response(3) - + logging.info('Xb') + self.assertEqual(3, len(self.messages)) for i in range(0, 3): self.assertNotEqual(b'NULL', self.messages[i]) self.assertLess(15, len(self.messages[i])) + logging.info('Xc') #'["0.0.0.0:7517/j7qWfWaJRVp25ZsnCu9rJ4PmhigZBtesB4YmQHqqPvtR"]' like phase_1 = self.create_phase("2.1") From 228bafab4250b51642db8c0aba0b232aaab08b23 Mon Sep 17 00:00:00 2001 From: moshe Date: Tue, 6 Nov 2018 15:34:18 +0200 Subject: [PATCH 123/209] typo --- tests/base_test_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index 43fb54f..255a0b2 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -94,7 +94,7 @@ def get_node_id(self): def get_dht(self): pattern = re.compile(u'.*DHT State with (\d+)') - for i in range(1..60): + for i in range(1, 60): for line in open('/opt/logs/node.log', "r", encoding="utf-8"): results = pattern.match(line) if results != None: From 4d74f4b2431608f287b4d5acbedc5d07f630ef5c Mon Sep 17 00:00:00 2001 From: moshe Date: Tue, 6 Nov 2018 15:38:55 +0200 Subject: [PATCH 124/209] add the import re --- tests/base_test_agent.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index 255a0b2..52064ed 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -5,6 +5,7 @@ import datetime from subprocess import call import os +import re import logging from logging import Logger From c45551886c2d9abc30f53336fb14f66d88ff718c Mon Sep 17 00:00:00 2001 From: moshe Date: Tue, 6 Nov 2018 15:54:17 +0200 Subject: [PATCH 125/209] write the node output to log so we could analyze the DHT --- tests/base_test_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index 52064ed..66e4923 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -35,7 +35,7 @@ def start_node(self): logging.info('seeders:' + os.environ['SEEDERS']) self.modify_seeders(os.environ['SEEDERS'], os.environ['BOOTSTRAP']) self.docker.stop('node_' + self.node) - self.docker.start('docker run --network=devnet --name node_' + self.node + ' -p ' + str(self.node_port + int(self.node)) + ':' + str(self.node_port) + ' -v /root/spacemesh/devnet/logs' + self.node + ':/root/.spacemesh/nodes/ -v /root/spacemesh/devnet/cnf' + self.node + '/test.config.toml:/root/config.toml spacemesh/node:latest /go/src/github.com/spacemeshos/go-spacemesh/go-spacemesh --config=/root/config.toml') + self.docker.start('docker run --network=devnet --name node_' + self.node + ' -p ' + str(self.node_port + int(self.node)) + ':' + str(self.node_port) + ' -v /root/spacemesh/devnet/logs' + self.node + ':/root/.spacemesh/nodes/ -v /root/spacemesh/devnet/cnf' + self.node + '/test.config.toml:/root/config.toml spacemesh/node:latest /go/src/github.com/spacemeshos/go-spacemesh/go-spacemesh --config=/root/config.toml > /root/.spacemesh/nodes/node.log') while self.get_node_id() == 'NULL': time.sleep(1) From 84ecd1c1f5db8281c9ba5246560277687d533132 Mon Sep 17 00:00:00 2001 From: moshe Date: Tue, 6 Nov 2018 16:07:52 +0200 Subject: [PATCH 126/209] host path --- tests/base_test_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index 66e4923..d6f30c6 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -35,7 +35,7 @@ def start_node(self): logging.info('seeders:' + os.environ['SEEDERS']) self.modify_seeders(os.environ['SEEDERS'], os.environ['BOOTSTRAP']) self.docker.stop('node_' + self.node) - self.docker.start('docker run --network=devnet --name node_' + self.node + ' -p ' + str(self.node_port + int(self.node)) + ':' + str(self.node_port) + ' -v /root/spacemesh/devnet/logs' + self.node + ':/root/.spacemesh/nodes/ -v /root/spacemesh/devnet/cnf' + self.node + '/test.config.toml:/root/config.toml spacemesh/node:latest /go/src/github.com/spacemeshos/go-spacemesh/go-spacemesh --config=/root/config.toml > /root/.spacemesh/nodes/node.log') + self.docker.start('docker run --network=devnet --name node_' + self.node + ' -p ' + str(self.node_port + int(self.node)) + ':' + str(self.node_port) + ' -v /root/spacemesh/devnet/logs' + self.node + ':/root/.spacemesh/nodes/ -v /root/spacemesh/devnet/cnf' + self.node + '/test.config.toml:/root/config.toml spacemesh/node:latest /go/src/github.com/spacemeshos/go-spacemesh/go-spacemesh --config=/root/config.toml > /root/spacemesh/devnet/logs' + self.node + '/node.log') while self.get_node_id() == 'NULL': time.sleep(1) From 7a5efd4035be671693daf2ccbc34fdee2995ec98 Mon Sep 17 00:00:00 2001 From: moshe Date: Tue, 6 Nov 2018 16:17:53 +0200 Subject: [PATCH 127/209] use the node detailed log instead of the stdout --- tests/base_test_agent.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index d6f30c6..3b0c751 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -14,6 +14,7 @@ def __init__(self): logging.basicConfig(format='%(asctime)s %(message)s', level=logging.INFO) self.endFlag = False + self.node_id = "NULL" self.node = os.environ['NODE'] self.phase = os.environ['PHASE'] self.node_port = config.CONFIG['node_port'] @@ -84,19 +85,19 @@ def send(self, data): def get_node_id(self): try: - node_id = next(os.walk('/opt/logs'))[1][0] + self.node_id = next(os.walk('/opt/logs'))[1][0] except Exception as e: logging.warning('Error finding log folder') logging.warning(e.__doc__ ) return 'NULL' logging.info('NodeId:' + node_id) - return 'node_' + self.node + ':' + str(self.node_port) + '/' + node_id + return 'node_' + self.node + ':' + str(self.node_port) + '/' + self.node_id def get_dht(self): pattern = re.compile(u'.*DHT State with (\d+)') for i in range(1, 60): - for line in open('/opt/logs/node.log', "r", encoding="utf-8"): + for line in open('/opt/logs/' + self.node_id '/node.log', "r", encoding="utf-8"): results = pattern.match(line) if results != None: return results.group(1) From 947d3ca30d4821efb93d4570738959d4f88119ef Mon Sep 17 00:00:00 2001 From: moshe Date: Tue, 6 Nov 2018 16:39:13 +0200 Subject: [PATCH 128/209] typo --- tests/base_test_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index 3b0c751..e07ef33 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -97,7 +97,7 @@ def get_node_id(self): def get_dht(self): pattern = re.compile(u'.*DHT State with (\d+)') for i in range(1, 60): - for line in open('/opt/logs/' + self.node_id '/node.log', "r", encoding="utf-8"): + for line in open('/opt/logs/' + self.node_id + '/node.log', "r", encoding="utf-8"): results = pattern.match(line) if results != None: return results.group(1) From cdf2289cd07527acdb2fba1d86690f6206c638ea Mon Sep 17 00:00:00 2001 From: moshe Date: Tue, 6 Nov 2018 16:40:57 +0200 Subject: [PATCH 129/209] typo --- tests/base_test_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index e07ef33..f8902f0 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -91,7 +91,7 @@ def get_node_id(self): logging.warning(e.__doc__ ) return 'NULL' - logging.info('NodeId:' + node_id) + logging.info('NodeId:' + self.node_id) return 'node_' + self.node + ':' + str(self.node_port) + '/' + self.node_id def get_dht(self): From af5a775d029786e9b8d198e4d1e839b76de0b5d3 Mon Sep 17 00:00:00 2001 From: moshe Date: Tue, 6 Nov 2018 16:43:53 +0200 Subject: [PATCH 130/209] add the wait to get the answers --- tests/test_2.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_2.py b/tests/test_2.py index 5294d72..ac9f77f 100644 --- a/tests/test_2.py +++ b/tests/test_2.py @@ -45,6 +45,7 @@ def test_sendId(self): self.messages = [] logging.info('Xx') self.send('GET_DHT') + self.wait_for_response(3) logging.info('Xy') self.assertEqual(3, len(self.messages)) for i in range(0, 3): From 913a5d9650f019db8c383f42e9e78b36ff25f180 Mon Sep 17 00:00:00 2001 From: moshe Date: Tue, 6 Nov 2018 16:51:31 +0200 Subject: [PATCH 131/209] take care of int --- tests/base_test_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index f8902f0..3ae6a4e 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -79,7 +79,7 @@ def act_on_request(self): time.sleep(1) def send(self, data): - logging.info("sent: " + data) + logging.info("sent: " + str(data)) data = data.encode('utf-8') self.publisher_upstream.publish(self.topic_path_upstream, data=data, phase=self.phase) From 0e05158ba9a725f01ba2a0332701cdd793cc349a Mon Sep 17 00:00:00 2001 From: moshe Date: Tue, 6 Nov 2018 16:54:20 +0200 Subject: [PATCH 132/209] convert to int --- tests/test_2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_2.py b/tests/test_2.py index ac9f77f..c44c34f 100644 --- a/tests/test_2.py +++ b/tests/test_2.py @@ -49,7 +49,7 @@ def test_sendId(self): logging.info('Xy') self.assertEqual(3, len(self.messages)) for i in range(0, 3): - self.assertEqual(6, self.messages[i]) + self.assertEqual(6, int(self.messages[i])) logging.info('Xz') self.phase = phase_0 self.send("END") From 01753d4f1e65ef50f179ab33ce03224a5b4b21fd Mon Sep 17 00:00:00 2001 From: moshe Date: Tue, 6 Nov 2018 17:00:31 +0200 Subject: [PATCH 133/209] cleanup --- tests/test_2.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/tests/test_2.py b/tests/test_2.py index c44c34f..1df3714 100644 --- a/tests/test_2.py +++ b/tests/test_2.py @@ -13,15 +13,12 @@ def test_sendId(self): for i in range(0, 3): self.start_node_agent_pair(bootstrap='true') self.send('GET_NODE_ID') - logging.info('Xa') self.wait_for_response(3) - logging.info('Xb') self.assertEqual(3, len(self.messages)) for i in range(0, 3): self.assertNotEqual(b'NULL', self.messages[i]) self.assertLess(15, len(self.messages[i])) - logging.info('Xc') #'["0.0.0.0:7517/j7qWfWaJRVp25ZsnCu9rJ4PmhigZBtesB4YmQHqqPvtR"]' like phase_1 = self.create_phase("2.1") @@ -43,17 +40,17 @@ def test_sendId(self): self.assertLess(15, len(self.messages[i])) self.messages = [] - logging.info('Xx') self.send('GET_DHT') self.wait_for_response(3) - logging.info('Xy') + self.assertEqual(3, len(self.messages)) for i in range(0, 3): self.assertEqual(6, int(self.messages[i])) - logging.info('Xz') + self.phase = phase_0 - self.send("END") + self.send('END') self.phase = phase_1 - self.send("END") + self.send('END') + if __name__ == '__main__': unittest.main() \ No newline at end of file From f82903b7d0b2a2aef2ca86e9268a1272efd1580d Mon Sep 17 00:00:00 2001 From: moshe Date: Tue, 6 Nov 2018 17:17:49 +0200 Subject: [PATCH 134/209] extract timeout const --- tests/base_test_agent.py | 3 ++- tests/config.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index 3ae6a4e..e7e32e7 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -14,6 +14,7 @@ def __init__(self): logging.basicConfig(format='%(asctime)s %(message)s', level=logging.INFO) self.endFlag = False + self.dht_timeout = config.CONFIG['dht_timeout'] self.node_id = "NULL" self.node = os.environ['NODE'] self.phase = os.environ['PHASE'] @@ -96,7 +97,7 @@ def get_node_id(self): def get_dht(self): pattern = re.compile(u'.*DHT State with (\d+)') - for i in range(1, 60): + for i in range(0, self.dht_timeout): for line in open('/opt/logs/' + self.node_id + '/node.log', "r", encoding="utf-8"): results = pattern.match(line) if results != None: diff --git a/tests/config.py b/tests/config.py index de427b4..1f2da9f 100644 --- a/tests/config.py +++ b/tests/config.py @@ -8,5 +8,6 @@ 'host_user': 'deploy', 'host_password': 'deploy_password', 'no_seeders': '["0.0.0.0:7517/j7qWfWaJRVp25ZsnCu9rJ4PmhigZBtesB4YmQHqqPvtR"]', - 'node_port': 7513 + 'node_port': 7513, + 'dht_timeout': 60 } \ No newline at end of file From a80f08ec1ab326a79b98c44d916d8a305db38233 Mon Sep 17 00:00:00 2001 From: moshe Date: Tue, 6 Nov 2018 17:43:38 +0200 Subject: [PATCH 135/209] Recover the tests --- tests/tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/tests.py b/tests/tests.py index 9a51fe6..054577b 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -1,6 +1,6 @@ import unittest -#from test_0 import Test0 -#from test_1 import Test1 +from test_0 import Test0 +from test_1 import Test1 from test_2 import Test2 if __name__ == '__main__': From 784ce55cae1605a99bc78666f33813e171b80148 Mon Sep 17 00:00:00 2001 From: moshe Date: Tue, 6 Nov 2018 17:52:41 +0200 Subject: [PATCH 136/209] align test0 --- tests/test_0.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/test_0.py b/tests/test_0.py index 9d14a53..9d74c95 100644 --- a/tests/test_0.py +++ b/tests/test_0.py @@ -7,10 +7,13 @@ class Test0(BaseTest): def test_verifyUp(self): - self.start_node_agent_pair() - self.send_and_wait('SEND_UP') - - self.assertEqual(b'UP', self.message) + phase_0 = self.create_phase("0") + self.start_node_agent_pair(bootstrap='false') + self.send('SEND_UP') + self.wait_for_response(1) + + self.assertEqual(1, len(self.messages)) + self.assertEqual(b'UP', self.messages[0]) if __name__ == '__main__': unittest.main() \ No newline at end of file From 5f91d8554c3c7a19daa9a90cb752e120cf86fd90 Mon Sep 17 00:00:00 2001 From: moshe Date: Tue, 6 Nov 2018 18:08:37 +0200 Subject: [PATCH 137/209] fix --- tests/test_0.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_0.py b/tests/test_0.py index 9d74c95..9f1e00d 100644 --- a/tests/test_0.py +++ b/tests/test_0.py @@ -13,7 +13,7 @@ def test_verifyUp(self): self.wait_for_response(1) self.assertEqual(1, len(self.messages)) - self.assertEqual(b'UP', self.messages[0]) + self.assertEqual('UP', self.messages[0]) if __name__ == '__main__': unittest.main() \ No newline at end of file From 5ef04fc990676afa39ae83469c76b9af48500fb6 Mon Sep 17 00:00:00 2001 From: moshe Date: Tue, 6 Nov 2018 18:16:35 +0200 Subject: [PATCH 138/209] Rename labels --- tests/base_test_agent.py | 6 +++--- tests/test_2.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index e7e32e7..4bd9846 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -69,8 +69,8 @@ def callback(self, message): self.send('UP') elif 'GET_NODE_ID' == self.message: self.send(self.get_node_id()) - elif 'GET_DHT' == self.message: - self.send(self.get_dht()) + elif 'GET_DHT_SIZE' == self.message: + self.send(self.get_dht_size()) elif 'SHUTDOWN_NODE' == self.message: self.docker.stop('node_' + self.node) @@ -95,7 +95,7 @@ def get_node_id(self): logging.info('NodeId:' + self.node_id) return 'node_' + self.node + ':' + str(self.node_port) + '/' + self.node_id - def get_dht(self): + def get_dht_size(self): pattern = re.compile(u'.*DHT State with (\d+)') for i in range(0, self.dht_timeout): for line in open('/opt/logs/' + self.node_id + '/node.log', "r", encoding="utf-8"): diff --git a/tests/test_2.py b/tests/test_2.py index 1df3714..68edde5 100644 --- a/tests/test_2.py +++ b/tests/test_2.py @@ -40,7 +40,7 @@ def test_sendId(self): self.assertLess(15, len(self.messages[i])) self.messages = [] - self.send('GET_DHT') + self.send('GET_DHT_SIZE') self.wait_for_response(3) self.assertEqual(3, len(self.messages)) From fc17f79fe22ae0ef5ffd5b86ec71b49effee4d5c Mon Sep 17 00:00:00 2001 From: moshe Date: Tue, 6 Nov 2018 18:26:25 +0200 Subject: [PATCH 139/209] 1. Align test_1 2. Change phase 0 of test 2 to non bootstrap --- tests/test_1.py | 11 +++++++---- tests/test_2.py | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/test_1.py b/tests/test_1.py index 5054b69..d989585 100644 --- a/tests/test_1.py +++ b/tests/test_1.py @@ -7,11 +7,14 @@ class Test1(BaseTest): def test_sendId(self): - self.start_node_agent_pair() - self.send_and_wait('GET_NODE_ID') + phase_0 = self.create_phase("1") + self.start_node_agent_pair(bootstrap='false') + self.send('GET_NODE_ID') + self.wait_for_response(1) - self.assertNotEqual(b'NULL', self.message) - self.assertLess(5, len(self.message)) + self.assertEqual(1, len(self.messages)) + self.assertNotEqual(b'NULL', self.messages[0]) + self.assertLess(15, len(self.messages[0])) if __name__ == '__main__': unittest.main() \ No newline at end of file diff --git a/tests/test_2.py b/tests/test_2.py index 68edde5..b4e6fe0 100644 --- a/tests/test_2.py +++ b/tests/test_2.py @@ -11,7 +11,7 @@ class Test2(BaseTest): def test_sendId(self): phase_0 = self.create_phase("2.0") for i in range(0, 3): - self.start_node_agent_pair(bootstrap='true') + self.start_node_agent_pair(bootstrap='false') self.send('GET_NODE_ID') self.wait_for_response(3) From 8397329eb3b247fff6710a404560db10a748ef2b Mon Sep 17 00:00:00 2001 From: moshe Date: Tue, 6 Nov 2018 18:40:22 +0200 Subject: [PATCH 140/209] revert --- tests/test_2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_2.py b/tests/test_2.py index b4e6fe0..68edde5 100644 --- a/tests/test_2.py +++ b/tests/test_2.py @@ -11,7 +11,7 @@ class Test2(BaseTest): def test_sendId(self): phase_0 = self.create_phase("2.0") for i in range(0, 3): - self.start_node_agent_pair(bootstrap='false') + self.start_node_agent_pair(bootstrap='true') self.send('GET_NODE_ID') self.wait_for_response(3) From 7d38197e276b53754a8ee4a7660b117cab593aaa Mon Sep 17 00:00:00 2001 From: moshe Date: Tue, 6 Nov 2018 18:43:19 +0200 Subject: [PATCH 141/209] debug --- tests/tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/tests.py b/tests/tests.py index 054577b..9a51fe6 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -1,6 +1,6 @@ import unittest -from test_0 import Test0 -from test_1 import Test1 +#from test_0 import Test0 +#from test_1 import Test1 from test_2 import Test2 if __name__ == '__main__': From 3b00a11d602a26e96790694125fc98d2a0bea761 Mon Sep 17 00:00:00 2001 From: moshe Date: Tue, 6 Nov 2018 18:47:30 +0200 Subject: [PATCH 142/209] avoid encoding errors --- tests/base_test_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index 4bd9846..7076165 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -70,7 +70,7 @@ def callback(self, message): elif 'GET_NODE_ID' == self.message: self.send(self.get_node_id()) elif 'GET_DHT_SIZE' == self.message: - self.send(self.get_dht_size()) + self.send(str(self.get_dht_size())) elif 'SHUTDOWN_NODE' == self.message: self.docker.stop('node_' + self.node) From 02f49a34641258fb0094cafd5fe035ae838680a7 Mon Sep 17 00:00:00 2001 From: moshe Date: Tue, 6 Nov 2018 19:02:34 +0200 Subject: [PATCH 143/209] return tests --- tests/tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/tests.py b/tests/tests.py index 9a51fe6..054577b 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -1,6 +1,6 @@ import unittest -#from test_0 import Test0 -#from test_1 import Test1 +from test_0 import Test0 +from test_1 import Test1 from test_2 import Test2 if __name__ == '__main__': From 6a69f42c16b2b3c975869f41a02e0dbe6fb45d97 Mon Sep 17 00:00:00 2001 From: moshe Date: Tue, 6 Nov 2018 19:31:55 +0200 Subject: [PATCH 144/209] debug --- tests/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tests.py b/tests/tests.py index 054577b..71277cd 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -1,7 +1,7 @@ import unittest +from test_2 import Test2 from test_0 import Test0 from test_1 import Test1 -from test_2 import Test2 if __name__ == '__main__': unittest.main() \ No newline at end of file From e0965492ee9a85930c4f3b7d6a8f292d9927dec6 Mon Sep 17 00:00:00 2001 From: moshe Date: Tue, 6 Nov 2018 19:42:52 +0200 Subject: [PATCH 145/209] Revert stuff. looks like we need dedicated subscribers per ci test --- tests/tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/tests.py b/tests/tests.py index 71277cd..9a51fe6 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -1,7 +1,7 @@ import unittest +#from test_0 import Test0 +#from test_1 import Test1 from test_2 import Test2 -from test_0 import Test0 -from test_1 import Test1 if __name__ == '__main__': unittest.main() \ No newline at end of file From 67bf93d35eb39fa5271bc7c45bc7204e5dd7cf1e Mon Sep 17 00:00:00 2001 From: moshe Date: Sun, 11 Nov 2018 01:01:05 +0200 Subject: [PATCH 146/209] Refactor test2, so phases will be more declerative --- tests/base_test_ci.py | 19 +++++++++++++++++-- tests/test_2.py | 44 ++++++++----------------------------------- 2 files changed, 25 insertions(+), 38 deletions(-) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index f69adc3..d6553a7 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -13,7 +13,8 @@ class BaseTest(unittest.TestCase): def setUp(self): logging.basicConfig(format='%(asctime)s %(message)s', level=logging.INFO) - self.create_phase() + self.phases = [] + self.current_phase = 0 self.endFlag = False self.testLen = 30 self.message = b'NULL' @@ -31,7 +32,9 @@ def setUp(self): self.messages = [] def tearDown(self): - self.send('END') + for self.phase in self.phases + self.send('END') + def callback(self, message): if self.phase != message.attributes['phase']: @@ -42,9 +45,11 @@ def callback(self, message): def create_phase(self, phase = ""): self.phase = 'phase_' + str(phase) + '_' + str(calendar.timegm(time.gmtime())) + self.phases.append(self.phase) return self.phase def send(self, data): + self.messages = [] logging.info(data) data = data.encode('utf-8') self.publisher_downstream.publish(self.topic_path_downstream, data=data, phase=self.phase) @@ -67,5 +72,15 @@ def start_node_agent_pair(self, seeders=config.CONFIG['no_seeders'], bootstrap = docker.start(cmd) self.agents += 1 + def run_phase(self, nodes = 1, seeders=config.CONFIG['no_seeders'], bootstrap = 'false', message = ''): + self.create_phase(self.current_phase++) + for i in range(0, nodes): + self.start_node_agent_pair(seeders = seeders, bootstrap = bootstrap) + self.send(message) + self.wait_for_response(nodes) + self.assertEqual(nodes, len(self.messages)) + + return self.messages + if __name__ == '__main__': unittest.main() \ No newline at end of file diff --git a/tests/test_2.py b/tests/test_2.py index 68edde5..4364bc0 100644 --- a/tests/test_2.py +++ b/tests/test_2.py @@ -9,48 +9,20 @@ class Test2(BaseTest): def test_sendId(self): - phase_0 = self.create_phase("2.0") - for i in range(0, 3): - self.start_node_agent_pair(bootstrap='true') - self.send('GET_NODE_ID') - self.wait_for_response(3) + seeders_nodes = 3 + testers_nodes = 3 - self.assertEqual(3, len(self.messages)) - for i in range(0, 3): - self.assertNotEqual(b'NULL', self.messages[i]) - self.assertLess(15, len(self.messages[i])) + messages = run_phase(self, nodes = 3, bootstrap = 'true', message = 'GET_NODE_ID'): + for i in range(0, seeders_nodes): + self.assertLess(15, len(messages[i])) #'["0.0.0.0:7517/j7qWfWaJRVp25ZsnCu9rJ4PmhigZBtesB4YmQHqqPvtR"]' like - phase_1 = self.create_phase("2.1") seeders_str = '\'["' + '","'.join(self.messages) + '"]\'' logging.info(seeders_str) - logging.info(self.messages) - self.messages = [] - logging.info(self.messages) + messages = run_phase(self, nodes = 3, bootstrap = 'true', seeders = seeders_str, message = 'GET_DHT_SIZE'): - for i in range(0, 3): - self.start_node_agent_pair(seeders=seeders_str, bootstrap='true') - self.send('GET_NODE_ID') - self.wait_for_response(3) - - logging.info(self.messages) - self.assertEqual(3, len(self.messages)) - for i in range(0, 3): - self.assertNotEqual(b'NULL', self.messages[i]) - self.assertLess(15, len(self.messages[i])) - - self.messages = [] - self.send('GET_DHT_SIZE') - self.wait_for_response(3) - - self.assertEqual(3, len(self.messages)) - for i in range(0, 3): - self.assertEqual(6, int(self.messages[i])) - - self.phase = phase_0 - self.send('END') - self.phase = phase_1 - self.send('END') + for i in range(0, testers_nodes): + self.assertEqual(seeders_nodes + testers_nodes, int(messages[i])) if __name__ == '__main__': unittest.main() \ No newline at end of file From 67fbaaaa3723bde566cd3f1797449015fb42a749 Mon Sep 17 00:00:00 2001 From: moshe Date: Sun, 11 Nov 2018 01:07:26 +0200 Subject: [PATCH 147/209] typo --- tests/test_2.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_2.py b/tests/test_2.py index 4364bc0..1608ec9 100644 --- a/tests/test_2.py +++ b/tests/test_2.py @@ -12,14 +12,14 @@ def test_sendId(self): seeders_nodes = 3 testers_nodes = 3 - messages = run_phase(self, nodes = 3, bootstrap = 'true', message = 'GET_NODE_ID'): + messages = run_phase(self, nodes = 3, bootstrap = 'true', message = 'GET_NODE_ID') for i in range(0, seeders_nodes): self.assertLess(15, len(messages[i])) #'["0.0.0.0:7517/j7qWfWaJRVp25ZsnCu9rJ4PmhigZBtesB4YmQHqqPvtR"]' like seeders_str = '\'["' + '","'.join(self.messages) + '"]\'' logging.info(seeders_str) - messages = run_phase(self, nodes = 3, bootstrap = 'true', seeders = seeders_str, message = 'GET_DHT_SIZE'): + messages = run_phase(self, nodes = 3, bootstrap = 'true', seeders = seeders_str, message = 'GET_DHT_SIZE') for i in range(0, testers_nodes): self.assertEqual(seeders_nodes + testers_nodes, int(messages[i])) From c1946305cfafcd8fe06a3097acac5d0805e6cc16 Mon Sep 17 00:00:00 2001 From: moshe Date: Sun, 11 Nov 2018 01:08:41 +0200 Subject: [PATCH 148/209] typo --- tests/base_test_ci.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index d6553a7..9a78d2c 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -32,10 +32,9 @@ def setUp(self): self.messages = [] def tearDown(self): - for self.phase in self.phases + for self.phase in self.phases: self.send('END') - def callback(self, message): if self.phase != message.attributes['phase']: return From bcfd8843420e95b541bcb87c90324c7405e4203f Mon Sep 17 00:00:00 2001 From: moshe Date: Sun, 11 Nov 2018 01:11:54 +0200 Subject: [PATCH 149/209] simplify phases count --- tests/base_test_ci.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index 9a78d2c..de552f3 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -14,7 +14,6 @@ def setUp(self): logging.basicConfig(format='%(asctime)s %(message)s', level=logging.INFO) self.phases = [] - self.current_phase = 0 self.endFlag = False self.testLen = 30 self.message = b'NULL' @@ -72,7 +71,7 @@ def start_node_agent_pair(self, seeders=config.CONFIG['no_seeders'], bootstrap = self.agents += 1 def run_phase(self, nodes = 1, seeders=config.CONFIG['no_seeders'], bootstrap = 'false', message = ''): - self.create_phase(self.current_phase++) + self.create_phase(len(self.phases)) for i in range(0, nodes): self.start_node_agent_pair(seeders = seeders, bootstrap = bootstrap) self.send(message) From e99d4ae72a0a73c1565a3b1f8fd548ddb2247ebe Mon Sep 17 00:00:00 2001 From: moshe Date: Sun, 11 Nov 2018 01:13:32 +0200 Subject: [PATCH 150/209] typo --- tests/test_2.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_2.py b/tests/test_2.py index 1608ec9..28ad05c 100644 --- a/tests/test_2.py +++ b/tests/test_2.py @@ -12,14 +12,14 @@ def test_sendId(self): seeders_nodes = 3 testers_nodes = 3 - messages = run_phase(self, nodes = 3, bootstrap = 'true', message = 'GET_NODE_ID') + messages = self.run_phase(self, nodes = 3, bootstrap = 'true', message = 'GET_NODE_ID') for i in range(0, seeders_nodes): self.assertLess(15, len(messages[i])) #'["0.0.0.0:7517/j7qWfWaJRVp25ZsnCu9rJ4PmhigZBtesB4YmQHqqPvtR"]' like seeders_str = '\'["' + '","'.join(self.messages) + '"]\'' logging.info(seeders_str) - messages = run_phase(self, nodes = 3, bootstrap = 'true', seeders = seeders_str, message = 'GET_DHT_SIZE') + messages = self.run_phase(self, nodes = 3, bootstrap = 'true', seeders = seeders_str, message = 'GET_DHT_SIZE') for i in range(0, testers_nodes): self.assertEqual(seeders_nodes + testers_nodes, int(messages[i])) From 6b76c5bc04f2968030c8c28f7d3be844ac956c62 Mon Sep 17 00:00:00 2001 From: moshe Date: Sun, 11 Nov 2018 01:15:20 +0200 Subject: [PATCH 151/209] typo --- tests/base_test_ci.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index de552f3..160d17b 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -70,7 +70,7 @@ def start_node_agent_pair(self, seeders=config.CONFIG['no_seeders'], bootstrap = docker.start(cmd) self.agents += 1 - def run_phase(self, nodes = 1, seeders=config.CONFIG['no_seeders'], bootstrap = 'false', message = ''): + def run_phase(self, nodes=1, seeders=config.CONFIG['no_seeders'], bootstrap='false', message=''): self.create_phase(len(self.phases)) for i in range(0, nodes): self.start_node_agent_pair(seeders = seeders, bootstrap = bootstrap) From 1a148e713efbdd0332c92522725efbc94d639644 Mon Sep 17 00:00:00 2001 From: moshe Date: Sun, 11 Nov 2018 01:16:16 +0200 Subject: [PATCH 152/209] remove --- tests/base_test_ci.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index 160d17b..bd20b57 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -70,7 +70,7 @@ def start_node_agent_pair(self, seeders=config.CONFIG['no_seeders'], bootstrap = docker.start(cmd) self.agents += 1 - def run_phase(self, nodes=1, seeders=config.CONFIG['no_seeders'], bootstrap='false', message=''): + def run_phase(self, nodes, seeders=config.CONFIG['no_seeders'], bootstrap='false', message=''): self.create_phase(len(self.phases)) for i in range(0, nodes): self.start_node_agent_pair(seeders = seeders, bootstrap = bootstrap) From 24813cae66468258610b3ee9670f6f579218b423 Mon Sep 17 00:00:00 2001 From: moshe Date: Sun, 11 Nov 2018 01:17:48 +0200 Subject: [PATCH 153/209] typo --- tests/base_test_ci.py | 2 +- tests/test_2.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index bd20b57..de552f3 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -70,7 +70,7 @@ def start_node_agent_pair(self, seeders=config.CONFIG['no_seeders'], bootstrap = docker.start(cmd) self.agents += 1 - def run_phase(self, nodes, seeders=config.CONFIG['no_seeders'], bootstrap='false', message=''): + def run_phase(self, nodes = 1, seeders=config.CONFIG['no_seeders'], bootstrap = 'false', message = ''): self.create_phase(len(self.phases)) for i in range(0, nodes): self.start_node_agent_pair(seeders = seeders, bootstrap = bootstrap) diff --git a/tests/test_2.py b/tests/test_2.py index 28ad05c..466fa98 100644 --- a/tests/test_2.py +++ b/tests/test_2.py @@ -12,14 +12,14 @@ def test_sendId(self): seeders_nodes = 3 testers_nodes = 3 - messages = self.run_phase(self, nodes = 3, bootstrap = 'true', message = 'GET_NODE_ID') + messages = self.run_phase(nodes = 3, bootstrap = 'true', message = 'GET_NODE_ID') for i in range(0, seeders_nodes): self.assertLess(15, len(messages[i])) #'["0.0.0.0:7517/j7qWfWaJRVp25ZsnCu9rJ4PmhigZBtesB4YmQHqqPvtR"]' like seeders_str = '\'["' + '","'.join(self.messages) + '"]\'' logging.info(seeders_str) - messages = self.run_phase(self, nodes = 3, bootstrap = 'true', seeders = seeders_str, message = 'GET_DHT_SIZE') + messages = self.run_phase(nodes = 3, bootstrap = 'true', seeders = seeders_str, message = 'GET_DHT_SIZE') for i in range(0, testers_nodes): self.assertEqual(seeders_nodes + testers_nodes, int(messages[i])) From 3f7e6bcbe837b2b3e1296cebf507e9d2ace8cf4e Mon Sep 17 00:00:00 2001 From: moshe Date: Sun, 11 Nov 2018 01:21:58 +0200 Subject: [PATCH 154/209] implement for test 0/1 as well --- tests/test_0.py | 9 ++------- tests/test_1.py | 10 ++-------- 2 files changed, 4 insertions(+), 15 deletions(-) diff --git a/tests/test_0.py b/tests/test_0.py index 9f1e00d..75b6543 100644 --- a/tests/test_0.py +++ b/tests/test_0.py @@ -7,13 +7,8 @@ class Test0(BaseTest): def test_verifyUp(self): - phase_0 = self.create_phase("0") - self.start_node_agent_pair(bootstrap='false') - self.send('SEND_UP') - self.wait_for_response(1) - - self.assertEqual(1, len(self.messages)) - self.assertEqual('UP', self.messages[0]) + messages = self.run_phase(nodes = 1, bootstrap = 'false', message = 'SEND_UP') + self.assertEqual('UP', messages[0]) if __name__ == '__main__': unittest.main() \ No newline at end of file diff --git a/tests/test_1.py b/tests/test_1.py index d989585..82da4d8 100644 --- a/tests/test_1.py +++ b/tests/test_1.py @@ -7,14 +7,8 @@ class Test1(BaseTest): def test_sendId(self): - phase_0 = self.create_phase("1") - self.start_node_agent_pair(bootstrap='false') - self.send('GET_NODE_ID') - self.wait_for_response(1) - - self.assertEqual(1, len(self.messages)) - self.assertNotEqual(b'NULL', self.messages[0]) - self.assertLess(15, len(self.messages[0])) + messages = self.run_phase(nodes = 1, bootstrap = 'false', message = 'GET_NODE_ID') + self.assertLess(15, len(messages[0])) if __name__ == '__main__': unittest.main() \ No newline at end of file From 8406d624872b51efab4d44f41e0c8b574935cbc8 Mon Sep 17 00:00:00 2001 From: moshe Date: Sun, 11 Nov 2018 12:04:48 +0200 Subject: [PATCH 155/209] Create a list of all nodes so we could send DM --- tests/base_test_ci.py | 7 +++++++ tests/test_2.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index de552f3..e57b4c0 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -13,6 +13,7 @@ class BaseTest(unittest.TestCase): def setUp(self): logging.basicConfig(format='%(asctime)s %(message)s', level=logging.INFO) + self.nodes_list = [] self.phases = [] self.endFlag = False self.testLen = 30 @@ -77,6 +78,12 @@ def run_phase(self, nodes = 1, seeders=config.CONFIG['no_seeders'], bootstrap = self.send(message) self.wait_for_response(nodes) self.assertEqual(nodes, len(self.messages)) + self.nodes_list += self.messages + + if message != '': + self.send(message) + self.wait_for_response(nodes) + self.assertEqual(nodes, len(self.messages)) return self.messages diff --git a/tests/test_2.py b/tests/test_2.py index 466fa98..6d4bbd6 100644 --- a/tests/test_2.py +++ b/tests/test_2.py @@ -12,7 +12,7 @@ def test_sendId(self): seeders_nodes = 3 testers_nodes = 3 - messages = self.run_phase(nodes = 3, bootstrap = 'true', message = 'GET_NODE_ID') + messages = self.run_phase(nodes = 3, bootstrap = 'true') for i in range(0, seeders_nodes): self.assertLess(15, len(messages[i])) From 400bf9bf35ae3ac63b8ca2bd956037225673e209 Mon Sep 17 00:00:00 2001 From: moshe Date: Sun, 11 Nov 2018 12:08:43 +0200 Subject: [PATCH 156/209] fix --- tests/base_test_ci.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index e57b4c0..00e3e59 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -75,7 +75,7 @@ def run_phase(self, nodes = 1, seeders=config.CONFIG['no_seeders'], bootstrap = self.create_phase(len(self.phases)) for i in range(0, nodes): self.start_node_agent_pair(seeders = seeders, bootstrap = bootstrap) - self.send(message) + self.send('GET_NODE_ID') self.wait_for_response(nodes) self.assertEqual(nodes, len(self.messages)) self.nodes_list += self.messages From 0dddc992ee7427c13b96d59295089b9e463ca5a9 Mon Sep 17 00:00:00 2001 From: moshe Date: Sun, 11 Nov 2018 12:16:10 +0200 Subject: [PATCH 157/209] increase test length --- tests/base_test_ci.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index 00e3e59..f26693e 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -16,7 +16,7 @@ def setUp(self): self.nodes_list = [] self.phases = [] self.endFlag = False - self.testLen = 30 + self.testLen = 60 self.message = b'NULL' project = config.CONFIG['project'] subscription_name_upstream = config.CONFIG['subscription_name_upstream'] From 3124b96d4c1d5746448832da3dbdcad948411477 Mon Sep 17 00:00:00 2001 From: moshe Date: Sun, 11 Nov 2018 12:19:24 +0200 Subject: [PATCH 158/209] debug --- tests/base_test_ci.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index f26693e..68d6f99 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -59,6 +59,7 @@ def wait_for_response(self, num_messages = 1): logging.info(self.messages) break time.sleep(1) + time.sleep(10) def send_and_wait(self, data): self.send(data) From c1c609d41fa5f1d2291d029a93ade2f056f49ce0 Mon Sep 17 00:00:00 2001 From: moshe Date: Sun, 11 Nov 2018 12:25:16 +0200 Subject: [PATCH 159/209] revert debug --- tests/base_test_ci.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index 68d6f99..f26693e 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -59,7 +59,6 @@ def wait_for_response(self, num_messages = 1): logging.info(self.messages) break time.sleep(1) - time.sleep(10) def send_and_wait(self, data): self.send(data) From f58d753c04ccdd2df12772456f158aa6ead4d512 Mon Sep 17 00:00:00 2001 From: moshe Date: Sun, 11 Nov 2018 14:24:57 +0200 Subject: [PATCH 160/209] 1. Re-enable send_and_wait 2. Break create_phase test to resue it --- tests/base_test_ci.py | 16 +++++----------- tests/test_2.py | 5 +++-- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index f26693e..7d78e23 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -60,9 +60,11 @@ def wait_for_response(self, num_messages = 1): break time.sleep(1) - def send_and_wait(self, data): + def send_and_wait(self, data, nodes): self.send(data) - self.wait_for_response() + self.wait_for_response(nodes) + self.assertEqual(nodes, len(self.messages)) + return self.messages def start_node_agent_pair(self, seeders=config.CONFIG['no_seeders'], bootstrap = 'false'): docker = Docker() @@ -75,16 +77,8 @@ def run_phase(self, nodes = 1, seeders=config.CONFIG['no_seeders'], bootstrap = self.create_phase(len(self.phases)) for i in range(0, nodes): self.start_node_agent_pair(seeders = seeders, bootstrap = bootstrap) - self.send('GET_NODE_ID') - self.wait_for_response(nodes) - self.assertEqual(nodes, len(self.messages)) + self.send_and_wait('GET_NODE_ID', nodes): self.nodes_list += self.messages - - if message != '': - self.send(message) - self.wait_for_response(nodes) - self.assertEqual(nodes, len(self.messages)) - return self.messages if __name__ == '__main__': diff --git a/tests/test_2.py b/tests/test_2.py index 6d4bbd6..88e32ec 100644 --- a/tests/test_2.py +++ b/tests/test_2.py @@ -12,14 +12,15 @@ def test_sendId(self): seeders_nodes = 3 testers_nodes = 3 - messages = self.run_phase(nodes = 3, bootstrap = 'true') + messages = self.run_phase(nodes = seeders_nodes, bootstrap = 'true') for i in range(0, seeders_nodes): self.assertLess(15, len(messages[i])) #'["0.0.0.0:7517/j7qWfWaJRVp25ZsnCu9rJ4PmhigZBtesB4YmQHqqPvtR"]' like seeders_str = '\'["' + '","'.join(self.messages) + '"]\'' logging.info(seeders_str) - messages = self.run_phase(nodes = 3, bootstrap = 'true', seeders = seeders_str, message = 'GET_DHT_SIZE') + self.run_phase(nodes = testers_nodes, bootstrap = 'true', seeders = seeders_str) + messages = self.send_and_wait('GET_DHT_SIZE', testers_nodes): for i in range(0, testers_nodes): self.assertEqual(seeders_nodes + testers_nodes, int(messages[i])) From b737bcd27d9fa79109a852d9b4f8f993d76e7809 Mon Sep 17 00:00:00 2001 From: moshe Date: Sun, 11 Nov 2018 14:26:32 +0200 Subject: [PATCH 161/209] typo --- tests/test_2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_2.py b/tests/test_2.py index 88e32ec..3451b45 100644 --- a/tests/test_2.py +++ b/tests/test_2.py @@ -20,7 +20,7 @@ def test_sendId(self): seeders_str = '\'["' + '","'.join(self.messages) + '"]\'' logging.info(seeders_str) self.run_phase(nodes = testers_nodes, bootstrap = 'true', seeders = seeders_str) - messages = self.send_and_wait('GET_DHT_SIZE', testers_nodes): + messages = self.send_and_wait('GET_DHT_SIZE', testers_nodes) for i in range(0, testers_nodes): self.assertEqual(seeders_nodes + testers_nodes, int(messages[i])) From 5f21c0bdd3197ba18b4c19bae3b1dfea7a8a97ef Mon Sep 17 00:00:00 2001 From: moshe Date: Sun, 11 Nov 2018 14:29:08 +0200 Subject: [PATCH 162/209] typo --- tests/base_test_ci.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index 7d78e23..da6722b 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -77,7 +77,7 @@ def run_phase(self, nodes = 1, seeders=config.CONFIG['no_seeders'], bootstrap = self.create_phase(len(self.phases)) for i in range(0, nodes): self.start_node_agent_pair(seeders = seeders, bootstrap = bootstrap) - self.send_and_wait('GET_NODE_ID', nodes): + self.send_and_wait('GET_NODE_ID', nodes) self.nodes_list += self.messages return self.messages From f4233f61f39c866a6094e49528a7f355d00f9314 Mon Sep 17 00:00:00 2001 From: moshe Date: Sun, 11 Nov 2018 19:15:35 +0200 Subject: [PATCH 163/209] Support dynamic size of dht table --- tests/base_test_agent.py | 6 +++--- tests/base_test_ci.py | 8 ++++---- tests/test.config.toml | 2 +- tests/test_2.py | 5 +++-- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index 7076165..8f7f8c5 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -35,19 +35,19 @@ def establish_links(self): def start_node(self): logging.info('seeders:' + os.environ['SEEDERS']) - self.modify_seeders(os.environ['SEEDERS'], os.environ['BOOTSTRAP']) + self.modify_seeders(os.environ['SEEDERS'], os.environ['BOOTSTRAP'], os.environ['RANDCON']) self.docker.stop('node_' + self.node) self.docker.start('docker run --network=devnet --name node_' + self.node + ' -p ' + str(self.node_port + int(self.node)) + ':' + str(self.node_port) + ' -v /root/spacemesh/devnet/logs' + self.node + ':/root/.spacemesh/nodes/ -v /root/spacemesh/devnet/cnf' + self.node + '/test.config.toml:/root/config.toml spacemesh/node:latest /go/src/github.com/spacemeshos/go-spacemesh/go-spacemesh --config=/root/config.toml > /root/spacemesh/devnet/logs' + self.node + '/node.log') while self.get_node_id() == 'NULL': time.sleep(1) - def modify_seeders(self, seeders, bootstrap): + def modify_seeders(self, seeders, bootstrap, randcon): file_name = "/opt/basecnf/test.config.toml" out_file_name = "/opt/cnf/test.config.toml" with open(file_name) as f: - new_config = f.read().replace('BOOT_NODES', 'bootnodes = ' + seeders[1:-1]).replace('BOOTSTRAP_VALUE', bootstrap) + new_config = f.read().replace('BOOT_NODES', 'bootnodes = ' + seeders[1:-1]).replace('BOOTSTRAP_VALUE', bootstrap).replace('RANDCON', randcon) with open(out_file_name, "w") as f: f.write(new_config) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index da6722b..a7040d1 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -66,17 +66,17 @@ def send_and_wait(self, data, nodes): self.assertEqual(nodes, len(self.messages)) return self.messages - def start_node_agent_pair(self, seeders=config.CONFIG['no_seeders'], bootstrap = 'false'): + def start_node_agent_pair(self, seeders=config.CONFIG['no_seeders'], bootstrap = 'false', randcon = 5): docker = Docker() docker.stop('agent_' + str(self.agents)) - cmd = 'docker run --network=devnet --name agent_' + str(self.agents) + ' -v /root/spacemesh/devnet/tests:/opt/devnet -v /root/spacemesh/devnet/logs' + str(self.agents) + ':/opt/logs -v /root/spacemesh/devnet/cnf' + str(self.agents) + ':/opt/cnf/ -e SUBSCRIPTION_NAME_DOWNSTREAM=devnet_tests_agent_' + str(self.agents) + ' -e PHASE=' + self.phase + ' -e BOOTSTRAP=' + bootstrap +' -e NODE=' + str(self.agents) + ' -e SEEDERS=' + seeders + ' spacemesh/devnet_agent:latest python3 /opt/devnet/base_test_agent.py' + cmd = 'docker run --network=devnet --name agent_' + str(self.agents) + ' -v /root/spacemesh/devnet/tests:/opt/devnet -v /root/spacemesh/devnet/logs' + str(self.agents) + ':/opt/logs -v /root/spacemesh/devnet/cnf' + str(self.agents) + ':/opt/cnf/ -e SUBSCRIPTION_NAME_DOWNSTREAM=devnet_tests_agent_' + str(self.agents) + ' -e PHASE=' + self.phase + ' -e BOOTSTRAP=' + bootstrap +' -e NODE=' + str(self.agents) + ' -e SEEDERS=' + seeders + ' -e RANDCON=' + str(randcon) + ' spacemesh/devnet_agent:latest python3 /opt/devnet/base_test_agent.py' docker.start(cmd) self.agents += 1 - def run_phase(self, nodes = 1, seeders=config.CONFIG['no_seeders'], bootstrap = 'false', message = ''): + def run_phase(self, nodes = 1, seeders=config.CONFIG['no_seeders'], bootstrap = 'false', randcon = 5, message = ''): self.create_phase(len(self.phases)) for i in range(0, nodes): - self.start_node_agent_pair(seeders = seeders, bootstrap = bootstrap) + self.start_node_agent_pair(seeders = seeders, bootstrap = bootstrap, randcon = randcon) self.send_and_wait('GET_NODE_ID', nodes) self.nodes_list += self.messages return self.messages diff --git a/tests/test.config.toml b/tests/test.config.toml index 490cd7f..e798bf3 100644 --- a/tests/test.config.toml +++ b/tests/test.config.toml @@ -24,7 +24,7 @@ gossip = true bootstrap = BOOTSTRAP_VALUE bucketsize = 20 # Routing table bucket size. recommended higher for bootstrap node rtalpha = 4 # Routing table alpha -randcon = 5 # Number of random connections (neighbors for gossip). increase on bigger networks decrease for small +randcon = RANDCON # Number of random connections (neighbors for gossip). increase on bigger networks decrease for small BOOT_NODES # API Config diff --git a/tests/test_2.py b/tests/test_2.py index 3451b45..1e8c639 100644 --- a/tests/test_2.py +++ b/tests/test_2.py @@ -11,6 +11,7 @@ class Test2(BaseTest): def test_sendId(self): seeders_nodes = 3 testers_nodes = 3 + randcon = seeders_nodes + testers_nodes - 1 messages = self.run_phase(nodes = seeders_nodes, bootstrap = 'true') for i in range(0, seeders_nodes): @@ -19,11 +20,11 @@ def test_sendId(self): #'["0.0.0.0:7517/j7qWfWaJRVp25ZsnCu9rJ4PmhigZBtesB4YmQHqqPvtR"]' like seeders_str = '\'["' + '","'.join(self.messages) + '"]\'' logging.info(seeders_str) - self.run_phase(nodes = testers_nodes, bootstrap = 'true', seeders = seeders_str) + self.run_phase(nodes = testers_nodes, bootstrap = 'true', randcon = randcon, seeders = seeders_str) messages = self.send_and_wait('GET_DHT_SIZE', testers_nodes) for i in range(0, testers_nodes): - self.assertEqual(seeders_nodes + testers_nodes, int(messages[i])) + self.assertLess(randcon, int(messages[i])) if __name__ == '__main__': unittest.main() \ No newline at end of file From c445599a970a8cdcc85e9c10b9471a45d07da7a2 Mon Sep 17 00:00:00 2001 From: moshe Date: Sun, 11 Nov 2018 19:20:34 +0200 Subject: [PATCH 164/209] fix --- tests/base_test_agent.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index 8f7f8c5..90e22cf 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -102,6 +102,7 @@ def get_dht_size(self): results = pattern.match(line) if results != None: return results.group(1) + time.sleep(1) return 0 if __name__ == '__main__': From bd7e737e4a0ce9ffdc10ab9f48e03dec12467166 Mon Sep 17 00:00:00 2001 From: moshe Date: Mon, 12 Nov 2018 11:40:45 +0200 Subject: [PATCH 165/209] Refactor the CI side of then topic --- tests/base_test_ci.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index a7040d1..7efdcd2 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -18,15 +18,10 @@ def setUp(self): self.endFlag = False self.testLen = 60 self.message = b'NULL' - project = config.CONFIG['project'] - subscription_name_upstream = config.CONFIG['subscription_name_upstream'] - self.subscriber_upstream = pubsub_v1.SubscriberClient() - self.subscription_path_upstream = self.subscriber_upstream.subscription_path(project, subscription_name_upstream) - self.subscriber_upstream.subscribe(self.subscription_path_upstream, callback=self.callback) - topic_name_downstream = config.CONFIG['topic_name_downstream'] - self.publisher_downstream = pubsub_v1.PublisherClient() - self.topic_path_downstream = self.publisher_downstream.topic_path(project, topic_name_downstream) + self.project = config.CONFIG['project'] + self.subscribe() + self.enable_publish() self.agents = 0 self.messages = [] @@ -35,6 +30,17 @@ def tearDown(self): for self.phase in self.phases: self.send('END') + def subscribe(self): + subscription_name_upstream = config.CONFIG['subscription_name_upstream'] + self.subscriber_upstream = pubsub_v1.SubscriberClient() + self.subscription_path_upstream = self.subscriber_upstream.subscription_path(self.project, subscription_name_upstream) + self.subscriber_upstream.subscribe(self.subscription_path_upstream, callback=self.callback) + + def downstream(self): + topic_name_downstream = config.CONFIG['topic_name_downstream'] + self.publisher_downstream = pubsub_v1.PublisherClient() + self.topic_path_downstream = self.publisher_downstream.topic_path(self.project, topic_name_downstream) + def callback(self, message): if self.phase != message.attributes['phase']: return From 6d6eba80534d99c64226f0d73730793028caca01 Mon Sep 17 00:00:00 2001 From: moshe Date: Mon, 12 Nov 2018 11:45:55 +0200 Subject: [PATCH 166/209] fix --- tests/base_test_ci.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index 7efdcd2..552327e 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -36,7 +36,7 @@ def subscribe(self): self.subscription_path_upstream = self.subscriber_upstream.subscription_path(self.project, subscription_name_upstream) self.subscriber_upstream.subscribe(self.subscription_path_upstream, callback=self.callback) - def downstream(self): + def enable_publish(self): topic_name_downstream = config.CONFIG['topic_name_downstream'] self.publisher_downstream = pubsub_v1.PublisherClient() self.topic_path_downstream = self.publisher_downstream.topic_path(self.project, topic_name_downstream) From fa06a0cf24719b875769b61cde61f97dd482fbae Mon Sep 17 00:00:00 2001 From: moshe Date: Mon, 12 Nov 2018 12:03:18 +0200 Subject: [PATCH 167/209] fix test to match the code --- tests/base_test_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index 90e22cf..175beae 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -96,7 +96,7 @@ def get_node_id(self): return 'node_' + self.node + ':' + str(self.node_port) + '/' + self.node_id def get_dht_size(self): - pattern = re.compile(u'.*DHT State with (\d+)') + pattern = re.compile(u'.*DHT Bootstrapped with (\d+)') for i in range(0, self.dht_timeout): for line in open('/opt/logs/' + self.node_id + '/node.log', "r", encoding="utf-8"): results = pattern.match(line) From 0d0fbc9132b0c5116f8c8e3178fe52be0551b4bd Mon Sep 17 00:00:00 2001 From: moshe Date: Mon, 12 Nov 2018 12:12:42 +0200 Subject: [PATCH 168/209] test extract publish --- tests/base_test_ci.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index 552327e..e622f61 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -41,6 +41,9 @@ def enable_publish(self): self.publisher_downstream = pubsub_v1.PublisherClient() self.topic_path_downstream = self.publisher_downstream.topic_path(self.project, topic_name_downstream) + def publish(self, **kwargs): + self.publisher_downstream.publish(self.topic_path_downstream, kwargs) + def callback(self, message): if self.phase != message.attributes['phase']: return @@ -57,7 +60,7 @@ def send(self, data): self.messages = [] logging.info(data) data = data.encode('utf-8') - self.publisher_downstream.publish(self.topic_path_downstream, data=data, phase=self.phase) + self.publish(data=data, phase=self.phase) def wait_for_response(self, num_messages = 1): for i in range(0, self.testLen): From 86f82f98dda5602e1e85e016c00c83e066f50b04 Mon Sep 17 00:00:00 2001 From: moshe Date: Mon, 12 Nov 2018 12:15:03 +0200 Subject: [PATCH 169/209] test --- tests/base_test_ci.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index e622f61..6c7e797 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -42,7 +42,7 @@ def enable_publish(self): self.topic_path_downstream = self.publisher_downstream.topic_path(self.project, topic_name_downstream) def publish(self, **kwargs): - self.publisher_downstream.publish(self.topic_path_downstream, kwargs) + self.publisher_downstream.publish(self.topic_path_downstream, **kwargs) def callback(self, message): if self.phase != message.attributes['phase']: From 2ce99cf7762a105644919c5c3389499081131a47 Mon Sep 17 00:00:00 2001 From: moshe Date: Mon, 12 Nov 2018 12:35:52 +0200 Subject: [PATCH 170/209] extract the publish to another class --- tests/base_test_ci.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index 6c7e797..4e32077 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -9,6 +9,15 @@ import logging from logging import Logger +class Publisher(): + def enable(self, project): + topic_name_downstream = config.CONFIG['topic_name_downstream'] + self.publisher_downstream = pubsub_v1.PublisherClient() + self.topic_path_downstream = self.publisher_downstream.topic_path(project, topic_name_downstream) + + def publish(self, **kwargs): + self.publisher_downstream.publish(self.topic_path_downstream, **kwargs) + class BaseTest(unittest.TestCase): def setUp(self): logging.basicConfig(format='%(asctime)s %(message)s', level=logging.INFO) @@ -21,7 +30,8 @@ def setUp(self): self.project = config.CONFIG['project'] self.subscribe() - self.enable_publish() + self.publihser = Publisher(self.project) + self.publihser.enable() self.agents = 0 self.messages = [] @@ -36,14 +46,6 @@ def subscribe(self): self.subscription_path_upstream = self.subscriber_upstream.subscription_path(self.project, subscription_name_upstream) self.subscriber_upstream.subscribe(self.subscription_path_upstream, callback=self.callback) - def enable_publish(self): - topic_name_downstream = config.CONFIG['topic_name_downstream'] - self.publisher_downstream = pubsub_v1.PublisherClient() - self.topic_path_downstream = self.publisher_downstream.topic_path(self.project, topic_name_downstream) - - def publish(self, **kwargs): - self.publisher_downstream.publish(self.topic_path_downstream, **kwargs) - def callback(self, message): if self.phase != message.attributes['phase']: return From 2bab4a070e7fea8fc9689b12a4695568dcf51c4e Mon Sep 17 00:00:00 2001 From: moshe Date: Mon, 12 Nov 2018 12:36:48 +0200 Subject: [PATCH 171/209] fix --- tests/base_test_ci.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index 4e32077..1ff174b 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -30,8 +30,8 @@ def setUp(self): self.project = config.CONFIG['project'] self.subscribe() - self.publihser = Publisher(self.project) - self.publihser.enable() + self.publihser = Publisher() + self.publihser.enable(self.project) self.agents = 0 self.messages = [] From 4ea0cd1ff39fb5c17a6b5c40aa754e422c24627b Mon Sep 17 00:00:00 2001 From: moshe Date: Mon, 12 Nov 2018 12:38:44 +0200 Subject: [PATCH 172/209] fix --- tests/base_test_ci.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index 1ff174b..25940f7 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -30,8 +30,8 @@ def setUp(self): self.project = config.CONFIG['project'] self.subscribe() - self.publihser = Publisher() - self.publihser.enable(self.project) + self.publisher = Publisher() + self.publisher.enable(self.project) self.agents = 0 self.messages = [] @@ -62,7 +62,7 @@ def send(self, data): self.messages = [] logging.info(data) data = data.encode('utf-8') - self.publish(data=data, phase=self.phase) + self.publisher.publish(data=data, phase=self.phase) def wait_for_response(self, num_messages = 1): for i in range(0, self.testLen): From 2f9297bce8694bfa6a5945bc2d7ea87a94f2091e Mon Sep 17 00:00:00 2001 From: moshe Date: Mon, 12 Nov 2018 12:43:26 +0200 Subject: [PATCH 173/209] extract subscriber as well --- tests/base_test_ci.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index 25940f7..30eb29e 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -17,7 +17,15 @@ def enable(self, project): def publish(self, **kwargs): self.publisher_downstream.publish(self.topic_path_downstream, **kwargs) - + +class Subscriber(): + def subscribe(self): + subscription_name_upstream = config.CONFIG['subscription_name_upstream'] + self.subscriber_upstream = pubsub_v1.SubscriberClient() + self.subscription_path_upstream = self.subscriber_upstream.subscription_path(self.project, subscription_name_upstream) + self.subscriber_upstream.subscribe(self.subscription_path_upstream, callback=self.callback) + + class BaseTest(unittest.TestCase): def setUp(self): logging.basicConfig(format='%(asctime)s %(message)s', level=logging.INFO) @@ -29,9 +37,10 @@ def setUp(self): self.message = b'NULL' self.project = config.CONFIG['project'] - self.subscribe() - self.publisher = Publisher() - self.publisher.enable(self.project) + self.up_subscriber = Subscribe() + self.up_subscriber.subscribe() + self.down_publisher = Publisher() + self.down_publisher.enable(self.project) self.agents = 0 self.messages = [] @@ -40,12 +49,6 @@ def tearDown(self): for self.phase in self.phases: self.send('END') - def subscribe(self): - subscription_name_upstream = config.CONFIG['subscription_name_upstream'] - self.subscriber_upstream = pubsub_v1.SubscriberClient() - self.subscription_path_upstream = self.subscriber_upstream.subscription_path(self.project, subscription_name_upstream) - self.subscriber_upstream.subscribe(self.subscription_path_upstream, callback=self.callback) - def callback(self, message): if self.phase != message.attributes['phase']: return @@ -62,7 +65,7 @@ def send(self, data): self.messages = [] logging.info(data) data = data.encode('utf-8') - self.publisher.publish(data=data, phase=self.phase) + self.down_publisher.publish(data=data, phase=self.phase) def wait_for_response(self, num_messages = 1): for i in range(0, self.testLen): From cb587e9c7b3d0974dc2acb5a78b6d3f58ef4eb07 Mon Sep 17 00:00:00 2001 From: moshe Date: Mon, 12 Nov 2018 12:45:29 +0200 Subject: [PATCH 174/209] fix --- tests/base_test_ci.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index 30eb29e..4ab7d92 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -37,7 +37,7 @@ def setUp(self): self.message = b'NULL' self.project = config.CONFIG['project'] - self.up_subscriber = Subscribe() + self.up_subscriber = Subscriber() self.up_subscriber.subscribe() self.down_publisher = Publisher() self.down_publisher.enable(self.project) From f01dcdb141af52eb21f6f5dbc8e51ffd09b2f438 Mon Sep 17 00:00:00 2001 From: moshe Date: Mon, 12 Nov 2018 12:49:46 +0200 Subject: [PATCH 175/209] fixes --- tests/base_test_ci.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index 4ab7d92..6de9562 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -19,11 +19,11 @@ def publish(self, **kwargs): self.publisher_downstream.publish(self.topic_path_downstream, **kwargs) class Subscriber(): - def subscribe(self): + def subscribe(self, project, callback): subscription_name_upstream = config.CONFIG['subscription_name_upstream'] self.subscriber_upstream = pubsub_v1.SubscriberClient() - self.subscription_path_upstream = self.subscriber_upstream.subscription_path(self.project, subscription_name_upstream) - self.subscriber_upstream.subscribe(self.subscription_path_upstream, callback=self.callback) + self.subscription_path_upstream = self.subscriber_upstream.subscription_path(project, subscription_name_upstream) + self.subscriber_upstream.subscribe(self.subscription_path_upstream, callback=callback) class BaseTest(unittest.TestCase): @@ -38,7 +38,7 @@ def setUp(self): self.project = config.CONFIG['project'] self.up_subscriber = Subscriber() - self.up_subscriber.subscribe() + self.up_subscriber.subscribe(self.project, self.callback) self.down_publisher = Publisher() self.down_publisher.enable(self.project) From ca314ba08caf088d8e70885f51a34accf1eb6a01 Mon Sep 17 00:00:00 2001 From: moshe Date: Mon, 12 Nov 2018 13:02:11 +0200 Subject: [PATCH 176/209] generalize the code --- tests/base_test_ci.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index 6de9562..9321434 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -10,13 +10,12 @@ from logging import Logger class Publisher(): - def enable(self, project): - topic_name_downstream = config.CONFIG['topic_name_downstream'] - self.publisher_downstream = pubsub_v1.PublisherClient() - self.topic_path_downstream = self.publisher_downstream.topic_path(project, topic_name_downstream) + def enable(self, project, topic_name): + self.publisher = pubsub_v1.PublisherClient() + self.topic_path = self.publisher.topic_path(project, topic_name) def publish(self, **kwargs): - self.publisher_downstream.publish(self.topic_path_downstream, **kwargs) + self.publisher.publish(self.topic_path, **kwargs) class Subscriber(): def subscribe(self, project, callback): @@ -25,7 +24,6 @@ def subscribe(self, project, callback): self.subscription_path_upstream = self.subscriber_upstream.subscription_path(project, subscription_name_upstream) self.subscriber_upstream.subscribe(self.subscription_path_upstream, callback=callback) - class BaseTest(unittest.TestCase): def setUp(self): logging.basicConfig(format='%(asctime)s %(message)s', level=logging.INFO) @@ -40,7 +38,7 @@ def setUp(self): self.up_subscriber = Subscriber() self.up_subscriber.subscribe(self.project, self.callback) self.down_publisher = Publisher() - self.down_publisher.enable(self.project) + self.down_publisher.enable(self.project, config.CONFIG['topic_name_downstream']) self.agents = 0 self.messages = [] From aae79487693063e248b3c1eb812cc9598f7860b4 Mon Sep 17 00:00:00 2001 From: moshe Date: Mon, 12 Nov 2018 13:07:06 +0200 Subject: [PATCH 177/209] generalize as well --- tests/base_test_ci.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index 9321434..7b89e49 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -18,11 +18,10 @@ def publish(self, **kwargs): self.publisher.publish(self.topic_path, **kwargs) class Subscriber(): - def subscribe(self, project, callback): - subscription_name_upstream = config.CONFIG['subscription_name_upstream'] - self.subscriber_upstream = pubsub_v1.SubscriberClient() - self.subscription_path_upstream = self.subscriber_upstream.subscription_path(project, subscription_name_upstream) - self.subscriber_upstream.subscribe(self.subscription_path_upstream, callback=callback) + def subscribe(self, project, subscription_name, callback): + self.subscriber = pubsub_v1.SubscriberClient() + self.subscription_path = self.subscriber.subscription_path(project, subscription_name) + self.subscriber.subscribe(self.subscription_path, callback=callback) class BaseTest(unittest.TestCase): def setUp(self): @@ -36,7 +35,7 @@ def setUp(self): self.project = config.CONFIG['project'] self.up_subscriber = Subscriber() - self.up_subscriber.subscribe(self.project, self.callback) + self.up_subscriber.subscribe(self.project, config.CONFIG['subscription_name_upstream'], self.callback) self.down_publisher = Publisher() self.down_publisher.enable(self.project, config.CONFIG['topic_name_downstream']) From 2757d0867abad3b0597a00e2e6d9bf857232be2d Mon Sep 17 00:00:00 2001 From: moshe Date: Mon, 12 Nov 2018 13:10:05 +0200 Subject: [PATCH 178/209] Extract to external class --- tests/base_test_ci.py | 15 +-------------- tests/topics.py | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 14 deletions(-) create mode 100644 tests/topics.py diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index 7b89e49..bac1ae0 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -1,4 +1,5 @@ from dockers import Docker +from Topics import Publisher, Subscriber import config import os import time @@ -9,20 +10,6 @@ import logging from logging import Logger -class Publisher(): - def enable(self, project, topic_name): - self.publisher = pubsub_v1.PublisherClient() - self.topic_path = self.publisher.topic_path(project, topic_name) - - def publish(self, **kwargs): - self.publisher.publish(self.topic_path, **kwargs) - -class Subscriber(): - def subscribe(self, project, subscription_name, callback): - self.subscriber = pubsub_v1.SubscriberClient() - self.subscription_path = self.subscriber.subscription_path(project, subscription_name) - self.subscriber.subscribe(self.subscription_path, callback=callback) - class BaseTest(unittest.TestCase): def setUp(self): logging.basicConfig(format='%(asctime)s %(message)s', level=logging.INFO) diff --git a/tests/topics.py b/tests/topics.py new file mode 100644 index 0000000..737e295 --- /dev/null +++ b/tests/topics.py @@ -0,0 +1,18 @@ +import config +from google.cloud import pubsub_v1 +import logging +from logging import Logger + +class Publisher(): + def enable(self, project, topic_name): + self.publisher = pubsub_v1.PublisherClient() + self.topic_path = self.publisher.topic_path(project, topic_name) + + def publish(self, **kwargs): + self.publisher.publish(self.topic_path, **kwargs) + +class Subscriber(): + def subscribe(self, project, subscription_name, callback): + self.subscriber = pubsub_v1.SubscriberClient() + self.subscription_path = self.subscriber.subscription_path(project, subscription_name) + self.subscriber.subscribe(self.subscription_path, callback=callback) \ No newline at end of file From 67a9ad44580912791fd5b60322234dc411fbb446 Mon Sep 17 00:00:00 2001 From: moshe Date: Mon, 12 Nov 2018 13:46:13 +0200 Subject: [PATCH 179/209] typo --- tests/base_test_ci.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index bac1ae0..db36719 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -1,5 +1,5 @@ from dockers import Docker -from Topics import Publisher, Subscriber +from topics import Publisher, Subscriber import config import os import time From e266e42d69f6b29fb88b844adfe56f86c1376966 Mon Sep 17 00:00:00 2001 From: moshe Date: Mon, 12 Nov 2018 14:34:09 +0200 Subject: [PATCH 180/209] Migrate agent to external class as well --- tests/base_test_agent.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index 175beae..33f8e29 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -1,5 +1,6 @@ import config from dockers import Docker +from topics import Publisher, Subscriber from google.cloud import pubsub_v1 import time import datetime @@ -24,14 +25,11 @@ def __init__(self): self.establish_links() def establish_links(self): - project = config.CONFIG['project'] - topic_name_upstream = config.CONFIG['topic_name_upstream'] - self.publisher_upstream = pubsub_v1.PublisherClient() - self.topic_path_upstream = self.publisher_upstream.topic_path(project, topic_name_upstream) - - subscription_name_downstream = os.environ['SUBSCRIPTION_NAME_DOWNSTREAM'] - self.subscriber_downstream = pubsub_v1.SubscriberClient() - self.subscription_path_downstream = self.subscriber_downstream.subscription_path(project, subscription_name_downstream) + self.project = config.CONFIG['project'] + self.down_subscriber = Subscriber() + self.down_subscriber.subscribe(self.project, config.CONFIG['subscription_name_downstream'], self.callback) + self.up_publisher = Publisher() + self.up_publisher.enable(self.project, config.CONFIG['topic_name_upstreastream']) def start_node(self): logging.info('seeders:' + os.environ['SEEDERS']) @@ -75,14 +73,13 @@ def callback(self, message): self.docker.stop('node_' + self.node) def act_on_request(self): - self.subscriber_downstream.subscribe(self.subscription_path_downstream, callback=self.callback) while not self.endFlag: time.sleep(1) def send(self, data): logging.info("sent: " + str(data)) data = data.encode('utf-8') - self.publisher_upstream.publish(self.topic_path_upstream, data=data, phase=self.phase) + self.up_publisher.publish(data=data, phase=self.phase) def get_node_id(self): try: From e2d228f242299352194f9ae125071e76f9292f42 Mon Sep 17 00:00:00 2001 From: moshe Date: Mon, 12 Nov 2018 14:36:48 +0200 Subject: [PATCH 181/209] typo --- tests/base_test_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index 33f8e29..a821281 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -29,7 +29,7 @@ def establish_links(self): self.down_subscriber = Subscriber() self.down_subscriber.subscribe(self.project, config.CONFIG['subscription_name_downstream'], self.callback) self.up_publisher = Publisher() - self.up_publisher.enable(self.project, config.CONFIG['topic_name_upstreastream']) + self.up_publisher.enable(self.project, config.CONFIG['topic_name_upstream']) def start_node(self): logging.info('seeders:' + os.environ['SEEDERS']) From 5234e3e9511e5cf1190b1362f8673a1c39a4f617 Mon Sep 17 00:00:00 2001 From: moshe Date: Mon, 12 Nov 2018 14:43:55 +0200 Subject: [PATCH 182/209] fix --- tests/base_test_agent.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index a821281..fdbbfc8 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -27,7 +27,8 @@ def __init__(self): def establish_links(self): self.project = config.CONFIG['project'] self.down_subscriber = Subscriber() - self.down_subscriber.subscribe(self.project, config.CONFIG['subscription_name_downstream'], self.callback) + self.down_subscriber.subscribe(self.project, os.environ['SUBSCRIPTION_NAME_DOWNSTREAM'] +, self.callback) self.up_publisher = Publisher() self.up_publisher.enable(self.project, config.CONFIG['topic_name_upstream']) From ed44dc06948348afbc18e1e308cc4e7ff2157851 Mon Sep 17 00:00:00 2001 From: moshe Date: Mon, 12 Nov 2018 15:06:22 +0200 Subject: [PATCH 183/209] Create new subscription on demand --- tests/base_test_agent.py | 2 +- tests/base_test_ci.py | 2 +- tests/topics.py | 12 ++++++++++-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index fdbbfc8..8530eb5 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -27,7 +27,7 @@ def __init__(self): def establish_links(self): self.project = config.CONFIG['project'] self.down_subscriber = Subscriber() - self.down_subscriber.subscribe(self.project, os.environ['SUBSCRIPTION_NAME_DOWNSTREAM'] + self.down_subscriber.subscribe(self.project, config.CONFIG['topic_name_downstream'] , self.callback) self.up_publisher = Publisher() self.up_publisher.enable(self.project, config.CONFIG['topic_name_upstream']) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index db36719..fabc7c7 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -22,7 +22,7 @@ def setUp(self): self.project = config.CONFIG['project'] self.up_subscriber = Subscriber() - self.up_subscriber.subscribe(self.project, config.CONFIG['subscription_name_upstream'], self.callback) + self.up_subscriber.subscribe(self.project, config.CONFIG['topic_name_upstream'], self.callback) self.down_publisher = Publisher() self.down_publisher.enable(self.project, config.CONFIG['topic_name_downstream']) diff --git a/tests/topics.py b/tests/topics.py index 737e295..14e41b1 100644 --- a/tests/topics.py +++ b/tests/topics.py @@ -12,7 +12,15 @@ def publish(self, **kwargs): self.publisher.publish(self.topic_path, **kwargs) class Subscriber(): - def subscribe(self, project, subscription_name, callback): + def subscribe(self, project, topic_name, callback): + self.publisher = pubsub_v1.PublisherClient() + topic_path = self.publisher.topic_path(project, topic_name) + self.subscriber = pubsub_v1.SubscriberClient() + subscription_name = 'devnet_sub_' + str(random.randint(0, sys.maxint)) + ' ' + str(calendar.timegm(time.gmtime())) self.subscription_path = self.subscriber.subscription_path(project, subscription_name) - self.subscriber.subscribe(self.subscription_path, callback=callback) \ No newline at end of file + subscription = self.subscriber.create_subscription(self.subscription_path, topic_path) + self.subscriber.subscribe(self.subscription_path, callback=callback) + + def __del__(self): + self.subscriber.delete_subscription(self.subscription_path) \ No newline at end of file From 5ba2899812197c5d3a448b0efd2e21ccb6d90c71 Mon Sep 17 00:00:00 2001 From: moshe Date: Mon, 12 Nov 2018 15:08:44 +0200 Subject: [PATCH 184/209] fix --- tests/topics.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/topics.py b/tests/topics.py index 14e41b1..68812e7 100644 --- a/tests/topics.py +++ b/tests/topics.py @@ -1,4 +1,5 @@ import config +import random from google.cloud import pubsub_v1 import logging from logging import Logger From 78a5815a72fadde871d7105baedb40eb265ec81a Mon Sep 17 00:00:00 2001 From: moshe Date: Mon, 12 Nov 2018 15:10:09 +0200 Subject: [PATCH 185/209] fix --- tests/topics.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/topics.py b/tests/topics.py index 68812e7..665bc4f 100644 --- a/tests/topics.py +++ b/tests/topics.py @@ -1,5 +1,7 @@ import config import random +import calendar +import sys from google.cloud import pubsub_v1 import logging from logging import Logger From 0e512f3df1279702bab4f00ea7208fde11787f3b Mon Sep 17 00:00:00 2001 From: moshe Date: Mon, 12 Nov 2018 15:11:59 +0200 Subject: [PATCH 186/209] avoid sys in python3 --- tests/topics.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/topics.py b/tests/topics.py index 665bc4f..87ada1e 100644 --- a/tests/topics.py +++ b/tests/topics.py @@ -1,7 +1,6 @@ import config import random import calendar -import sys from google.cloud import pubsub_v1 import logging from logging import Logger @@ -20,7 +19,7 @@ def subscribe(self, project, topic_name, callback): topic_path = self.publisher.topic_path(project, topic_name) self.subscriber = pubsub_v1.SubscriberClient() - subscription_name = 'devnet_sub_' + str(random.randint(0, sys.maxint)) + ' ' + str(calendar.timegm(time.gmtime())) + subscription_name = 'devnet_sub_' + str(random.randint(0, 9999999999)) + ' ' + str(calendar.timegm(time.gmtime())) self.subscription_path = self.subscriber.subscription_path(project, subscription_name) subscription = self.subscriber.create_subscription(self.subscription_path, topic_path) self.subscriber.subscribe(self.subscription_path, callback=callback) From 2366bba75d5276f0b133d88c270fa6b10154aa69 Mon Sep 17 00:00:00 2001 From: moshe Date: Mon, 12 Nov 2018 15:12:43 +0200 Subject: [PATCH 187/209] fix --- tests/topics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/topics.py b/tests/topics.py index 87ada1e..899891d 100644 --- a/tests/topics.py +++ b/tests/topics.py @@ -1,6 +1,6 @@ import config import random -import calendar +import calendar, time from google.cloud import pubsub_v1 import logging from logging import Logger From 8f983acf6815f8d7bade6b69ab96be696dc31750 Mon Sep 17 00:00:00 2001 From: moshe Date: Mon, 12 Nov 2018 15:16:09 +0200 Subject: [PATCH 188/209] no spaces --- tests/topics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/topics.py b/tests/topics.py index 899891d..2725f94 100644 --- a/tests/topics.py +++ b/tests/topics.py @@ -19,7 +19,7 @@ def subscribe(self, project, topic_name, callback): topic_path = self.publisher.topic_path(project, topic_name) self.subscriber = pubsub_v1.SubscriberClient() - subscription_name = 'devnet_sub_' + str(random.randint(0, 9999999999)) + ' ' + str(calendar.timegm(time.gmtime())) + subscription_name = 'devnet_sub_' + str(random.randint(0, 9999999999)) + '_' + str(calendar.timegm(time.gmtime())) self.subscription_path = self.subscriber.subscription_path(project, subscription_name) subscription = self.subscriber.create_subscription(self.subscription_path, topic_path) self.subscriber.subscribe(self.subscription_path, callback=callback) From af4cda98b0d649929c49b619d5085fe49ae546ec Mon Sep 17 00:00:00 2001 From: moshe Date: Mon, 12 Nov 2018 15:28:52 +0200 Subject: [PATCH 189/209] force subscribers deletion --- tests/base_test_agent.py | 1 + tests/base_test_ci.py | 1 + tests/topics.py | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index 8530eb5..211d34a 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -63,6 +63,7 @@ def callback(self, message): if 'END' == self.message: self.docker.stop('node_' + self.node) + self.down_subscriber.delete() self.endFlag = True elif 'SEND_UP' == self.message: self.send('UP') diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index fabc7c7..d35e24f 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -32,6 +32,7 @@ def setUp(self): def tearDown(self): for self.phase in self.phases: self.send('END') + self.up_subscriber.delete() def callback(self, message): if self.phase != message.attributes['phase']: diff --git a/tests/topics.py b/tests/topics.py index 2725f94..66b5a2a 100644 --- a/tests/topics.py +++ b/tests/topics.py @@ -24,5 +24,5 @@ def subscribe(self, project, topic_name, callback): subscription = self.subscriber.create_subscription(self.subscription_path, topic_path) self.subscriber.subscribe(self.subscription_path, callback=callback) - def __del__(self): + def delete(self): self.subscriber.delete_subscription(self.subscription_path) \ No newline at end of file From 670856f5a268a129a6081a1d3754fea278e2cb2f Mon Sep 17 00:00:00 2001 From: moshe Date: Thu, 15 Nov 2018 00:35:58 +0200 Subject: [PATCH 190/209] Create topics and subscriptions and don't use pre created --- tests/base_test_agent.py | 8 +++----- tests/base_test_ci.py | 15 ++++++++------ tests/config.py | 4 ---- tests/topics.py | 44 ++++++++++++++++++++++++++++++++-------- 4 files changed, 48 insertions(+), 23 deletions(-) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index 211d34a..743533d 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -26,11 +26,9 @@ def __init__(self): def establish_links(self): self.project = config.CONFIG['project'] - self.down_subscriber = Subscriber() - self.down_subscriber.subscribe(self.project, config.CONFIG['topic_name_downstream'] -, self.callback) - self.up_publisher = Publisher() - self.up_publisher.enable(self.project, config.CONFIG['topic_name_upstream']) + self.down_subscriber = Subscriber(self.project, os.environ['SUBSCRIPTION_NAME_DOWNSTREAM']) + self.down_subscriber.subscribe(self.callback) + self.up_publisher = Publisher(self.project, os.environ['TOPIC_PATH_UPSTREAM']) def start_node(self): logging.info('seeders:' + os.environ['SEEDERS']) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index d35e24f..38e9e4a 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -21,10 +21,11 @@ def setUp(self): self.message = b'NULL' self.project = config.CONFIG['project'] - self.up_subscriber = Subscriber() - self.up_subscriber.subscribe(self.project, config.CONFIG['topic_name_upstream'], self.callback) - self.down_publisher = Publisher() - self.down_publisher.enable(self.project, config.CONFIG['topic_name_downstream']) + self.up_publisher = Publisher(self.project) + self.up_topic_path = self.up_publisher.create() + self.up_publisher.add_subscription().subscribe(self.callback) + self.down_publisher = Publisher(self.project) + self.down_publisher.create() self.agents = 0 self.messages = [] @@ -32,7 +33,8 @@ def setUp(self): def tearDown(self): for self.phase in self.phases: self.send('END') - self.up_subscriber.delete() + self.up_publisher.delete() + self.down_publisher.delete() def callback(self, message): if self.phase != message.attributes['phase']: @@ -68,7 +70,8 @@ def send_and_wait(self, data, nodes): def start_node_agent_pair(self, seeders=config.CONFIG['no_seeders'], bootstrap = 'false', randcon = 5): docker = Docker() docker.stop('agent_' + str(self.agents)) - cmd = 'docker run --network=devnet --name agent_' + str(self.agents) + ' -v /root/spacemesh/devnet/tests:/opt/devnet -v /root/spacemesh/devnet/logs' + str(self.agents) + ':/opt/logs -v /root/spacemesh/devnet/cnf' + str(self.agents) + ':/opt/cnf/ -e SUBSCRIPTION_NAME_DOWNSTREAM=devnet_tests_agent_' + str(self.agents) + ' -e PHASE=' + self.phase + ' -e BOOTSTRAP=' + bootstrap +' -e NODE=' + str(self.agents) + ' -e SEEDERS=' + seeders + ' -e RANDCON=' + str(randcon) + ' spacemesh/devnet_agent:latest python3 /opt/devnet/base_test_agent.py' + down_subscription = self.down_publisher.add_subscription() + cmd = 'docker run --network=devnet --name agent_' + str(self.agents) + ' -v /root/spacemesh/devnet/tests:/opt/devnet -v /root/spacemesh/devnet/logs' + str(self.agents) + ':/opt/logs -v /root/spacemesh/devnet/cnf' + str(self.agents) + ':/opt/cnf/ -e SUBSCRIPTION_NAME_DOWNSTREAM=' + down_subscription + ' -e TOPIC_PATH_UPSTREAM=' + self.up_topic_path + ' -e PHASE=' + self.phase + ' -e BOOTSTRAP=' + bootstrap +' -e NODE=' + str(self.agents) + ' -e SEEDERS=' + seeders + ' -e RANDCON=' + str(randcon) + ' spacemesh/devnet_agent:latest python3 /opt/devnet/base_test_agent.py' docker.start(cmd) self.agents += 1 diff --git a/tests/config.py b/tests/config.py index 1f2da9f..a1b9c00 100644 --- a/tests/config.py +++ b/tests/config.py @@ -1,9 +1,5 @@ CONFIG = { 'project': 'spacemesh-198810', - 'topic_name_upstream': 'devnet_tests', - 'subscription_name_upstream': 'devnet_tests_ci', - 'topic_name_downstream': 'devnet_tests_downstream', - 'subscription_name_downstream': 'devnet_tests_agent', 'host': '127.0.0.1', 'host_user': 'deploy', 'host_password': 'deploy_password', diff --git a/tests/topics.py b/tests/topics.py index 66b5a2a..d2a80e4 100644 --- a/tests/topics.py +++ b/tests/topics.py @@ -6,22 +6,50 @@ from logging import Logger class Publisher(): - def enable(self, project, topic_name): + def __init__(self, project, topic_path = ''): + self.project = project + self.topic_path = topic_path self.publisher = pubsub_v1.PublisherClient() - self.topic_path = self.publisher.topic_path(project, topic_name) + + def create(self): + self.subscribers = [] + topic_name = 'devnet_topic_' + str(random.randint(0, 9999999999)) + '_' + str(calendar.timegm(time.gmtime())) + self.topic_path = self.publisher.topic_path(self.project, topic_name) + publisher.create_topic(self.topic_path) + return self.topic_path def publish(self, **kwargs): self.publisher.publish(self.topic_path, **kwargs) + def add_subscription(self): + subscriber = Subscriber(self.project) + subscriber.create_for_publisher(self.topic_path) + self.subscribers.append(subscriber) + return subscriber + + def delete(self): + for s in self.subscribers: + s.delete() + self.publisher.delete_topic(self.topic_path) + class Subscriber(): - def subscribe(self, project, topic_name, callback): + def __init__(self, project, subscription_path = ''): + self.project = project + self.subscription_path = subscription_path + self.subscriber = pubsub_v1.SubscriberClient() + + def create(self, topic_name): self.publisher = pubsub_v1.PublisherClient() - topic_path = self.publisher.topic_path(project, topic_name) + topic_path = self.publisher.topic_path(self.project, topic_name) + return self.create_for_publisher(self, topic_path) - self.subscriber = pubsub_v1.SubscriberClient() - subscription_name = 'devnet_sub_' + str(random.randint(0, 9999999999)) + '_' + str(calendar.timegm(time.gmtime())) - self.subscription_path = self.subscriber.subscription_path(project, subscription_name) - subscription = self.subscriber.create_subscription(self.subscription_path, topic_path) + def create_for_publisher(self, topic_path): + self.subscription_name = 'devnet_sub_' + str(random.randint(0, 9999999999)) + '_' + str(calendar.timegm(time.gmtime())) + self.subscription_path = self.subscriber.subscription_path(self.project, self.subscription_name) + self.subscriber.create_subscription(self.subscription_path, topic_path) + return self.subscription_name + + def subscribe(self, callback): self.subscriber.subscribe(self.subscription_path, callback=callback) def delete(self): From 862b86a0fdeec57fc97ed3cb43d81766c00f8fe7 Mon Sep 17 00:00:00 2001 From: moshe Date: Thu, 15 Nov 2018 00:44:55 +0200 Subject: [PATCH 191/209] typo --- tests/topics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/topics.py b/tests/topics.py index d2a80e4..aa2f13e 100644 --- a/tests/topics.py +++ b/tests/topics.py @@ -15,7 +15,7 @@ def create(self): self.subscribers = [] topic_name = 'devnet_topic_' + str(random.randint(0, 9999999999)) + '_' + str(calendar.timegm(time.gmtime())) self.topic_path = self.publisher.topic_path(self.project, topic_name) - publisher.create_topic(self.topic_path) + self.publisher.create_topic(self.topic_path) return self.topic_path def publish(self, **kwargs): From 9ee73ff24b7404f2c2148def7f3107df497f16d0 Mon Sep 17 00:00:00 2001 From: moshe Date: Thu, 15 Nov 2018 00:52:45 +0200 Subject: [PATCH 192/209] Rename and fix missing topic path --- tests/base_test_ci.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index 38e9e4a..5e27a69 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -70,8 +70,8 @@ def send_and_wait(self, data, nodes): def start_node_agent_pair(self, seeders=config.CONFIG['no_seeders'], bootstrap = 'false', randcon = 5): docker = Docker() docker.stop('agent_' + str(self.agents)) - down_subscription = self.down_publisher.add_subscription() - cmd = 'docker run --network=devnet --name agent_' + str(self.agents) + ' -v /root/spacemesh/devnet/tests:/opt/devnet -v /root/spacemesh/devnet/logs' + str(self.agents) + ':/opt/logs -v /root/spacemesh/devnet/cnf' + str(self.agents) + ':/opt/cnf/ -e SUBSCRIPTION_NAME_DOWNSTREAM=' + down_subscription + ' -e TOPIC_PATH_UPSTREAM=' + self.up_topic_path + ' -e PHASE=' + self.phase + ' -e BOOTSTRAP=' + bootstrap +' -e NODE=' + str(self.agents) + ' -e SEEDERS=' + seeders + ' -e RANDCON=' + str(randcon) + ' spacemesh/devnet_agent:latest python3 /opt/devnet/base_test_agent.py' + down_subscriber = self.down_publisher.add_subscription() + cmd = 'docker run --network=devnet --name agent_' + str(self.agents) + ' -v /root/spacemesh/devnet/tests:/opt/devnet -v /root/spacemesh/devnet/logs' + str(self.agents) + ':/opt/logs -v /root/spacemesh/devnet/cnf' + str(self.agents) + ':/opt/cnf/ -e SUBSCRIPTION_PATH_DOWNSTREAM=' + down_subscriber.topic_path + ' -e TOPIC_PATH_UPSTREAM=' + self.up_topic_path + ' -e PHASE=' + self.phase + ' -e BOOTSTRAP=' + bootstrap +' -e NODE=' + str(self.agents) + ' -e SEEDERS=' + seeders + ' -e RANDCON=' + str(randcon) + ' spacemesh/devnet_agent:latest python3 /opt/devnet/base_test_agent.py' docker.start(cmd) self.agents += 1 From d6484b705acb091233a3af7768720fc75a3962e1 Mon Sep 17 00:00:00 2001 From: moshe Date: Thu, 15 Nov 2018 00:55:13 +0200 Subject: [PATCH 193/209] typo --- tests/base_test_agent.py | 2 +- tests/base_test_ci.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index 743533d..a23da25 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -26,7 +26,7 @@ def __init__(self): def establish_links(self): self.project = config.CONFIG['project'] - self.down_subscriber = Subscriber(self.project, os.environ['SUBSCRIPTION_NAME_DOWNSTREAM']) + self.down_subscriber = Subscriber(self.project, os.environ['SUBSCRIPTION_PATH_DOWNSTREAM']) self.down_subscriber.subscribe(self.callback) self.up_publisher = Publisher(self.project, os.environ['TOPIC_PATH_UPSTREAM']) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index 5e27a69..645a63e 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -71,7 +71,7 @@ def start_node_agent_pair(self, seeders=config.CONFIG['no_seeders'], bootstrap = docker = Docker() docker.stop('agent_' + str(self.agents)) down_subscriber = self.down_publisher.add_subscription() - cmd = 'docker run --network=devnet --name agent_' + str(self.agents) + ' -v /root/spacemesh/devnet/tests:/opt/devnet -v /root/spacemesh/devnet/logs' + str(self.agents) + ':/opt/logs -v /root/spacemesh/devnet/cnf' + str(self.agents) + ':/opt/cnf/ -e SUBSCRIPTION_PATH_DOWNSTREAM=' + down_subscriber.topic_path + ' -e TOPIC_PATH_UPSTREAM=' + self.up_topic_path + ' -e PHASE=' + self.phase + ' -e BOOTSTRAP=' + bootstrap +' -e NODE=' + str(self.agents) + ' -e SEEDERS=' + seeders + ' -e RANDCON=' + str(randcon) + ' spacemesh/devnet_agent:latest python3 /opt/devnet/base_test_agent.py' + cmd = 'docker run --network=devnet --name agent_' + str(self.agents) + ' -v /root/spacemesh/devnet/tests:/opt/devnet -v /root/spacemesh/devnet/logs' + str(self.agents) + ':/opt/logs -v /root/spacemesh/devnet/cnf' + str(self.agents) + ':/opt/cnf/ -e SUBSCRIPTION_PATH_DOWNSTREAM=' + down_subscriber.subscription_path + ' -e TOPIC_PATH_UPSTREAM=' + self.up_topic_path + ' -e PHASE=' + self.phase + ' -e BOOTSTRAP=' + bootstrap +' -e NODE=' + str(self.agents) + ' -e SEEDERS=' + seeders + ' -e RANDCON=' + str(randcon) + ' spacemesh/devnet_agent:latest python3 /opt/devnet/base_test_agent.py' docker.start(cmd) self.agents += 1 From 4b10683d08c81c923591734f546916a6c634ac3a Mon Sep 17 00:00:00 2001 From: moshe Date: Thu, 15 Nov 2018 01:10:57 +0200 Subject: [PATCH 194/209] should be <= to defined DHT size --- tests/test_2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_2.py b/tests/test_2.py index 1e8c639..23756fa 100644 --- a/tests/test_2.py +++ b/tests/test_2.py @@ -24,7 +24,7 @@ def test_sendId(self): messages = self.send_and_wait('GET_DHT_SIZE', testers_nodes) for i in range(0, testers_nodes): - self.assertLess(randcon, int(messages[i])) + self.assertLessEqual(randcon, int(messages[i])) if __name__ == '__main__': unittest.main() \ No newline at end of file From 1a06425b5b0aa0604e262e9ad4ebde8abedc3a3c Mon Sep 17 00:00:00 2001 From: moshe Date: Thu, 15 Nov 2018 01:17:41 +0200 Subject: [PATCH 195/209] refix --- tests/base_test_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index 90e22cf..175beae 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -96,7 +96,7 @@ def get_node_id(self): return 'node_' + self.node + ':' + str(self.node_port) + '/' + self.node_id def get_dht_size(self): - pattern = re.compile(u'.*DHT State with (\d+)') + pattern = re.compile(u'.*DHT Bootstrapped with (\d+)') for i in range(0, self.dht_timeout): for line in open('/opt/logs/' + self.node_id + '/node.log', "r", encoding="utf-8"): results = pattern.match(line) From eb6dd35ec6fbb663931137bb1fc9852a3b254522 Mon Sep 17 00:00:00 2001 From: moshe Date: Thu, 15 Nov 2018 01:26:41 +0200 Subject: [PATCH 196/209] deleted in the ci side --- tests/base_test_agent.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index a23da25..8b6a926 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -61,7 +61,6 @@ def callback(self, message): if 'END' == self.message: self.docker.stop('node_' + self.node) - self.down_subscriber.delete() self.endFlag = True elif 'SEND_UP' == self.message: self.send('UP') From 877c7777435def94a6cb806cb49fae6202bf8316 Mon Sep 17 00:00:00 2001 From: moshe Date: Thu, 15 Nov 2018 01:47:34 +0200 Subject: [PATCH 197/209] Recover old tests --- tests/tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/tests.py b/tests/tests.py index 9a51fe6..054577b 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -1,6 +1,6 @@ import unittest -#from test_0 import Test0 -#from test_1 import Test1 +from test_0 import Test0 +from test_1 import Test1 from test_2 import Test2 if __name__ == '__main__': From c9eadce2bec8f79676e3b2029f0e3147ca118188 Mon Sep 17 00:00:00 2001 From: moshe Date: Thu, 15 Nov 2018 01:49:22 +0200 Subject: [PATCH 198/209] fit test 0 + 1 to new method --- tests/test_0.py | 3 ++- tests/test_1.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_0.py b/tests/test_0.py index 75b6543..c3dcd37 100644 --- a/tests/test_0.py +++ b/tests/test_0.py @@ -7,7 +7,8 @@ class Test0(BaseTest): def test_verifyUp(self): - messages = self.run_phase(nodes = 1, bootstrap = 'false', message = 'SEND_UP') + messages = self.run_phase(nodes = 1, bootstrap = 'false') + messages = self.send_and_wait('GET_DHT_SIZE', testers_nodes) self.assertEqual('UP', messages[0]) if __name__ == '__main__': diff --git a/tests/test_1.py b/tests/test_1.py index 82da4d8..e0945a5 100644 --- a/tests/test_1.py +++ b/tests/test_1.py @@ -7,7 +7,7 @@ class Test1(BaseTest): def test_sendId(self): - messages = self.run_phase(nodes = 1, bootstrap = 'false', message = 'GET_NODE_ID') + messages = self.run_phase(nodes = 1, bootstrap = 'false') self.assertLess(15, len(messages[0])) if __name__ == '__main__': From 18c2e6a18ede0f289809a3499bf4ae59d165ee10 Mon Sep 17 00:00:00 2001 From: moshe Date: Thu, 15 Nov 2018 01:53:15 +0200 Subject: [PATCH 199/209] fix --- tests/test_0.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_0.py b/tests/test_0.py index c3dcd37..58d4da1 100644 --- a/tests/test_0.py +++ b/tests/test_0.py @@ -7,7 +7,8 @@ class Test0(BaseTest): def test_verifyUp(self): - messages = self.run_phase(nodes = 1, bootstrap = 'false') + testers_nodes = 1 + messages = self.run_phase(nodes = testers_nodes, bootstrap = 'false') messages = self.send_and_wait('GET_DHT_SIZE', testers_nodes) self.assertEqual('UP', messages[0]) From fdd21e268f726e5a7ee3f8b66156b62f83357eaf Mon Sep 17 00:00:00 2001 From: moshe Date: Thu, 15 Nov 2018 01:54:11 +0200 Subject: [PATCH 200/209] typo --- tests/test_0.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_0.py b/tests/test_0.py index 58d4da1..d984887 100644 --- a/tests/test_0.py +++ b/tests/test_0.py @@ -9,7 +9,7 @@ class Test0(BaseTest): def test_verifyUp(self): testers_nodes = 1 messages = self.run_phase(nodes = testers_nodes, bootstrap = 'false') - messages = self.send_and_wait('GET_DHT_SIZE', testers_nodes) + messages = self.send_and_wait('SEND_UP', testers_nodes) self.assertEqual('UP', messages[0]) if __name__ == '__main__': From 8d3116a669b7afc24bf399325f9d9e5dbf63ab95 Mon Sep 17 00:00:00 2001 From: moshe Date: Thu, 15 Nov 2018 01:47:34 +0200 Subject: [PATCH 201/209] Recover old tests --- tests/tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/tests.py b/tests/tests.py index 9a51fe6..054577b 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -1,6 +1,6 @@ import unittest -#from test_0 import Test0 -#from test_1 import Test1 +from test_0 import Test0 +from test_1 import Test1 from test_2 import Test2 if __name__ == '__main__': From d6b203635040857d22553346b341fd7e3419c7a6 Mon Sep 17 00:00:00 2001 From: moshe Date: Thu, 15 Nov 2018 01:49:22 +0200 Subject: [PATCH 202/209] fit test 0 + 1 to new method --- tests/test_0.py | 3 ++- tests/test_1.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_0.py b/tests/test_0.py index 75b6543..c3dcd37 100644 --- a/tests/test_0.py +++ b/tests/test_0.py @@ -7,7 +7,8 @@ class Test0(BaseTest): def test_verifyUp(self): - messages = self.run_phase(nodes = 1, bootstrap = 'false', message = 'SEND_UP') + messages = self.run_phase(nodes = 1, bootstrap = 'false') + messages = self.send_and_wait('GET_DHT_SIZE', testers_nodes) self.assertEqual('UP', messages[0]) if __name__ == '__main__': diff --git a/tests/test_1.py b/tests/test_1.py index 82da4d8..e0945a5 100644 --- a/tests/test_1.py +++ b/tests/test_1.py @@ -7,7 +7,7 @@ class Test1(BaseTest): def test_sendId(self): - messages = self.run_phase(nodes = 1, bootstrap = 'false', message = 'GET_NODE_ID') + messages = self.run_phase(nodes = 1, bootstrap = 'false') self.assertLess(15, len(messages[0])) if __name__ == '__main__': From 1e636d4443beba4adf81aceadbba0c1581625d6e Mon Sep 17 00:00:00 2001 From: moshe Date: Thu, 15 Nov 2018 01:53:15 +0200 Subject: [PATCH 203/209] fix --- tests/test_0.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_0.py b/tests/test_0.py index c3dcd37..58d4da1 100644 --- a/tests/test_0.py +++ b/tests/test_0.py @@ -7,7 +7,8 @@ class Test0(BaseTest): def test_verifyUp(self): - messages = self.run_phase(nodes = 1, bootstrap = 'false') + testers_nodes = 1 + messages = self.run_phase(nodes = testers_nodes, bootstrap = 'false') messages = self.send_and_wait('GET_DHT_SIZE', testers_nodes) self.assertEqual('UP', messages[0]) From 7b84f40da08348f6df8ef80563881adb3ca8502d Mon Sep 17 00:00:00 2001 From: moshe Date: Thu, 15 Nov 2018 01:54:11 +0200 Subject: [PATCH 204/209] typo --- tests/test_0.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_0.py b/tests/test_0.py index 58d4da1..d984887 100644 --- a/tests/test_0.py +++ b/tests/test_0.py @@ -9,7 +9,7 @@ class Test0(BaseTest): def test_verifyUp(self): testers_nodes = 1 messages = self.run_phase(nodes = testers_nodes, bootstrap = 'false') - messages = self.send_and_wait('GET_DHT_SIZE', testers_nodes) + messages = self.send_and_wait('SEND_UP', testers_nodes) self.assertEqual('UP', messages[0]) if __name__ == '__main__': From 1477ee9c048f1df36c71468c8c1d3b374eb55198 Mon Sep 17 00:00:00 2001 From: anton Date: Thu, 29 Nov 2018 13:47:34 +0200 Subject: [PATCH 205/209] basic building blocks --- google_pubsub.py | 31 +++++++++++++++++++++++ google_pubsub.pyc | Bin 0 -> 1996 bytes pubsub.py | 20 +++++++++++++++ pubsub.pyc | Bin 0 -> 1223 bytes runner.py | 18 +++++++++++++ secret.json | 12 +++++++++ test_runner.py | 63 ++++++++++++++++++++++++++++++++++++++++++++++ tools.py | 60 +++++++++++++++++++++++++++++++++++++++++++ 8 files changed, 204 insertions(+) create mode 100644 google_pubsub.py create mode 100644 google_pubsub.pyc create mode 100644 pubsub.py create mode 100644 pubsub.pyc create mode 100644 runner.py create mode 100644 secret.json create mode 100644 test_runner.py create mode 100644 tools.py diff --git a/google_pubsub.py b/google_pubsub.py new file mode 100644 index 0000000..8f95105 --- /dev/null +++ b/google_pubsub.py @@ -0,0 +1,31 @@ +from pubsub import PubSub +from google.cloud import pubsub_v1 + + +class GooglePubSub(PubSub): + def __init__(self): + self.publisher = pubsub_v1.PublisherClient() + self.subscriber = pubsub_v1.SubscriberClient() + self.project = "spacemesh-198810" + + def Subscribe(self, channel_name, callback): + topic_path = self.publisher.topic_path(self.project, channel_name) + subscription_path = self.subscriber.subscription_path(self.project, "kaplan") + print subscription_path + self.subscriber.create_subscription(subscription_path, topic_path) + self.subscriber.subscribe(subscription_path, callback=callback) + + def Publish(self, channel_name, **kwargs): + self.publisher.publish(channel_name, **kwargs) + + def UnSubscribe(self, channel_name): + self.subscriber.delete_subscription(channel_name) + + def CreateChannel(self, channel_name): + topic_path = self.publisher.topic_path(self.project, topic_name) + self.publisher.create_topic(topic_path) + return tpoic_path + + + def DeleteChannel(self, channel_name): + self.publisher.delete_topic(channel_name) \ No newline at end of file diff --git a/google_pubsub.pyc b/google_pubsub.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6b366e5b12181951b5a0f451323561cd126943c6 GIT binary patch literal 1996 zcmb_dO>fgc5S_J?loYB4%14C|=Sa|o-U_NJL@E^tDMF5VARH{m*(OdM+sbaJkkV7R z@~8Mq{D3lV)=i^|3k`90GPCR5ee>q+#{a#s{_EiCYfR`D)fWslITmXLW3%Gn>C)ET=A!=*s};f&-&e2-^F!~i&@{y z`r)@d`+%e4JRhdoy|X}rxgWThLpGWKf-&$`JrBs>_<#h5H}=J;mT-DCK*7)&1rAY+ z1<^ciZEofj8%s1+KtCr6Jqi8mkXKdG&#MyWg^3;0z|1mUHNsQTH zuXSSd)U=|^=2`0~KhM%U8gM-1ZCgX3HFR9>7T@hXo)Lygme?>vRSXQNO@3mZy(%|s zB<%FrVN%9I5FC%@%#=qLzV!Pvu81S$KtG2SI{t`vd9aFT&SZL|%B~u^xMXvWE|r3$ zx=rlc5sjQhMaqc)WE`bwKZ?(q;2{7c3xaH3Bym_ob`;0};M<36O6CfiHo~}=JJ62ucdu)Ubq$gswarge zZK)W-0uN}PxS3~cZn$R=di2ws!LG=mBU*ICh*X^5giOgNfXE!H=h1X%R>F@RN=ZKg zI>04|8ZMeB<}o|Vs4xElSxScD40Nh*6~`roatQpDFoWikY{B{y4wrFXvn{w^^WY6` zQMwh~Ua(X-Wrz)pH(XJaW@>0dO>Lu%}lr^N< zmi^jJ%iG*hp7L$gxMR)a{NX5+HIr2SXR!eLNc2#`ehJu4W=-g!yt%+>7z&ADILQaI zl*0hWE5I!fK!CwhnCCDr*xb-5-oKyT05EV*7%)&%^=f^s?(UsORjlD?+=xA5p8H(@ z5?Mf!nT*HdH_tP6{k_+ny?s0;w0Y|Mzh*J#tWy5!lzQ|^^g{Fy^0-DLzrl01#p=F{ zM`gTQ?6K>H#q?PzqR;~bFAi%Pl#t5Mjskq`I;fLj>=7?~Mz zP<-}AY(`FEEd7hamcpXv!ogb@&09PePrCR=D#u)2eIZ*CD z=g{M%gQbeB8mv2g`d=)DDpxxeXL}YtPV?qS;}lA1yX=v2UZiEtL*aVSWS>As1>yx7 pDiF7CjkA7njRdbIU>kI%<#p=Y_u!_wpz1l?zOL3g>APG=`UxsP;QIgo literal 0 HcmV?d00001 diff --git a/runner.py b/runner.py new file mode 100644 index 0000000..31999e2 --- /dev/null +++ b/runner.py @@ -0,0 +1,18 @@ +from google_pubsub import GooglePubSub +import time +import pubsub + + +def PubSubFactory(provider): + if provider == "google": + return GooglePubSub() + + +def printlogs(message): + print message + +if __name__ == '__main__': + pubsub = PubSubFactory("google") + pubsub.Subscribe("devnet_tests_downstream", printlogs) + while True: + time.sleep(1) diff --git a/secret.json b/secret.json new file mode 100644 index 0000000..72dc33b --- /dev/null +++ b/secret.json @@ -0,0 +1,12 @@ +{ + "type": "service_account", + "project_id": "spacemesh-198810", + "private_key_id": "cb0c8f84b56329a953ab4a5492a10fbb58c9d7a3", + "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCwzHvxIyJF9Vzn\nZxEt1ve/n9ioN5RRX2gylIjykLuHC9BVCAGKr/QBTvW4jO8+3sq+Is+9HeZhH+k1\ns5+YTEmHP0LUTg8wRymeklu2XrYXjBd5H9uoKpjnA70cqnMQT44Ea5oTEUOfM0WJ\n6+1q3Ut2V7RECuL1ZiV+s2yDjRVqKbQ2td/n0vhcq8h5xUqmKwaPwSr8WwkCKZNz\nbPd+ir8gPBu1gQg4RUFBEAZ1EbcmELNjJ2nEtbdZMiHRRsnqhl5UKY3sLc8HCPEs\n5CEcQAbmMrSIUs4kZA90vIy/xu2BXx1RjupDOdBpui3k9DPyRt1KQNi7kVLY9CGZ\ngKyw3a/bAgMBAAECggEAIrccNSatcagypFtoE6SsxAs6ixlSvcGqn5fr5hlGQUew\nEio+x0Eye9lJxYGi7qjnjXsfaGaJGJQgAkPxeAGuxGhHEpH7SIMIqJLbgE1LmwcO\niuGXC4fFu0LQLONJMbA1Vifcupa+vPFf26ugVEX8rdGd22h51S7H+P95NMGMrWoM\nZJxmbGNms+mNvd2PsRLP3z8wt+PUexbsfCN26G06gDTZq1mF3avJS/xs8AiDjaXq\nD+DfQ9sW7cRBAimXlBihpGYSkk7vT62qbeTswfUsSmKZWeG/tlN7rZFNufgiOOfp\ntUedDhyS9nCMWaLzIvz/AQTGdkKtFPLR2Huswj5wDQKBgQDWBiHACYV7sEYXJbSp\nOjckCPCuxyJ93Y/7qvmriLZDVUuV+Y9UDWcxvH/WIi1Al0uEQ2cvnbyOyrVQVIqi\nrCW3DPi0lKi/gxEkChCDbt4DLF158DNSc69KNsPsKrlvH1pHZ9YeIgNVkvbTqcpN\nj/1Q01fBgrdY+cnnmPu73Rlb9QKBgQDTeVLNqZuyVwdqBYd5MbvyAKvIRO0FMQFn\n5eTRelkdf6fIvdcU+bo0AdGH1hdedBVuMM+XkVjPk14oKkEd7gNwANfOXO5BWLhJ\n/yP5DPZ3E9LTtxXVOsX6ckMA3I2I3bG/eqtzvP8dWVWIzFc7XERRHKDJvGyxrPSj\nMWrORhTKjwKBgE2VSXTAvteYjb/kVjomNUjlyMHCxBkWURRs5TauR0vfRpgmMDPK\n7AaWi0zHdNn+PkzC1kvnhSvKBbIKA3t2pfFWTW3Ame9ahsMmtte+P9KsqXIDhcc0\n0m03rwwUbMYAeox58p9XuxexP7J40TIOqUg8ZIpsZSRm4C8JQO/X9VJlAoGAbSBv\nXVfH05jbZ5eg2CaVAQXdn4QJ96LZQd89W+fR8sn8MDfetKFFuC60DG+rMX+fy3P+\nvA0yEjPO49UiMyqutW6J7qDABhKdTvoNeyh4Sux83+nbp2Qj4+BwuRIXK2iTn+bS\ncPA3idKes/AWl7g+PMkUI+ZLBMjmmQwYPIenlPsCgYEAqrh9Hx/YkjJTsILk9SDt\nkXLqlxVlr35vW6qJ2whH5Ywh2gyFgMDDOyjuV/rrky5mxbGKeNvEy5lY4vyyF25n\ntAaJwQfepz8L+TtEwZDQEPUf1KvixmhgDWaqcKh/gygleEewPvbpWVQZ/aTeubyc\n3DWfUuOIbeLTQXAVMt4yngE=\n-----END PRIVATE KEY-----\n", + "client_email": "spacemesh-198810@appspot.gserviceaccount.com", + "client_id": "", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/spacemesh-198810%40appspot.gserviceaccount.com" +} diff --git a/test_runner.py b/test_runner.py new file mode 100644 index 0000000..f59d9c4 --- /dev/null +++ b/test_runner.py @@ -0,0 +1,63 @@ +import os +from kubernetes import client, config + +DEPLOYMENT_NAME = "lalaa" +os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "secret.json" + +def create_deployment_object(): + # Configureate Pod template container + container = client.V1Container(name="lala", image="gcr.io/spacemesh-198810/node:46f1ad099cdfefc65dda5f726d26ecea7b1f35fb", ports=[client.V1ContainerPort(container_port=9091)]) + # Create and configurate a spec section + template = client.V1PodTemplateSpec(metadata=client.V1ObjectMeta(labels={"app": "nginx"}), spec=client.V1PodSpec(containers=[container])) + # Create the specification of deployment + spec = client.ExtensionsV1beta1DeploymentSpec(replicas=1, template=template) + # Instantiate the deployment object + deployment = client.ExtensionsV1beta1Deployment(api_version="extensions/v1beta1", kind="Deployment",metadata=client.V1ObjectMeta(name=DEPLOYMENT_NAME),spec=spec) + return deployment + +def create_deployment(api_instance, deployment): +# Create deployement + api_response = api_instance.create_namespaced_deployment(body=deployment,namespace="default") + print("Deployment created. status='%s'" % str(api_response.status)) + +def update_deployment(api_instance, deployment): + # Update container image + deployment.spec.template.spec.containers[0].image = "nginx:1.9.1" + # Update the deployment + api_response = api_instance.patch_namespaced_deployment( + name=DEPLOYMENT_NAME, + namespace="default", + body=deployment) + print("Deployment updated. status='%s'" % str(api_response.status)) + +def delete_deployment(api_instance): +# Delete deployment + api_response = api_instance.delete_namespaced_deployment( + name=DEPLOYMENT_NAME, + namespace="default", + body=client.V1DeleteOptions(propagation_policy='Foreground', grace_period_seconds=5)) + print("Deployment deleted. status='%s'" % str(api_response.status)) + +def list_all(): + config_file = os.path.join(os.path.expanduser('~'), '.kube', 'config') + contexts, active_context = config.list_kube_config_contexts(config_file) + if not contexts: + print("Cannot find any context in kube-config file.") + return + contextsNames = [context['name'] for context in contexts] + for ctx in contextsNames: + v1 = client.CoreV1Api(config.new_client_from_config(config_file, ctx)) + print("cluster: " + ctx + " Listing pods with their IPs:") + ret = v1.list_namespaced_pod("default") + for item in ret.items: + print("%s\t%s\t%s" % (item.status.pod_ip, item.metadata.namespace, item.metadata.name)) + +if __name__ == '__main__': + list_all() + config.load_kube_config() + extensions_v1beta1 = client.ExtensionsV1beta1Api() + deployment = create_deployment_object() + create_deployment(extensions_v1beta1, deployment) + list_all() + update_deployment(extensions_v1beta1, deployment) + #delete_deployment(extensions_v1beta1) \ No newline at end of file diff --git a/tools.py b/tools.py new file mode 100644 index 0000000..8bf7bad --- /dev/null +++ b/tools.py @@ -0,0 +1,60 @@ +import sys +import requests + +def getRPCPort(advclient, cont): + inspect = advclient.inspect_container(cont.name) + rpcport = inspect['NetworkSettings']['Ports']['9090/tcp'][0]['HostPort'] + return rpcport + +def waitForRPC(cont): + for l in cont.logs(stream=True, stdout=True): + if "Started GRPC" in l: + return + +def registerProtocol(advclient, cont, proto, port): + rpcport = getRPCPort(advclient, cont) + print "Register on " + "http://127.0.0.1:" + str(rpcport) + "/v1/register" + r = requests.post("http://127.0.0.1:" + str(rpcport) + "/v1/register", json={ "name": proto, "port": port }) + print("Registerd protocol response ", r.status_code, r.reason, r.text) + + +def broadcast(advclient, cont, proto, message): + rpcport = getRPCPort(advclient, cont) + print "Sending message to " + "http://127.0.0.1:" + str(rpcport) + "/v1/broadcast" + r = requests.post("http://127.0.0.1:" + str(rpcport) + "/v1/broadcast", json={ "protocolName": proto, "payload": [10, 10, 10] }) + print("Send broadcast, response ", r.status_code, r.reason, r.text) + +def getExternalIP(advclient, cont): + ip = "" + nets = advclient.inspect_container(cont.name)['NetworkSettings']['Networks'] + for net in nets: + ip = nets[net]["IPAddress"] + if ip is not "": + print ip + break + return ip + +def getPublicKey(cont): + id = "" + for l in cont.logs(stream=True, stdout=True): + if "identity" in l: + id = l.split('>>')[1].strip() + break + return id + +def cleanUp(contlist): + ### Kill all dockers with nice bar + toolbar_width = len(contlist) + print "Killing all dockers" + sys.stdout.write("[%s]" % (" " * toolbar_width)) + sys.stdout.flush() + sys.stdout.write("\b" * (toolbar_width+1)) # return to start of line, after '[' + for i in contlist: + cont = i + if isinstance(i, dict): + cont = cont["cont"] + cont.kill() + # update the bar + sys.stdout.write("-") + sys.stdout.flush() + sys.stdout.write("\n") \ No newline at end of file From 74b26aafedd5b3c9ebc25118b6842b7b7caaccfe Mon Sep 17 00:00:00 2001 From: anton Date: Thu, 29 Nov 2018 13:47:34 +0200 Subject: [PATCH 206/209] basic building blocks --- secret.json | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100644 secret.json diff --git a/secret.json b/secret.json deleted file mode 100644 index 72dc33b..0000000 --- a/secret.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "type": "service_account", - "project_id": "spacemesh-198810", - "private_key_id": "cb0c8f84b56329a953ab4a5492a10fbb58c9d7a3", - "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCwzHvxIyJF9Vzn\nZxEt1ve/n9ioN5RRX2gylIjykLuHC9BVCAGKr/QBTvW4jO8+3sq+Is+9HeZhH+k1\ns5+YTEmHP0LUTg8wRymeklu2XrYXjBd5H9uoKpjnA70cqnMQT44Ea5oTEUOfM0WJ\n6+1q3Ut2V7RECuL1ZiV+s2yDjRVqKbQ2td/n0vhcq8h5xUqmKwaPwSr8WwkCKZNz\nbPd+ir8gPBu1gQg4RUFBEAZ1EbcmELNjJ2nEtbdZMiHRRsnqhl5UKY3sLc8HCPEs\n5CEcQAbmMrSIUs4kZA90vIy/xu2BXx1RjupDOdBpui3k9DPyRt1KQNi7kVLY9CGZ\ngKyw3a/bAgMBAAECggEAIrccNSatcagypFtoE6SsxAs6ixlSvcGqn5fr5hlGQUew\nEio+x0Eye9lJxYGi7qjnjXsfaGaJGJQgAkPxeAGuxGhHEpH7SIMIqJLbgE1LmwcO\niuGXC4fFu0LQLONJMbA1Vifcupa+vPFf26ugVEX8rdGd22h51S7H+P95NMGMrWoM\nZJxmbGNms+mNvd2PsRLP3z8wt+PUexbsfCN26G06gDTZq1mF3avJS/xs8AiDjaXq\nD+DfQ9sW7cRBAimXlBihpGYSkk7vT62qbeTswfUsSmKZWeG/tlN7rZFNufgiOOfp\ntUedDhyS9nCMWaLzIvz/AQTGdkKtFPLR2Huswj5wDQKBgQDWBiHACYV7sEYXJbSp\nOjckCPCuxyJ93Y/7qvmriLZDVUuV+Y9UDWcxvH/WIi1Al0uEQ2cvnbyOyrVQVIqi\nrCW3DPi0lKi/gxEkChCDbt4DLF158DNSc69KNsPsKrlvH1pHZ9YeIgNVkvbTqcpN\nj/1Q01fBgrdY+cnnmPu73Rlb9QKBgQDTeVLNqZuyVwdqBYd5MbvyAKvIRO0FMQFn\n5eTRelkdf6fIvdcU+bo0AdGH1hdedBVuMM+XkVjPk14oKkEd7gNwANfOXO5BWLhJ\n/yP5DPZ3E9LTtxXVOsX6ckMA3I2I3bG/eqtzvP8dWVWIzFc7XERRHKDJvGyxrPSj\nMWrORhTKjwKBgE2VSXTAvteYjb/kVjomNUjlyMHCxBkWURRs5TauR0vfRpgmMDPK\n7AaWi0zHdNn+PkzC1kvnhSvKBbIKA3t2pfFWTW3Ame9ahsMmtte+P9KsqXIDhcc0\n0m03rwwUbMYAeox58p9XuxexP7J40TIOqUg8ZIpsZSRm4C8JQO/X9VJlAoGAbSBv\nXVfH05jbZ5eg2CaVAQXdn4QJ96LZQd89W+fR8sn8MDfetKFFuC60DG+rMX+fy3P+\nvA0yEjPO49UiMyqutW6J7qDABhKdTvoNeyh4Sux83+nbp2Qj4+BwuRIXK2iTn+bS\ncPA3idKes/AWl7g+PMkUI+ZLBMjmmQwYPIenlPsCgYEAqrh9Hx/YkjJTsILk9SDt\nkXLqlxVlr35vW6qJ2whH5Ywh2gyFgMDDOyjuV/rrky5mxbGKeNvEy5lY4vyyF25n\ntAaJwQfepz8L+TtEwZDQEPUf1KvixmhgDWaqcKh/gygleEewPvbpWVQZ/aTeubyc\n3DWfUuOIbeLTQXAVMt4yngE=\n-----END PRIVATE KEY-----\n", - "client_email": "spacemesh-198810@appspot.gserviceaccount.com", - "client_id": "", - "auth_uri": "https://accounts.google.com/o/oauth2/auth", - "token_uri": "https://oauth2.googleapis.com/token", - "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", - "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/spacemesh-198810%40appspot.gserviceaccount.com" -} From 229dbbbc01feae502df40f85b1ca9feaa5f6c990 Mon Sep 17 00:00:00 2001 From: almog Date: Thu, 29 Nov 2018 14:34:46 +0200 Subject: [PATCH 207/209] create on each cluster and teardown --- test_runner.py | 59 ++++++++++++++++++++++++++++++-------------------- 1 file changed, 36 insertions(+), 23 deletions(-) diff --git a/test_runner.py b/test_runner.py index f59d9c4..0cd846b 100644 --- a/test_runner.py +++ b/test_runner.py @@ -1,25 +1,33 @@ import os + from kubernetes import client, config DEPLOYMENT_NAME = "lalaa" os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "secret.json" + def create_deployment_object(): # Configureate Pod template container - container = client.V1Container(name="lala", image="gcr.io/spacemesh-198810/node:46f1ad099cdfefc65dda5f726d26ecea7b1f35fb", ports=[client.V1ContainerPort(container_port=9091)]) + container = client.V1Container(name="lala", + image="gcr.io/spacemesh-198810/node:46f1ad099cdfefc65dda5f726d26ecea7b1f35fb", + ports=[client.V1ContainerPort(container_port=9091)]) # Create and configurate a spec section - template = client.V1PodTemplateSpec(metadata=client.V1ObjectMeta(labels={"app": "nginx"}), spec=client.V1PodSpec(containers=[container])) + template = client.V1PodTemplateSpec(metadata=client.V1ObjectMeta(labels={"app": "nginx"}), + spec=client.V1PodSpec(containers=[container])) # Create the specification of deployment spec = client.ExtensionsV1beta1DeploymentSpec(replicas=1, template=template) # Instantiate the deployment object - deployment = client.ExtensionsV1beta1Deployment(api_version="extensions/v1beta1", kind="Deployment",metadata=client.V1ObjectMeta(name=DEPLOYMENT_NAME),spec=spec) + deployment = client.ExtensionsV1beta1Deployment(api_version="extensions/v1beta1", kind="Deployment", + metadata=client.V1ObjectMeta(name=DEPLOYMENT_NAME), spec=spec) return deployment + def create_deployment(api_instance, deployment): -# Create deployement - api_response = api_instance.create_namespaced_deployment(body=deployment,namespace="default") + # Create deployement + api_response = api_instance.create_namespaced_deployment(body=deployment, namespace="default") print("Deployment created. status='%s'" % str(api_response.status)) + def update_deployment(api_instance, deployment): # Update container image deployment.spec.template.spec.containers[0].image = "nginx:1.9.1" @@ -30,34 +38,39 @@ def update_deployment(api_instance, deployment): body=deployment) print("Deployment updated. status='%s'" % str(api_response.status)) + def delete_deployment(api_instance): -# Delete deployment + # Delete deployment api_response = api_instance.delete_namespaced_deployment( - name=DEPLOYMENT_NAME, - namespace="default", - body=client.V1DeleteOptions(propagation_policy='Foreground', grace_period_seconds=5)) + name=DEPLOYMENT_NAME, + namespace="default", + body=client.V1DeleteOptions(propagation_policy='Foreground', grace_period_seconds=5)) print("Deployment deleted. status='%s'" % str(api_response.status)) -def list_all(): - config_file = os.path.join(os.path.expanduser('~'), '.kube', 'config') - contexts, active_context = config.list_kube_config_contexts(config_file) + +def list_all(contexts): if not contexts: print("Cannot find any context in kube-config file.") - return - contextsNames = [context['name'] for context in contexts] + return + contextsNames = [context['name'] for context in contexts] for ctx in contextsNames: v1 = client.CoreV1Api(config.new_client_from_config(config_file, ctx)) print("cluster: " + ctx + " Listing pods with their IPs:") ret = v1.list_namespaced_pod("default") for item in ret.items: print("%s\t%s\t%s" % (item.status.pod_ip, item.metadata.namespace, item.metadata.name)) - + + if __name__ == '__main__': - list_all() - config.load_kube_config() - extensions_v1beta1 = client.ExtensionsV1beta1Api() - deployment = create_deployment_object() - create_deployment(extensions_v1beta1, deployment) - list_all() - update_deployment(extensions_v1beta1, deployment) - #delete_deployment(extensions_v1beta1) \ No newline at end of file + config_file = os.path.join(os.path.expanduser('~'), '.kube', 'config') + contexts, active_context = config.list_kube_config_contexts(config_file) + list_all(contexts) + # create deployment in every cluster + for ctx in contexts: + extensions_v1beta1 = client.ExtensionsV1beta1Api(config.new_client_from_config(config_file, ctx['name'])) + deployment = create_deployment_object() + create_deployment(extensions_v1beta1, deployment) + update_deployment(extensions_v1beta1, deployment) + # teardown deployment in every cluster + for ctx in contexts: + delete_deployment(extensions_v1beta1) From 480b022bb948cc8bedc47da881a973881fc82b0c Mon Sep 17 00:00:00 2001 From: almog Date: Thu, 29 Nov 2018 15:30:19 +0200 Subject: [PATCH 208/209] fix bug in test runner --- test_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_runner.py b/test_runner.py index 0cd846b..1e9f7b1 100644 --- a/test_runner.py +++ b/test_runner.py @@ -64,7 +64,6 @@ def list_all(contexts): if __name__ == '__main__': config_file = os.path.join(os.path.expanduser('~'), '.kube', 'config') contexts, active_context = config.list_kube_config_contexts(config_file) - list_all(contexts) # create deployment in every cluster for ctx in contexts: extensions_v1beta1 = client.ExtensionsV1beta1Api(config.new_client_from_config(config_file, ctx['name'])) @@ -73,4 +72,5 @@ def list_all(contexts): update_deployment(extensions_v1beta1, deployment) # teardown deployment in every cluster for ctx in contexts: + extensions_v1beta1 = client.ExtensionsV1beta1Api(config.new_client_from_config(config_file, ctx['name'])) delete_deployment(extensions_v1beta1) From 481baa28757695f80d1540fefb98f9eba8a623f9 Mon Sep 17 00:00:00 2001 From: almog Date: Thu, 29 Nov 2018 15:36:03 +0200 Subject: [PATCH 209/209] add gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6fa708c --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +secret.json \ No newline at end of file