From cac1bfde06400b27b6060ebe77639bc93281dee9 Mon Sep 17 00:00:00 2001 From: moshe Date: Sun, 11 Nov 2018 15:14:27 +0200 Subject: [PATCH 01/10] move the msg type to attribute, so we could use the data for DM and gossip messages --- tests/base_test_agent.py | 3 ++- tests/base_test_ci.py | 10 +++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index 7076165..996675c 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -54,7 +54,8 @@ def modify_seeders(self, seeders, bootstrap): def callback(self, message): logging.info(message) - self.message = "".join(map(chr, message.data)) + self.message = message.attributes['msg'] + self.data = "".join(map(chr, message.data)) message.ack() logging.info(message.attributes['phase']) if self.phase != message.attributes['phase']: diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index da6722b..f6cf380 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -47,11 +47,11 @@ def create_phase(self, phase = ""): self.phases.append(self.phase) return self.phase - def send(self, data): + def send(self, msg, data = 'NULL'): self.messages = [] logging.info(data) data = data.encode('utf-8') - self.publisher_downstream.publish(self.topic_path_downstream, data=data, phase=self.phase) + self.publisher_downstream.publish(self.topic_path_downstream, data=data, phase=self.phase, msg=msg) def wait_for_response(self, num_messages = 1): for i in range(0, self.testLen): @@ -60,8 +60,8 @@ def wait_for_response(self, num_messages = 1): break time.sleep(1) - def send_and_wait(self, data, nodes): - self.send(data) + def send_and_wait(self, msg, nodes, data = 'NULL'): + self.send(msg, data) self.wait_for_response(nodes) self.assertEqual(nodes, len(self.messages)) return self.messages @@ -77,7 +77,7 @@ def run_phase(self, nodes = 1, seeders=config.CONFIG['no_seeders'], bootstrap = self.create_phase(len(self.phases)) for i in range(0, nodes): self.start_node_agent_pair(seeders = seeders, bootstrap = bootstrap) - self.send_and_wait('GET_NODE_ID', nodes) + self.send_and_wait('GET_NODE_ID', 'NULL', nodes) self.nodes_list += self.messages return self.messages From 70ee5a95651c4974d631d8b1e79a00fbd3efd826 Mon Sep 17 00:00:00 2001 From: moshe Date: Sun, 11 Nov 2018 15:29:26 +0200 Subject: [PATCH 02/10] debug stuff --- tests/base_test_ci.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index f6cf380..495ed61 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -49,6 +49,7 @@ def create_phase(self, phase = ""): def send(self, msg, data = 'NULL'): self.messages = [] + logging.info(msg) logging.info(data) data = data.encode('utf-8') self.publisher_downstream.publish(self.topic_path_downstream, data=data, phase=self.phase, msg=msg) From 362d9739dcbf4d899414009714bdbcc5b74c342c Mon Sep 17 00:00:00 2001 From: moshe Date: Sun, 11 Nov 2018 16:11:50 +0200 Subject: [PATCH 03/10] add prams to avoid mismatch --- tests/base_test_ci.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index 495ed61..2b9ad1d 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -78,7 +78,7 @@ def run_phase(self, nodes = 1, seeders=config.CONFIG['no_seeders'], bootstrap = self.create_phase(len(self.phases)) for i in range(0, nodes): self.start_node_agent_pair(seeders = seeders, bootstrap = bootstrap) - self.send_and_wait('GET_NODE_ID', 'NULL', nodes) + self.send_and_wait(msg = 'GET_NODE_ID', data = 'NULL', nodes = nodes) self.nodes_list += self.messages return self.messages From e35d19b62b2aa9c2256610bee57375bd998cb97a Mon Sep 17 00:00:00 2001 From: moshe Date: Sun, 11 Nov 2018 16:58:05 +0200 Subject: [PATCH 04/10] add headers --- tests/test_2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_2.py b/tests/test_2.py index 3451b45..06eef48 100644 --- a/tests/test_2.py +++ b/tests/test_2.py @@ -20,7 +20,7 @@ def test_sendId(self): seeders_str = '\'["' + '","'.join(self.messages) + '"]\'' logging.info(seeders_str) self.run_phase(nodes = testers_nodes, bootstrap = 'true', seeders = seeders_str) - messages = self.send_and_wait('GET_DHT_SIZE', testers_nodes) + messages = self.send_and_wait(msg = 'GET_DHT_SIZE', nodes = testers_nodes) for i in range(0, testers_nodes): self.assertEqual(seeders_nodes + testers_nodes, int(messages[i])) From 0475c5885e352a06bea6cbb5381b7fa56406a284 Mon Sep 17 00:00:00 2001 From: moshe Date: Sun, 11 Nov 2018 17:01:21 +0200 Subject: [PATCH 05/10] debug --- tests/base_test_agent.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index 996675c..8c83ca1 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -103,6 +103,7 @@ def get_dht_size(self): results = pattern.match(line) if results != None: return results.group(1) + time.sleep(10) return 0 if __name__ == '__main__': From f2f3e6bccb9741830e5e8bcb1804d3bad459354e Mon Sep 17 00:00:00 2001 From: moshe Date: Sun, 11 Nov 2018 17:04:52 +0200 Subject: [PATCH 06/10] Restore the sleep :-) --- tests/base_test_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index 8c83ca1..b873b7e 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -103,7 +103,7 @@ def get_dht_size(self): results = pattern.match(line) if results != None: return results.group(1) - time.sleep(10) + time.sleep(1) return 0 if __name__ == '__main__': From 3e98e81f5b633936977ac6ee86c3e6740ddcd3d0 Mon Sep 17 00:00:00 2001 From: moshe Date: Sun, 11 Nov 2018 17:42:58 +0200 Subject: [PATCH 07/10] Encapsulate the nodes --- tests/base_test_ci.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index 2b9ad1d..3f2d459 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -1,4 +1,5 @@ from dockers import Docker +from nodes import Nodes import config import os import time @@ -13,7 +14,7 @@ class BaseTest(unittest.TestCase): def setUp(self): logging.basicConfig(format='%(asctime)s %(message)s', level=logging.INFO) - self.nodes_list = [] + self.nodes_list = Nodes() self.phases = [] self.endFlag = False self.testLen = 60 From 261ee3f97620a3be03826206987d145a18a4b801 Mon Sep 17 00:00:00 2001 From: moshe Date: Sun, 11 Nov 2018 17:43:13 +0200 Subject: [PATCH 08/10] encapsulate (2) --- tests/nodes.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 tests/nodes.py diff --git a/tests/nodes.py b/tests/nodes.py new file mode 100644 index 0000000..6cff756 --- /dev/null +++ b/tests/nodes.py @@ -0,0 +1,24 @@ +import logging +from logging import Logger + +class Nodes(): + def __init__(self): + self.nodes = [] + + def add(self, node): + self.nodes.append(node) + + def __iadd__(self, l): + self.nodes += l + return self + + def get_nodes(self, num): + tmp = self.nodes + selected_nodes = [] + for i in range(0, num): + if len(tmp) == 0: + break + select = random.randint(0, len(tmp)) + selected_nodes.append(tmp[select]) + tmp.remove(select) + return selected_nodes From f46f6ed903f3233ab814b32dda154c24b0c7f38c Mon Sep 17 00:00:00 2001 From: moshe Date: Sun, 11 Nov 2018 17:58:31 +0200 Subject: [PATCH 09/10] Refactor => too small list is a bug --- tests/nodes.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/nodes.py b/tests/nodes.py index 6cff756..c79a7be 100644 --- a/tests/nodes.py +++ b/tests/nodes.py @@ -13,12 +13,13 @@ def __iadd__(self, l): return self def get_nodes(self, num): + if num < len(self.nodes): + return None + tmp = self.nodes selected_nodes = [] for i in range(0, num): - if len(tmp) == 0: - break select = random.randint(0, len(tmp)) selected_nodes.append(tmp[select]) - tmp.remove(select) - return selected_nodes + del tmp[select] + return selected_nodes \ No newline at end of file From fcf3a9a1a90ac8e1e5e8ab777eb8d3aafe253af4 Mon Sep 17 00:00:00 2001 From: moshe Date: Sun, 11 Nov 2018 18:58:04 +0200 Subject: [PATCH 10/10] Support a single node send --- tests/base_test_ci.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index 3f2d459..e511131 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -48,12 +48,12 @@ def create_phase(self, phase = ""): self.phases.append(self.phase) return self.phase - def send(self, msg, data = 'NULL'): + def send(self, msg, data = 'NULL', target_node = None): self.messages = [] logging.info(msg) logging.info(data) data = data.encode('utf-8') - self.publisher_downstream.publish(self.topic_path_downstream, data=data, phase=self.phase, msg=msg) + self.publisher_downstream.publish(self.topic_path_downstream, data=data, phase=self.phase, msg=msg, node=target_node) def wait_for_response(self, num_messages = 1): for i in range(0, self.testLen): @@ -68,6 +68,12 @@ def send_and_wait(self, msg, nodes, data = 'NULL'): self.assertEqual(nodes, len(self.messages)) return self.messages + def send_to_node(self, msg, node, data = 'NULL', expected_msgs = 1) + self.send(msg, data) + self.wait_for_response(expected_msgs) + self.assertEqual(expected_msgs, len(self.messages)) + return self.messages + def start_node_agent_pair(self, seeders=config.CONFIG['no_seeders'], bootstrap = 'false'): docker = Docker() docker.stop('agent_' + str(self.agents))