diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6fa708c --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +secret.json \ No newline at end of file diff --git a/Makefile b/Makefile index e409e72..fb231bd 100644 --- a/Makefile +++ b/Makefile @@ -29,9 +29,6 @@ install: @make build clean_containers: - @docker stop agent 2>/dev/null || true - @docker stop node 2>/dev/null || true - @docker stop cleaner 2>/dev/null || true @docker stop test_server 2>/dev/null || true run: @@ -39,7 +36,6 @@ run: @rm -rf $(shell pwd)/logs/*.log @make clean_containers @if [ ! "$(sudo docker network ls | grep devnet)" ]; then sudo docker network create devnet || true; fi - @docker run --rm --network=devnet --name cleaner -e PUBSUB_VERIFICATION_TOKEN='1234' -e PUBSUB_TOPIC='topic' -e GOOGLE_CLOUD_PROJECT='spacemesh-198810' -v $(shell pwd)/tests:/opt/devnet spacemesh/devnet_agent:latest python3 /opt/devnet/base_cleaner.py >> $(shell pwd)/logs/cleaner.log 2>&1 @docker run --rm --network=devnet --name test_server -e PUBSUB_VERIFICATION_TOKEN='1234' -e PUBSUB_TOPIC='topic' -e GOOGLE_CLOUD_PROJECT='spacemesh-198810' -v $(shell pwd)/tests:/opt/devnet spacemesh/devnet_agent:latest python3 /opt/devnet/tests.py >> $(shell pwd)/logs/test.log 2>&1 & build: @@ -59,7 +55,7 @@ build_agent_packed: @docker push gcr.io/spacemesh-198810/devnet_agent_packed build_node: - @wget https://raw.githubusercontent.com/spacemeshos/go-spacemesh/develop/Dockerfile -O Dockerfile.spacemesh.node - @docker build -f $(shell pwd)/Dockerfile.spacemesh.node -t spacemesh/node:latest . + @wget https://raw.githubusercontent.com/spacemeshos/go-spacemesh/$(NODE)/Dockerfile -O Dockerfile.spacemesh.node + @docker build -f $(shell pwd)/Dockerfile.spacemesh.node --build-arg BRANCH=$(NODE) -t spacemesh/node:latest . @docker tag spacemesh/devnet_agent gcr.io/spacemesh-198810/devnet_agent @docker push gcr.io/spacemesh-198810/node \ No newline at end of file diff --git a/agent/Dockerfile b/agent/Dockerfile index 779eb88..903dc82 100644 --- a/agent/Dockerfile +++ b/agent/Dockerfile @@ -3,5 +3,9 @@ FROM ubuntu:16.04 RUN apt-get -y update && apt-get install -y python3 python3-pip RUN pip3 install --upgrade pip RUN pip3 install google-cloud google-cloud-pubsub spur -RUN mkdir /opt/devnet +RUN mkdir -p /opt/devnet +RUN mkdir -p /opt/cnf +RUN mkdir -p /opt/basecnf +RUN mkdir -p /opt/logs +ADD ./tests/test.config.toml /opt/basecnf/ WORKDIR /opt/devnet \ No newline at end of file diff --git a/google_pubsub.py b/google_pubsub.py new file mode 100644 index 0000000..8f95105 --- /dev/null +++ b/google_pubsub.py @@ -0,0 +1,31 @@ +from pubsub import PubSub +from google.cloud import pubsub_v1 + + +class GooglePubSub(PubSub): + def __init__(self): + self.publisher = pubsub_v1.PublisherClient() + self.subscriber = pubsub_v1.SubscriberClient() + self.project = "spacemesh-198810" + + def Subscribe(self, channel_name, callback): + topic_path = self.publisher.topic_path(self.project, channel_name) + subscription_path = self.subscriber.subscription_path(self.project, "kaplan") + print subscription_path + self.subscriber.create_subscription(subscription_path, topic_path) + self.subscriber.subscribe(subscription_path, callback=callback) + + def Publish(self, channel_name, **kwargs): + self.publisher.publish(channel_name, **kwargs) + + def UnSubscribe(self, channel_name): + self.subscriber.delete_subscription(channel_name) + + def CreateChannel(self, channel_name): + topic_path = self.publisher.topic_path(self.project, topic_name) + self.publisher.create_topic(topic_path) + return tpoic_path + + + def DeleteChannel(self, channel_name): + self.publisher.delete_topic(channel_name) \ No newline at end of file diff --git a/google_pubsub.pyc b/google_pubsub.pyc new file mode 100644 index 0000000..6b366e5 Binary files /dev/null and b/google_pubsub.pyc differ diff --git a/pubsub.py b/pubsub.py new file mode 100644 index 0000000..18f0cca --- /dev/null +++ b/pubsub.py @@ -0,0 +1,20 @@ + +class PubSub: + def __init__(self): + pass + + def Subscribe(self, channel_name, callback): + pass + + def Publish(self, channel_name, **kwargs): + pass + + def UnSubscribe(self, channel_name): + pass + + def CreateChannel(self, channel_name): + pass + + def DeleteChannel(self, channel_name): + pass + diff --git a/pubsub.pyc b/pubsub.pyc new file mode 100644 index 0000000..726f411 Binary files /dev/null and b/pubsub.pyc differ diff --git a/runner.py b/runner.py new file mode 100644 index 0000000..31999e2 --- /dev/null +++ b/runner.py @@ -0,0 +1,18 @@ +from google_pubsub import GooglePubSub +import time +import pubsub + + +def PubSubFactory(provider): + if provider == "google": + return GooglePubSub() + + +def printlogs(message): + print message + +if __name__ == '__main__': + pubsub = PubSubFactory("google") + pubsub.Subscribe("devnet_tests_downstream", printlogs) + while True: + time.sleep(1) diff --git a/test_runner.py b/test_runner.py new file mode 100644 index 0000000..1e9f7b1 --- /dev/null +++ b/test_runner.py @@ -0,0 +1,76 @@ +import os + +from kubernetes import client, config + +DEPLOYMENT_NAME = "lalaa" +os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "secret.json" + + +def create_deployment_object(): + # Configureate Pod template container + container = client.V1Container(name="lala", + image="gcr.io/spacemesh-198810/node:46f1ad099cdfefc65dda5f726d26ecea7b1f35fb", + ports=[client.V1ContainerPort(container_port=9091)]) + # Create and configurate a spec section + template = client.V1PodTemplateSpec(metadata=client.V1ObjectMeta(labels={"app": "nginx"}), + spec=client.V1PodSpec(containers=[container])) + # Create the specification of deployment + spec = client.ExtensionsV1beta1DeploymentSpec(replicas=1, template=template) + # Instantiate the deployment object + deployment = client.ExtensionsV1beta1Deployment(api_version="extensions/v1beta1", kind="Deployment", + metadata=client.V1ObjectMeta(name=DEPLOYMENT_NAME), spec=spec) + return deployment + + +def create_deployment(api_instance, deployment): + # Create deployement + api_response = api_instance.create_namespaced_deployment(body=deployment, namespace="default") + print("Deployment created. status='%s'" % str(api_response.status)) + + +def update_deployment(api_instance, deployment): + # Update container image + deployment.spec.template.spec.containers[0].image = "nginx:1.9.1" + # Update the deployment + api_response = api_instance.patch_namespaced_deployment( + name=DEPLOYMENT_NAME, + namespace="default", + body=deployment) + print("Deployment updated. status='%s'" % str(api_response.status)) + + +def delete_deployment(api_instance): + # Delete deployment + api_response = api_instance.delete_namespaced_deployment( + name=DEPLOYMENT_NAME, + namespace="default", + body=client.V1DeleteOptions(propagation_policy='Foreground', grace_period_seconds=5)) + print("Deployment deleted. status='%s'" % str(api_response.status)) + + +def list_all(contexts): + if not contexts: + print("Cannot find any context in kube-config file.") + return + contextsNames = [context['name'] for context in contexts] + for ctx in contextsNames: + v1 = client.CoreV1Api(config.new_client_from_config(config_file, ctx)) + print("cluster: " + ctx + " Listing pods with their IPs:") + ret = v1.list_namespaced_pod("default") + for item in ret.items: + print("%s\t%s\t%s" % (item.status.pod_ip, item.metadata.namespace, item.metadata.name)) + + +if __name__ == '__main__': + config_file = os.path.join(os.path.expanduser('~'), '.kube', 'config') + contexts, active_context = config.list_kube_config_contexts(config_file) + # create deployment in every cluster + for ctx in contexts: + extensions_v1beta1 = client.ExtensionsV1beta1Api(config.new_client_from_config(config_file, ctx['name'])) + deployment = create_deployment_object() + create_deployment(extensions_v1beta1, deployment) + update_deployment(extensions_v1beta1, deployment) + # teardown deployment in every cluster + for ctx in contexts: + extensions_v1beta1 = client.ExtensionsV1beta1Api(config.new_client_from_config(config_file, ctx['name'])) + delete_deployment(extensions_v1beta1) diff --git a/tests/base_cleaner.py b/tests/base_cleaner.py deleted file mode 100644 index 5eae2f4..0000000 --- a/tests/base_cleaner.py +++ /dev/null @@ -1,35 +0,0 @@ -import config -from google.cloud import pubsub_v1 -import time - -class BaseDevnetCleaner: - def __init__(self): - self.endFlag = False - - self.project = config.CONFIG['project'] - - subscription_name_upstream = config.CONFIG['subscription_name_upstream'] - self.subscriber_upstream = pubsub_v1.SubscriberClient() - self.subscription_path_upstream = self.subscriber_upstream.subscription_path(self.project, subscription_name_upstream) - - self.subscriber_downstream = pubsub_v1.SubscriberClient() - - def get_downstream_subscription_path(self, i): - return self.subscriber_downstream.subscription_path(self.project, config.CONFIG['subscription_name_downstream'] + '_' + str(i)) - - def callback(self, message): - self.endFlag = False - message.ack() - - def cleanup(self, subscriber, subscription_path): - subscriber.subscribe(subscription_path, callback=self.callback) - while not self.endFlag: - self.endFlag = True - time.sleep(1) - -if __name__ == '__main__': - t = BaseDevnetCleaner() - - for i in range(0, 10): - t.cleanup(t.subscriber_downstream, t.get_downstream_subscription_path(i)) - t.cleanup(t.subscriber_upstream, t.subscription_path_upstream) \ No newline at end of file diff --git a/tests/base_test_agent.py b/tests/base_test_agent.py index 1c79dcd..8b6a926 100644 --- a/tests/base_test_agent.py +++ b/tests/base_test_agent.py @@ -1,62 +1,105 @@ import config from dockers import Docker +from topics import Publisher, Subscriber from google.cloud import pubsub_v1 import time import datetime from subprocess import call import os +import re +import logging +from logging import Logger class BaseDevnetAgent: def __init__(self): + logging.basicConfig(format='%(asctime)s %(message)s', level=logging.INFO) self.endFlag = False - - project = config.CONFIG['project'] - topic_name_upstream = config.CONFIG['topic_name_upstream'] - self.publisher_upstream = pubsub_v1.PublisherClient() - self.topic_path_upstream = self.publisher_upstream.topic_path(project, topic_name_upstream) + self.dht_timeout = config.CONFIG['dht_timeout'] + self.node_id = "NULL" self.node = os.environ['NODE'] + self.phase = os.environ['PHASE'] + self.node_port = config.CONFIG['node_port'] self.docker = Docker() + self.start_node() + self.establish_links() + + def establish_links(self): + self.project = config.CONFIG['project'] + self.down_subscriber = Subscriber(self.project, os.environ['SUBSCRIPTION_PATH_DOWNSTREAM']) + self.down_subscriber.subscribe(self.callback) + self.up_publisher = Publisher(self.project, os.environ['TOPIC_PATH_UPSTREAM']) + + def start_node(self): + logging.info('seeders:' + os.environ['SEEDERS']) + self.modify_seeders(os.environ['SEEDERS'], os.environ['BOOTSTRAP'], os.environ['RANDCON']) self.docker.stop('node_' + self.node) - self.docker.start('docker run --network=devnet --name node_' + self.node + ' -p ' + str(7513 + int(self.node)) + ':7513 -v /root/spacemesh/devnet/logs:/root/.spacemesh/nodes/ spacemesh/node:latest /go/src/github.com/spacemeshos/go-spacemesh/go-spacemesh') - subscription_name_downstream = os.environ['SUBSCRIPTION_NAME_DOWNSTREAM'] - self.subscriber_downstream = pubsub_v1.SubscriberClient() - self.subscription_path_downstream = self.subscriber_downstream.subscription_path(project, subscription_name_downstream) + self.docker.start('docker run --network=devnet --name node_' + self.node + ' -p ' + str(self.node_port + int(self.node)) + ':' + str(self.node_port) + ' -v /root/spacemesh/devnet/logs' + self.node + ':/root/.spacemesh/nodes/ -v /root/spacemesh/devnet/cnf' + self.node + '/test.config.toml:/root/config.toml spacemesh/node:latest /go/src/github.com/spacemeshos/go-spacemesh/go-spacemesh --config=/root/config.toml > /root/spacemesh/devnet/logs' + self.node + '/node.log') + + while self.get_node_id() == 'NULL': + time.sleep(1) + + def modify_seeders(self, seeders, bootstrap, randcon): + file_name = "/opt/basecnf/test.config.toml" + out_file_name = "/opt/cnf/test.config.toml" + + with open(file_name) as f: + new_config = f.read().replace('BOOT_NODES', 'bootnodes = ' + seeders[1:-1]).replace('BOOTSTRAP_VALUE', bootstrap).replace('RANDCON', randcon) + + with open(out_file_name, "w") as f: + f.write(new_config) def callback(self, message): - self.message = message.data + logging.info(message) + self.message = "".join(map(chr, message.data)) message.ack() - print(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + " GOT_DOWN_MSG " + "".join(map(chr, self.message))) - if b'END' == self.message: + logging.info(message.attributes['phase']) + if self.phase != message.attributes['phase']: + logging.info("NO_MESSAGE") + return + logging.info("GOT_DOWN_MSG " + self.message) + + if 'END' == self.message: self.docker.stop('node_' + self.node) self.endFlag = True - elif b'SEND_UP' == self.message: + elif 'SEND_UP' == self.message: self.send('UP') - elif b'GET_NODE_ID' == self.message: + elif 'GET_NODE_ID' == self.message: self.send(self.get_node_id()) - elif b'SHUTDOWN_NODE' == self.message: + elif 'GET_DHT_SIZE' == self.message: + self.send(str(self.get_dht_size())) + elif 'SHUTDOWN_NODE' == self.message: self.docker.stop('node_' + self.node) def act_on_request(self): - self.subscriber_downstream.subscribe(self.subscription_path_downstream, callback=self.callback) while not self.endFlag: time.sleep(1) def send(self, data): + logging.info("sent: " + str(data)) data = data.encode('utf-8') - self.publisher_upstream.publish(self.topic_path_upstream, data=data) + self.up_publisher.publish(data=data, phase=self.phase) def get_node_id(self): - time.sleep(15) - node_id = 'NULL' try: - node_id = next(os.walk('/opt/logs'))[1][0] + self.node_id = next(os.walk('/opt/logs'))[1][0] except Exception as e: - print('Error finding log folder') - print(e.__doc__ ) + logging.warning('Error finding log folder') + logging.warning(e.__doc__ ) + return 'NULL' - print('NodeId:' + node_id) - return node_id + logging.info('NodeId:' + self.node_id) + return 'node_' + self.node + ':' + str(self.node_port) + '/' + self.node_id + + def get_dht_size(self): + pattern = re.compile(u'.*DHT Bootstrapped with (\d+)') + for i in range(0, self.dht_timeout): + for line in open('/opt/logs/' + self.node_id + '/node.log', "r", encoding="utf-8"): + results = pattern.match(line) + if results != None: + return results.group(1) + time.sleep(1) + return 0 if __name__ == '__main__': t = BaseDevnetAgent() diff --git a/tests/base_test_ci.py b/tests/base_test_ci.py index 54b063e..645a63e 100644 --- a/tests/base_test_ci.py +++ b/tests/base_test_ci.py @@ -1,56 +1,87 @@ from dockers import Docker +from topics import Publisher, Subscriber import config import os import time +import calendar from google.cloud import pubsub_v1 import unittest import spur +import logging +from logging import Logger class BaseTest(unittest.TestCase): def setUp(self): + logging.basicConfig(format='%(asctime)s %(message)s', level=logging.INFO) + + self.nodes_list = [] + self.phases = [] self.endFlag = False - self.testLen = 20 + self.testLen = 60 self.message = b'NULL' - project = config.CONFIG['project'] - subscription_name_upstream = config.CONFIG['subscription_name_upstream'] - self.subscriber_upstream = pubsub_v1.SubscriberClient() - self.subscription_path_upstream = self.subscriber_upstream.subscription_path(project, subscription_name_upstream) - self.subscriber_upstream.subscribe(self.subscription_path_upstream, callback=self.callback) - topic_name_downstream = config.CONFIG['topic_name_downstream'] - self.publisher_downstream = pubsub_v1.PublisherClient() - self.topic_path_downstream = self.publisher_downstream.topic_path(project, topic_name_downstream) + self.project = config.CONFIG['project'] + self.up_publisher = Publisher(self.project) + self.up_topic_path = self.up_publisher.create() + self.up_publisher.add_subscription().subscribe(self.callback) + self.down_publisher = Publisher(self.project) + self.down_publisher.create() self.agents = 0 + self.messages = [] def tearDown(self): - self.send('END') + for self.phase in self.phases: + self.send('END') + self.up_publisher.delete() + self.down_publisher.delete() def callback(self, message): - self.message = message.data + if self.phase != message.attributes['phase']: + return + logging.info(message) + self.messages.append(message.data.decode("utf-8")) message.ack() - self.endFlag = True + + def create_phase(self, phase = ""): + self.phase = 'phase_' + str(phase) + '_' + str(calendar.timegm(time.gmtime())) + self.phases.append(self.phase) + return self.phase def send(self, data): + self.messages = [] + logging.info(data) data = data.encode('utf-8') - self.publisher_downstream.publish(self.topic_path_downstream, data=data) + self.down_publisher.publish(data=data, phase=self.phase) - def wait_for_response(self): + def wait_for_response(self, num_messages = 1): for i in range(0, self.testLen): - if self.endFlag: - print(self.message) + if len(self.messages) == num_messages: + logging.info(self.messages) break time.sleep(1) - def send_and_wait(self, data): + def send_and_wait(self, data, nodes): self.send(data) - self.wait_for_response() + self.wait_for_response(nodes) + self.assertEqual(nodes, len(self.messages)) + return self.messages - def start_node_agent_pair(self): + def start_node_agent_pair(self, seeders=config.CONFIG['no_seeders'], bootstrap = 'false', randcon = 5): docker = Docker() docker.stop('agent_' + str(self.agents)) - docker.start('docker run --network=devnet --name agent_' + str(self.agents) + ' -v /root/spacemesh/devnet/tests:/opt/devnet -v /root/spacemesh/devnet/logs:/opt/logs -e SUBSCRIPTION_NAME_DOWNSTREAM=devnet_tests_agent_' + str(self.agents) + ' -e NODE=' + str(self.agents) + ' spacemesh/devnet_agent:latest python3 /opt/devnet/base_test_agent.py') + down_subscriber = self.down_publisher.add_subscription() + cmd = 'docker run --network=devnet --name agent_' + str(self.agents) + ' -v /root/spacemesh/devnet/tests:/opt/devnet -v /root/spacemesh/devnet/logs' + str(self.agents) + ':/opt/logs -v /root/spacemesh/devnet/cnf' + str(self.agents) + ':/opt/cnf/ -e SUBSCRIPTION_PATH_DOWNSTREAM=' + down_subscriber.subscription_path + ' -e TOPIC_PATH_UPSTREAM=' + self.up_topic_path + ' -e PHASE=' + self.phase + ' -e BOOTSTRAP=' + bootstrap +' -e NODE=' + str(self.agents) + ' -e SEEDERS=' + seeders + ' -e RANDCON=' + str(randcon) + ' spacemesh/devnet_agent:latest python3 /opt/devnet/base_test_agent.py' + docker.start(cmd) self.agents += 1 + def run_phase(self, nodes = 1, seeders=config.CONFIG['no_seeders'], bootstrap = 'false', randcon = 5, message = ''): + self.create_phase(len(self.phases)) + for i in range(0, nodes): + self.start_node_agent_pair(seeders = seeders, bootstrap = bootstrap, randcon = randcon) + self.send_and_wait('GET_NODE_ID', nodes) + self.nodes_list += self.messages + return self.messages + if __name__ == '__main__': unittest.main() \ No newline at end of file diff --git a/tests/config.py b/tests/config.py index 9127b8b..a1b9c00 100644 --- a/tests/config.py +++ b/tests/config.py @@ -1,10 +1,9 @@ CONFIG = { 'project': 'spacemesh-198810', - 'topic_name_upstream': 'devnet_tests', - 'subscription_name_upstream': 'devnet_tests_ci', - 'topic_name_downstream': 'devnet_tests_downstream', - 'subscription_name_downstream': 'devnet_tests_agent', 'host': '127.0.0.1', 'host_user': 'deploy', - 'host_password': 'deploy_password' + 'host_password': 'deploy_password', + 'no_seeders': '["0.0.0.0:7517/j7qWfWaJRVp25ZsnCu9rJ4PmhigZBtesB4YmQHqqPvtR"]', + 'node_port': 7513, + 'dht_timeout': 60 } \ No newline at end of file diff --git a/tests/dockers.py b/tests/dockers.py index 2b051bc..70976f6 100644 --- a/tests/dockers.py +++ b/tests/dockers.py @@ -1,28 +1,18 @@ import config import spur +import logging +from logging import Logger class Docker(): - def start(self, cmd): - try: - print(cmd) - print(config.CONFIG['host']) - print(config.CONFIG['host_user']) - print(config.CONFIG['host_password']) - shell = spur.SshShell( - hostname=config.CONFIG['host'], - username=config.CONFIG['host_user'], - password=config.CONFIG['host_password'], - missing_host_key=spur.ssh.MissingHostKey.accept - ) - with shell: - result = shell.spawn(cmd.split(' ')) - print('Docker started') - except Exception as e: - print('Docker start failed') - print(e.__doc__ ) + def __init__(self): + logging.basicConfig(format='%(asctime)s %(message)s', level=logging.INFO) - def run_cmd_interactive(self, cmd): + def run_cmd(self, cmd, interactive = True): try: + logging.info(cmd) + logging.info(config.CONFIG['host']) + logging.info(config.CONFIG['host_user']) + logging.info(config.CONFIG['host_password']) shell = spur.SshShell( hostname=config.CONFIG['host'], username=config.CONFIG['host_user'], @@ -30,15 +20,18 @@ def run_cmd_interactive(self, cmd): missing_host_key=spur.ssh.MissingHostKey.accept ) with shell: - result = shell.run(cmd.split(' ')) - print('Run interactive: ' + cmd) + if interactive: + result = shell.run(cmd.split(' ')) + else: + result = shell.spawn(cmd.split(' ')) + logging.info('Run: ' + cmd) except Exception as e: - print('Run interactive failed: ' + cmd) - print(e.__doc__ ) + logging.warning('Run failed: ' + cmd) + logging.warning(e.__doc__ ) - def stop(self, name): - self.run_cmd_interactive("docker stop " + name) - self.run_cmd_interactive("docker rm " + name) + def start(self, cmd): + self.run_cmd(cmd, interactive = False) -if __name__ == '__main__': - unittest.main() \ No newline at end of file + def stop(self, name): + self.run_cmd("docker stop " + name) + self.run_cmd("docker rm " + name) \ No newline at end of file diff --git a/tests/test.config.toml b/tests/test.config.toml new file mode 100644 index 0000000..e798bf3 --- /dev/null +++ b/tests/test.config.toml @@ -0,0 +1,42 @@ +# sample spacemesh config file +# use the config flag to start a node with a config file. +# e.g $./go-spacemash -config ./config.toml + +# Main Config +[main] +data-folder = "~/.spacemesh" + +# Node Config +[p2p] +security-param = 20 +fast-sync = true +tcp-port = 7513 +node-id = "" +new-node= false # will try to load from file if this is false +dial-timeout = "1m" +conn-keepalive = "48h" +network-id = 1 # 0 - MainNet, 1 - TestNet +response-timeout = "2s" + +# Node Swarm Config +[p2p.swarm] +gossip = true +bootstrap = BOOTSTRAP_VALUE +bucketsize = 20 # Routing table bucket size. recommended higher for bootstrap node +rtalpha = 4 # Routing table alpha +randcon = RANDCON # Number of random connections (neighbors for gossip). increase on bigger networks decrease for small +BOOT_NODES + +# API Config +[api] +grpc-server = true +json-server = true +grpc-port = 9091 +json-port = 9090 + +# Time sync NTP Config +[ntp] +max-allowed-time-drift = "10s" +ntp-queries = 5 +default-timeout-latency = "10s" +refresh-ntp-interval = "30m" \ No newline at end of file diff --git a/tests/test_0.py b/tests/test_0.py index 9d14a53..d984887 100644 --- a/tests/test_0.py +++ b/tests/test_0.py @@ -7,10 +7,10 @@ class Test0(BaseTest): def test_verifyUp(self): - self.start_node_agent_pair() - self.send_and_wait('SEND_UP') - - self.assertEqual(b'UP', self.message) + testers_nodes = 1 + messages = self.run_phase(nodes = testers_nodes, bootstrap = 'false') + messages = self.send_and_wait('SEND_UP', testers_nodes) + self.assertEqual('UP', messages[0]) if __name__ == '__main__': unittest.main() \ No newline at end of file diff --git a/tests/test_1.py b/tests/test_1.py index 5054b69..e0945a5 100644 --- a/tests/test_1.py +++ b/tests/test_1.py @@ -7,11 +7,8 @@ class Test1(BaseTest): def test_sendId(self): - self.start_node_agent_pair() - self.send_and_wait('GET_NODE_ID') - - self.assertNotEqual(b'NULL', self.message) - self.assertLess(5, len(self.message)) + messages = self.run_phase(nodes = 1, bootstrap = 'false') + self.assertLess(15, len(messages[0])) if __name__ == '__main__': unittest.main() \ No newline at end of file diff --git a/tests/test_2.py b/tests/test_2.py index 44a8b44..23756fa 100644 --- a/tests/test_2.py +++ b/tests/test_2.py @@ -2,20 +2,29 @@ import config import os import time +import calendar from google.cloud import pubsub_v1 import unittest +import logging class Test2(BaseTest): def test_sendId(self): - seeds = [] - for i in range(0, 3): - self.start_node_agent_pair() - self.send_and_wait('GET_NODE_ID') - self.assertNotEqual(b'NULL', self.message) - self.assertLess(5, len(self.message)) - seeds.append(self.message) + seeders_nodes = 3 + testers_nodes = 3 + randcon = seeders_nodes + testers_nodes - 1 - self.assertEqual(3, len(seeds)) + messages = self.run_phase(nodes = seeders_nodes, bootstrap = 'true') + for i in range(0, seeders_nodes): + self.assertLess(15, len(messages[i])) + + #'["0.0.0.0:7517/j7qWfWaJRVp25ZsnCu9rJ4PmhigZBtesB4YmQHqqPvtR"]' like + seeders_str = '\'["' + '","'.join(self.messages) + '"]\'' + logging.info(seeders_str) + self.run_phase(nodes = testers_nodes, bootstrap = 'true', randcon = randcon, seeders = seeders_str) + messages = self.send_and_wait('GET_DHT_SIZE', testers_nodes) + + for i in range(0, testers_nodes): + self.assertLessEqual(randcon, int(messages[i])) if __name__ == '__main__': unittest.main() \ No newline at end of file diff --git a/tests/tests.py b/tests/tests.py index 9a51fe6..054577b 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -1,6 +1,6 @@ import unittest -#from test_0 import Test0 -#from test_1 import Test1 +from test_0 import Test0 +from test_1 import Test1 from test_2 import Test2 if __name__ == '__main__': diff --git a/tests/topics.py b/tests/topics.py new file mode 100644 index 0000000..aa2f13e --- /dev/null +++ b/tests/topics.py @@ -0,0 +1,56 @@ +import config +import random +import calendar, time +from google.cloud import pubsub_v1 +import logging +from logging import Logger + +class Publisher(): + def __init__(self, project, topic_path = ''): + self.project = project + self.topic_path = topic_path + self.publisher = pubsub_v1.PublisherClient() + + def create(self): + self.subscribers = [] + topic_name = 'devnet_topic_' + str(random.randint(0, 9999999999)) + '_' + str(calendar.timegm(time.gmtime())) + self.topic_path = self.publisher.topic_path(self.project, topic_name) + self.publisher.create_topic(self.topic_path) + return self.topic_path + + def publish(self, **kwargs): + self.publisher.publish(self.topic_path, **kwargs) + + def add_subscription(self): + subscriber = Subscriber(self.project) + subscriber.create_for_publisher(self.topic_path) + self.subscribers.append(subscriber) + return subscriber + + def delete(self): + for s in self.subscribers: + s.delete() + self.publisher.delete_topic(self.topic_path) + +class Subscriber(): + def __init__(self, project, subscription_path = ''): + self.project = project + self.subscription_path = subscription_path + self.subscriber = pubsub_v1.SubscriberClient() + + def create(self, topic_name): + self.publisher = pubsub_v1.PublisherClient() + topic_path = self.publisher.topic_path(self.project, topic_name) + return self.create_for_publisher(self, topic_path) + + def create_for_publisher(self, topic_path): + self.subscription_name = 'devnet_sub_' + str(random.randint(0, 9999999999)) + '_' + str(calendar.timegm(time.gmtime())) + self.subscription_path = self.subscriber.subscription_path(self.project, self.subscription_name) + self.subscriber.create_subscription(self.subscription_path, topic_path) + return self.subscription_name + + def subscribe(self, callback): + self.subscriber.subscribe(self.subscription_path, callback=callback) + + def delete(self): + self.subscriber.delete_subscription(self.subscription_path) \ No newline at end of file diff --git a/tools.py b/tools.py new file mode 100644 index 0000000..8bf7bad --- /dev/null +++ b/tools.py @@ -0,0 +1,60 @@ +import sys +import requests + +def getRPCPort(advclient, cont): + inspect = advclient.inspect_container(cont.name) + rpcport = inspect['NetworkSettings']['Ports']['9090/tcp'][0]['HostPort'] + return rpcport + +def waitForRPC(cont): + for l in cont.logs(stream=True, stdout=True): + if "Started GRPC" in l: + return + +def registerProtocol(advclient, cont, proto, port): + rpcport = getRPCPort(advclient, cont) + print "Register on " + "http://127.0.0.1:" + str(rpcport) + "/v1/register" + r = requests.post("http://127.0.0.1:" + str(rpcport) + "/v1/register", json={ "name": proto, "port": port }) + print("Registerd protocol response ", r.status_code, r.reason, r.text) + + +def broadcast(advclient, cont, proto, message): + rpcport = getRPCPort(advclient, cont) + print "Sending message to " + "http://127.0.0.1:" + str(rpcport) + "/v1/broadcast" + r = requests.post("http://127.0.0.1:" + str(rpcport) + "/v1/broadcast", json={ "protocolName": proto, "payload": [10, 10, 10] }) + print("Send broadcast, response ", r.status_code, r.reason, r.text) + +def getExternalIP(advclient, cont): + ip = "" + nets = advclient.inspect_container(cont.name)['NetworkSettings']['Networks'] + for net in nets: + ip = nets[net]["IPAddress"] + if ip is not "": + print ip + break + return ip + +def getPublicKey(cont): + id = "" + for l in cont.logs(stream=True, stdout=True): + if "identity" in l: + id = l.split('>>')[1].strip() + break + return id + +def cleanUp(contlist): + ### Kill all dockers with nice bar + toolbar_width = len(contlist) + print "Killing all dockers" + sys.stdout.write("[%s]" % (" " * toolbar_width)) + sys.stdout.flush() + sys.stdout.write("\b" * (toolbar_width+1)) # return to start of line, after '[' + for i in contlist: + cont = i + if isinstance(i, dict): + cont = cont["cont"] + cont.kill() + # update the bar + sys.stdout.write("-") + sys.stdout.flush() + sys.stdout.write("\n") \ No newline at end of file