diff --git a/Nightly-Tests/Sync-Tests/IXI/TotalTransactions.ixi/index.js b/Nightly-Tests/Sync-Tests/IXI/TotalTransactions.ixi/index.js new file mode 100644 index 0000000..216d9b1 --- /dev/null +++ b/Nightly-Tests/Sync-Tests/IXI/TotalTransactions.ixi/index.js @@ -0,0 +1,24 @@ +var System = java.lang.System; + +var iri = com.iota.iri; +var Callable = iri.service.CallableRequest; +var Response = iri.service.dto.IXIResponse; +var ErrorResponse = iri.service.dto.ErrorResponse; + +var tangle = IOTA.tangle; + + + +function getTotalTransactions(){ + var count = tangle.getCount(com.iota.iri.model.persistables.Transaction.class) + + return Response.create({ + total: count + }); +} + + + + +API.put("getTotalTransactions", new Callable({ call: getTotalTransactions })) + diff --git a/Nightly-Tests/Sync-Tests/IXI/TotalTransactions.ixi/package.json b/Nightly-Tests/Sync-Tests/IXI/TotalTransactions.ixi/package.json new file mode 100644 index 0000000..14ab704 --- /dev/null +++ b/Nightly-Tests/Sync-Tests/IXI/TotalTransactions.ixi/package.json @@ -0,0 +1,3 @@ +{ + "main": "index.js" +} diff --git a/Nightly-Tests/Sync-Tests/config.yml b/Nightly-Tests/Sync-Tests/config.yml new file mode 100644 index 0000000..d23cb9f --- /dev/null +++ b/Nightly-Tests/Sync-Tests/config.yml @@ -0,0 +1,61 @@ +default_args: &args + ['--testnet-coordinator', + 'KSAFREMKHHYHSXNLGZPFVHANVHVMKWSGEAHGTXZCSQMXTCZXOGBLVPCWFKVAEQYDJMQALKZRKOTWLGBSC', + '--mwm', + '1', + '--milestone-start', + '0', + '--testnet-no-coo-validation', + 'true', + '--testnet', + 'true', + '--snapshot', + './snapshot.txt', + '--remote', + 'true', + '--remote-limit-api', + '""', + '--zmq-enable-tcp', + 'true', + '--zmq-port', + '5566', + '--local-snapshots-enabled', + 'false' + ] + +default_ixi: &ixi + ['IXI/TotalTransactions.ixi'] + +java_options: -agentlib:jdwp=transport=dt_socket,server=y,address=8000,suspend=n -javaagent:/opt/jacoco/lib/jacocoagent.jar=destfile=/iri/jacoco.exec,output=file,append=true,dumponexit=true + +defaults: &db_full + db: https://s3.eu-central-1.amazonaws.com/iotaledger-dbfiles/dev/SyncTestsDbComplete.tar + db_checksum: 12f8ae2dda157097e8929ee1ef26bf5b4f5b0f93d5b49f966bbb19afd6245e88 + iri_args: *args + ixis: *ixi + +db_empty: &db_empty + db: https://s3.eu-central-1.amazonaws.com/iotaledger-dbfiles/dev/EmptyDB.tar + db_checksum: f453e80b82ad5abd25102833f03b39379667fada962ee376a7f629a027f83a88 + iri_args: *args + ixis: *ixi + +nodes: + nodeA: #name + <<: *db_full + neighbors: + - tcp://nodeB:15600 + - tcp://nodeC:15600 + + nodeB: + <<: *db_full + neighbors: + - tcp://nodeA:15600 + - tcp://nodeC:15600 + + nodeC: + <<: *db_empty + neighbors: + - tcp://nodeA:15600 + - tcp://nodeB:15600 + diff --git a/Nightly-Tests/Sync-Tests/createCluster.sh b/Nightly-Tests/Sync-Tests/createCluster.sh new file mode 100644 index 0000000..ebb1ad1 --- /dev/null +++ b/Nightly-Tests/Sync-Tests/createCluster.sh @@ -0,0 +1,86 @@ +set -x + +trap ctrl_c INT + +function ctrl_c() { + echo -e "\nExit called by user" + for pod in $(kubectl get pods -o jsonpath='{.items[*].metadata.name}');do + echo "Pulling node logs..." + kubectl logs $pod > ./SyncOutput/$DATE/$(kubectl get pod $pod -o jsonpath='{.metadata.labels.nodenum}').log + done + + timeout 10 tiab/teardown_cluster.py -t $UUID -n $K8S_NAMESPACE + deactivate + exit +} + +if [ "$#" -ne 1 ]; then + echo "Please specify an image to test" + exit 1 +fi + +set -x + +echo "Downloading apt requirments " +sed 's/#.*//' requirements.txt | xargs apt-get install -y + +UUID="$(uuidgen)" +K8S_NAMESPACE=$(kubectl config get-contexts $(kubectl config current-context) | tail -n+2 | awk '{print $5}') +base_dir=$(pwd) +IMAGE=$1 +DATE=$(date '+%Y-%m-%d') + +if [ ! -d tiab ]; then + git clone --depth 1 https://github.com/iotaledger/tiab tiab +fi + +virtualenv -p python2 venv +source ./venv/bin/activate + +cd tiab +git fetch +git pull +echo "tiab revision: "; git rev-parse HEAD +pip install -r requirements.txt +cd $base_dir/../ + +echo "Installing python requirements" +pip install --upgrade pip +pip install -e . +cd $base_dir + +ERROR=0 + +python tiab/create_cluster.py -i $IMAGE -t $UUID -n $K8S_NAMESPACE -c config.yml -o output.yml -x $base_dir -e "apt update && apt install unzip && wget http://search.maven.org/remotecontent\?filepath\=org/jacoco/jacoco/0.8.4/jacoco-0.8.4.zip -O jacoco.zip && mkdir /opt/jacoco && unzip jacoco.zip -d /opt/jacoco" -d + +if [ $? -ne 0 ]; then + ERROR=1 + python < ./SyncOutput/$DATE/$(kubectl get pod $pod -o jsonpath='{.metadata.labels.nodenum}').log + done +fi + +echo "Tearing down cluster" +timeout 10 tiab/teardown_cluster.py -t $UUID -n $K8S_NAMESPACE + +echo "Deactivating" +deactivate + +exit $ERROR diff --git a/Nightly-Tests/Sync-Tests/finalize.py b/Nightly-Tests/Sync-Tests/finalize.py new file mode 100644 index 0000000..34916e9 --- /dev/null +++ b/Nightly-Tests/Sync-Tests/finalize.py @@ -0,0 +1,26 @@ +from iota import BundleHash, Fragment +from iota.crypto import HASH_LENGTH +from iota.crypto.kerl import Kerl + +def finalize(bundle): + sponge = Kerl() + last_index = len(bundle) - 1 + + for (i, txn) in enumerate(bundle): # type: Tuple[int, ProposedTransaction] + txn.current_index = i + txn.last_index = last_index + sponge.absorb(txn.get_signature_validation_trytes().as_trits()) + + bundle_hash_trits = [0] * HASH_LENGTH # type: MutableSequence[int] + sponge.squeeze(bundle_hash_trits) + bundle_hash = BundleHash.from_trits(bundle_hash_trits) + + for txn in bundle: + txn.bundle_hash = bundle_hash + # Initialize signature/message fragment. + txn.signature_message_fragment = Fragment(txn.message or b'') + + + + + diff --git a/Nightly-Tests/Sync-Tests/milestone.py b/Nightly-Tests/Sync-Tests/milestone.py new file mode 100644 index 0000000..13ca9e7 --- /dev/null +++ b/Nightly-Tests/Sync-Tests/milestone.py @@ -0,0 +1,56 @@ +from iota import TryteString, trits_from_int, ProposedTransaction, ProposedBundle, Tag, Address, Iota +import finalize +from yaml import full_load +import sys + +index = 1 +args = sys.argv + +print("Starting") +for arg in args: + if arg == '-i': + index = args[int(args.index(arg)) + 1] + +def int_to_trytes(int_input, length): + trits = trits_from_int(int(int_input)) + trytes = TryteString.from_trits(trits) + if len(trytes) < length: + trytes += '9' * (length - len(trytes)) + return trytes + +yaml_file = full_load(open('./output.yml', 'r')) +nodes = {} +keys = yaml_file.keys() +for key in keys: + if key != 'seeds' and key != 'defaults': + nodes[key] = yaml_file[key] + +host = yaml_file['nodes']['nodeA']['host'] +port = yaml_file['nodes']['nodeA']['ports']['api'] + +api = Iota('http://{}:{}'.format(host, port)) + +txn = ProposedTransaction( +address = Address('KSAFREMKHHYHSXNLGZPFVHANVHVMKWSGEAHGTXZCSQMXTCZXOGBLVPCWFKVAEQYDJMQALKZRKOTWLGBSC'), +value = 0 +) + +bundle = ProposedBundle() +bundle.add_transaction(txn) +bundle.add_transaction(txn) + +index_trytes = str(int_to_trytes(index, 9)) + +bundle[0]._legacy_tag = Tag(index_trytes) + +finalize.finalize(bundle) +bundle_trytes = bundle.as_tryte_strings() + +tips = api.get_transactions_to_approve(depth=3) +branch = tips['branchTransaction'] +trunk = tips['trunkTransaction'] + + +milestone_bundle = api.attach_to_tangle(trunk,branch, bundle_trytes,3) +api.broadcast_and_store(milestone_bundle.get('trytes')) +print("Milestone {} attached and stored: {}".format(index, milestone_bundle)) diff --git a/Nightly-Tests/Sync-Tests/requirements.txt b/Nightly-Tests/Sync-Tests/requirements.txt new file mode 100644 index 0000000..a9116ef --- /dev/null +++ b/Nightly-Tests/Sync-Tests/requirements.txt @@ -0,0 +1,6 @@ +virtualenv +python +python-pip +python-dev +python-tk +build-essential diff --git a/Nightly-Tests/Sync-Tests/runZMQScans.py b/Nightly-Tests/Sync-Tests/runZMQScans.py new file mode 100644 index 0000000..33341d8 --- /dev/null +++ b/Nightly-Tests/Sync-Tests/runZMQScans.py @@ -0,0 +1,183 @@ +from yaml import load, Loader +import zmq +import sys +from time import time +from iota import Iota +import os +import datetime +import json +import urllib3 + + +sys.path.insert(1, os.path.join(sys.path[0], '..')) + +from utils.sync_test_class import SyncTest +from utils import logger_tools as logging +from utils import graph_tools as graphing + + +def set_machine_config(): + yaml_path = './output.yml' + stream = open(yaml_path,'r') + yaml_file = load(stream,Loader=Loader) + + nodes = {} + keys = yaml_file.keys() + for key in keys: + if key == 'nodes': + for node in yaml_file[key]: + nodes[node] = {} + for node_key in yaml_file[key][node]: + if node_key != 'log': + nodes[node][node_key] = [] + nodes[node][node_key] = yaml_file[key][node][node_key] + + sync_test = SyncTest(nodes) + return sync_test + + + +def get_latest_solid_milestones(test, time_elapsed): + for node in test.get_nodes(): + api = Iota(test.get_api_address(node)) + response = api.get_node_info() + index = response.get('latestSolidSubtangleMilestoneIndex') + if test.get_latest_milestone() == 0: + index = response.get('latestMilestoneIndex') + if node == 'nodeC': + index = 0 + logger.info("Node: {} Index: {}".format(node, index)) + test.add_index(node, index, time_elapsed) + + + +def get_total_transactions(test, node): + headers = { + 'content-type': 'application/json', + 'X-IOTA-API-Version': '1' + } + + command = {'command': 'TotalTransactions.getTotalTransactions'} + command_string = json.dumps(command) + + logger.info("Sending command") + http = urllib3.PoolManager() + request = http.request("POST", test.get_api_address(node), body=command_string, headers=headers) + data = json.loads(request.data.decode('utf-8')) + total = data['ixi']['total'] + if total > test.get_num_transactions(node): + logger.info("Filling {}".format(total - test.get_num_transactions(node))) + test.set_num_transactions(node, total, time_elapsed) + + return total + + +def get_args(args): + global base_output_dir + for arg in args: + if arg == "-o": + base_output_dir = args[args.index(arg) + 1] + if not base_output_dir.endswith("/"): + base_output_dir += "/" + + +def scan_sockets(test, socket_list, socket_poll): + for node in socket_list: + socket = socket_list[node] + if socket in socket_poll and socket_poll[socket] == zmq.POLLIN: + received = socket.recv().split() + if received[0] == "lmsi": + data = received[2] + test.add_index(node, int(data), time_elapsed) + if int(data) == test.get_latest_milestone(): + logger.info("{} Synced".format(node)) + test.set_node_sync_status(node, True) + return {'node': node, 'index': data} + + elif received[0] == "tx" or received[0] == b"tx": + test.add_transaction(node, received[1], time_elapsed) + + return test.get_furthest_milestone() + + +def make_graphs(): + nodes = test.get_nodes() + for node in nodes: + if test.get_index_list_length(node) > 1: + graphing.make_graph(num_tests=test.get_index_list_length(node), + inputs=test.get_node_index_list(node), + file='{}-Sync.png'.format(node), + title='{} Sync Graph'.format(node), + test=test) + else: + logger.info('No graph was made for {}'.format(node)) + + +base_output_dir = './Output/' + +test = set_machine_config() + +get_args(sys.argv) + +print("Setting up logs") +test.set_base_directory(base_output_dir) +test.set_log_directory(base_output_dir + datetime.datetime.now().date().__str__() + "/") +logging.make_log_directory(test) + +logger = logging.get_sync_logger(test) +file_logger = logging.get_raw_logger(test) + +logger.info('Setting Syncing Milestone') +get_latest_solid_milestones(test, 0) + +context = zmq.Context() +sockets = {} + +logger.info("Generating Sockets") +poller = zmq.Poller() + +for node in test.get_nodes(): + sockets[node] = context.socket(zmq.SUB) + socket = sockets[node] + socket.connect(test.get_zmq_address(node)) + socket.setsockopt(zmq.SUBSCRIBE, b"lmsi") + socket.setsockopt(zmq.SUBSCRIBE, b"tx") + logger.info("Created Socket {} on {}".format(node, test.get_zmq_address(node))) + poller.register(socket, zmq.POLLIN) + +logger.info("Starting Test") +start = time() +iteration = 0 + +while True: + iteration += 1 + socket_poll = dict(poller.poll(500)) + data = test.get_furthest_milestone() + node = 'nodeC' + + time_elapsed = time() - start + if len(socket_poll) != 0: + data = scan_sockets(test, sockets, socket_poll) + node = data['node'] + indexes = test.get_node_index_list(node) + + sync_list = test.get_node_sync_list() + + if len(test.get_transactions(node)) % 1000 == 0 or all(sync_list[node] is True for node in sync_list) or time_elapsed % 60 < 1 or iteration % 1000 == 0: + logger.info("") + logger.info("Time elapsed: {}".format(int(time_elapsed))) + logger.info("Node states: {}".format(sync_list)) + logger.info("{} index: {}/{}\n".format(data['node'], data['index'], test.get_latest_milestone())) + logger.info("{} / {} transactions processed".format(get_total_transactions(test, 'nodeC'), get_total_transactions(test,'nodeA'))) + get_latest_solid_milestones(test, time_elapsed) + + if all(sync_list[state] is True for state in sync_list): + logger.info("Done") + logger.info(sync_list) + logger.info("Syncing took: {} seconds".format(time_elapsed)) + make_graphs() + + file_logger.info("nodeC\nindexes: \n{}\nindex timestamps: \n{}\ntransactions arrival timestamps: \n{}\n".format(test.get_node_index_list('nodeC'), test.get_node_index_list_timestamps('nodeC'), test.get_transactions_timestamps('nodeC'))) + + sys.exit() + diff --git a/Nightly-Tests/setup.py b/Nightly-Tests/setup.py index 3c5edd4..d912fd3 100644 --- a/Nightly-Tests/setup.py +++ b/Nightly-Tests/setup.py @@ -10,6 +10,8 @@ install_requires=[ "PyOTA == 2.0.7", "psutil == 5.6.1", - "matplotlib == 2.2.4" + "matplotlib == 2.2.4", + "pyzmq == 18.0.1", + "pyYAML == 5.1.2" ], ) diff --git a/Nightly-Tests/utils/graph_tools.py b/Nightly-Tests/utils/graph_tools.py index 2893d21..3222594 100644 --- a/Nightly-Tests/utils/graph_tools.py +++ b/Nightly-Tests/utils/graph_tools.py @@ -1,8 +1,7 @@ from os import environ import matplotlib -if 'DISPLAY' not in environ: - matplotlib.use('Agg') +matplotlib.use('agg') import matplotlib.pyplot as plt import math @@ -10,12 +9,38 @@ def make_graph(num_tests, inputs, file, title, test): log_directory = test.get_log_directory() - y_max = 100 - y_ticks = [] + class_name = test.__class__.__name__ + + if class_name == 'Test': + make_scan_graphs(num_tests, inputs, log_directory, file, title) + elif class_name == 'SyncTest': + furthest_milestone = inputs[0] + latest_milestone = inputs[-1] + sync_indexes = (furthest_milestone, latest_milestone) + + node = file.split('-')[0] + x_axis = test.get_node_index_list_timestamps(node) + make_sync_graphs(x_axis, inputs, log_directory, file, title, sync_indexes) + + transaction_indexes = [] + if len(test.get_transactions(node)) > 1: + for i in range(len(test.get_transactions(node))): + transaction_indexes.append(i + 1) + + sync_indexes = (0, transaction_indexes[-1]) + make_sync_graphs(test.get_transactions_timestamps(node), transaction_indexes, log_directory, '{}-Transaction-Sync.png'.format(node), 'Transactions Processed', sync_indexes) + else: + raise ValueError('Test class "{}" is not supported'.format(class_name)) + + +def make_scan_graphs(num_tests, inputs, log_directory, file, title): assert 'iri' in inputs, 'Inputs must contain a list labeled "iri"' assert type(inputs['iri']) is list, '"iri" inputs must be in list form' + y_max = 100 + y_ticks = [] x_axis = [] + for x in range(num_tests): x_axis.append(x+1) @@ -38,7 +63,28 @@ def make_graph(num_tests, inputs, file, title, test): plt.title(title) plt.legend() + plt.savefig(log_directory + file) + plt.clf() + +def make_sync_graphs(x_axis, inputs, log_directory, file, title, sync_indexes): + assert len(inputs) > 1, 'Inputs must have length greater than 1' + assert type(inputs) is list, 'Inputs must be in list form' + + y_min = (sync_indexes[0]) + y_max = (sync_indexes[1] + 10) + y_ticks = [] + + plt.plot(x_axis, inputs, label='Sync Rate') + + for y in range(10): + y_ticks.append(math.ceil((y_max-y_min)/10) * (y+1) + y_min) + + plt.yticks(y_ticks, y_ticks) + + plt.ylim(y_min, y_max) + plt.title(title) + plt.legend() plt.savefig(log_directory + file) - plt.clf() + plt.clf() \ No newline at end of file diff --git a/Nightly-Tests/utils/logger_tools.py b/Nightly-Tests/utils/logger_tools.py index 2a6ae87..18ed21e 100644 --- a/Nightly-Tests/utils/logger_tools.py +++ b/Nightly-Tests/utils/logger_tools.py @@ -21,8 +21,18 @@ def get_raw_logger(test): return raw_logger +def get_sync_logger(test): + log_file = test.log_directory + "sync.log" + logging.basicConfig(level=logging.INFO) + handler = logging.FileHandler(log_file) + sync_logger = logging.getLogger("Sync") + sync_logger.addHandler(handler) + return sync_logger + + def make_log_directory(test): directories = [test.get_base_directory(), test.get_log_directory()] + for directory in directories: if os.path.exists(directory): if directory == test.get_base_directory(): diff --git a/Nightly-Tests/utils/sync_test_class.py b/Nightly-Tests/utils/sync_test_class.py new file mode 100644 index 0000000..9179498 --- /dev/null +++ b/Nightly-Tests/utils/sync_test_class.py @@ -0,0 +1,179 @@ +class SyncTest: + + base_directory = "./" + log_directory = "./" + + latest_milestone = 0 + furthest_milestone = {'node': None, 'index': 0} + + nodes = {} + + zmq_addresses = {} + api_addresses = {} + + node_starting_indexes = {} + node_index_timestamps = {} + node_indexes = {} + node_synced = {} + + transactions = {} + transactions_timestamps = {} + + num_transactions = {} + + def __init__(self, nodes): + # Nodes in cluster, pulled from output.yml + self.set_nodes(nodes) + + def get_nodes(self): + return self.nodes + + def set_nodes(self, nodes): + self.nodes = nodes + self.set_zmq_addresses() + self.set_api_addresses() + self.setup_index_lists() + + def set_zmq_addresses(self): + for node in self.nodes: + host = self.nodes[node]['host'] + port = self.nodes[node]['ports']['zmq-feed'] + address = "tcp://{}:{}".format(host, port) + self.zmq_addresses[node] = address + + def set_api_addresses(self): + for node in self.nodes: + host = self.nodes[node]['host'] + port = self.nodes[node]['ports']['api'] + address = "http://{}:{}".format(host, port) + self.api_addresses[node] = address + + def setup_index_lists(self): + for node in self.nodes: + self.node_indexes[node] = [] + self.node_index_timestamps[node] = [] + self.transactions[node] = [] + self.num_transactions[node] = 0 + self.transactions_timestamps[node] = [] + self.node_synced[node] = False + + + + + + def get_zmq_address(self, node): + if node in self.zmq_addresses: + return self.zmq_addresses[node] + else: + raise IndexError("No ZMQ address found for {}".format(node)) + + + def get_api_address(self, node): + if node in self.api_addresses: + return self.api_addresses[node] + else: + raise IndexError("No API address found for {}".format(node)) + + + + + ## Node Sync + def get_node_sync_list(self): + return self.node_synced + + def set_node_sync_status(self, node, value): + self.node_synced[node] = value + + def get_node_sync_status(self, node): + return self.node_synced[node] + + + + + def add_index(self, node, index, time): + if node in self.node_indexes: + self.node_indexes[node].append(index) + self.node_index_timestamps[node].append(time) + current_index = self.get_latest_milestone() + + if current_index == 0 and node == 'nodeA': + self.set_latest_milestone(index) + + if index < self.furthest_milestone['index'] or self.furthest_milestone['node'] is None: + self.set_furthest_milestone(node, index) + + if index > self.furthest_milestone['index'] and node == self.furthest_milestone['node']: + self.set_furthest_milestone(node, index) + + if self.node_indexes[node][-1] == self.get_latest_milestone(): + self.node_synced[node] = True + + else: + raise IndexError("{} has no stored indexes".format(node)) + + + def get_node_indexes(self): + return self.node_indexes + + def get_node_index_list(self, node): + return self.node_indexes[node] + + def get_node_index_list_timestamps(self, node): + return self.node_index_timestamps[node] + + def get_index_list_length(self, node): + return len(self.node_indexes[node]) + + + + + def get_latest_milestone(self): + return self.latest_milestone + + def set_latest_milestone(self, index): + self.latest_milestone = index + + def set_furthest_milestone(self, node, index): + self.furthest_milestone['node'] = node + self.furthest_milestone['index'] = index + + def get_furthest_milestone(self): + return self.furthest_milestone + + def get_transactions(self, node): + return self.transactions[node] + + def get_transactions_timestamps(self, node): + return self.transactions_timestamps[node] + + def add_transaction(self, node, tx, time): + if node in self.node_indexes: + self.transactions[node].append(tx) + self.transactions_timestamps[node].append(time) + self.num_transactions[node] += 1 + + def get_num_transactions(self, node): + return self.num_transactions[node] + + def set_num_transactions(self, node, num_transactions, time): + if self.num_transactions[node] < num_transactions: + self.transactions_timestamps[node].append(time) + self.num_transactions[node] = num_transactions + for x in range(num_transactions - len(self.transactions[node])): + self.transactions[node].append('N/A') + for y in range(num_transactions - len(self.transactions_timestamps[node])): + self.transactions_timestamps[node].append(time) + + + def get_log_directory(self): + return self.log_directory + + def set_log_directory(self, directory): + self.log_directory = directory + + def get_base_directory(self): + return self.base_directory + + def set_base_directory(self, dir): + self.base_directory = dir +