From a050d4393d9414eb8a748372d09a76bd3441ec02 Mon Sep 17 00:00:00 2001
From: Nikolay Stanchev <ns17@it-innovation.soton.ac.uk>
Date: Thu, 21 Jun 2018 11:35:15 +0100
Subject: [PATCH] Clearing up the CLMC service and separating aggregation API
from a potential whoami API
---
src/service/clmcservice/__init__.py | 12 +++++------
.../clmcservice/aggregation/__init__.py | 2 ++
.../clmcservice/aggregation/aggregator.py | 2 +-
.../aggregation/influx_data_interface.py | 2 +-
.../clmcservice/aggregationapi/__init__.py | 1 +
.../clmcservice/{ => aggregationapi}/tests.py | 20 +++++++++----------
.../{ => aggregationapi}/utilities.py | 0
.../clmcservice/{ => aggregationapi}/views.py | 9 +++++----
src/service/clmcservice/whoamiapi/__init__.py | 0
src/service/clmcservice/whoamiapi/tests.py | 0
src/service/clmcservice/whoamiapi/views.py | 0
src/service/setup.py | 16 +++++++--------
.../clmctest/monitoring/test_e2eresults.py | 1 -
13 files changed, 34 insertions(+), 31 deletions(-)
create mode 100644 src/service/clmcservice/aggregationapi/__init__.py
rename src/service/clmcservice/{ => aggregationapi}/tests.py (94%)
rename src/service/clmcservice/{ => aggregationapi}/utilities.py (100%)
rename src/service/clmcservice/{ => aggregationapi}/views.py (97%)
create mode 100644 src/service/clmcservice/whoamiapi/__init__.py
create mode 100644 src/service/clmcservice/whoamiapi/tests.py
create mode 100644 src/service/clmcservice/whoamiapi/views.py
diff --git a/src/service/clmcservice/__init__.py b/src/service/clmcservice/__init__.py
index a3e6c20..648e1d8 100644
--- a/src/service/clmcservice/__init__.py
+++ b/src/service/clmcservice/__init__.py
@@ -23,7 +23,7 @@
"""
from pyramid.config import Configurator
-from clmcservice.utilities import validate_conf_file, RUNNING_FLAG, MALFORMED_FLAG, CONF_FILE_ATTRIBUTE, CONF_OBJECT, AGGREGATOR_CONFIG_SECTION
+from clmcservice.aggregationapi.utilities import validate_conf_file, RUNNING_FLAG, MALFORMED_FLAG, CONF_FILE_ATTRIBUTE, CONF_OBJECT, AGGREGATOR_CONFIG_SECTION
def main(global_config, **settings):
@@ -41,15 +41,15 @@ def main(global_config, **settings):
config = Configurator(settings=settings)
config.add_route('aggregator_config', '/aggregator/config')
- config.add_view('clmcservice.views.AggregatorConfig', attr='get', request_method='GET')
- config.add_view('clmcservice.views.AggregatorConfig', attr='put', request_method='PUT')
+ config.add_view('clmcservice.aggregationapi.views.AggregatorConfig', attr='get', request_method='GET')
+ config.add_view('clmcservice.aggregationapi.views.AggregatorConfig', attr='put', request_method='PUT')
config.add_route('aggregator_controller', '/aggregator/control')
- config.add_view('clmcservice.views.AggregatorController', attr='get', request_method='GET')
- config.add_view('clmcservice.views.AggregatorController', attr='put', request_method='PUT')
+ config.add_view('clmcservice.aggregationapi.views.AggregatorController', attr='get', request_method='GET')
+ config.add_view('clmcservice.aggregationapi.views.AggregatorController', attr='put', request_method='PUT')
config.add_route('round_trip_time_query', '/query/round-trip-time')
- config.add_view('clmcservice.views.RoundTripTimeQuery', attr='get', request_method='GET')
+ config.add_view('clmcservice.aggregationapi.views.RoundTripTimeQuery', attr='get', request_method='GET')
config.scan()
return config.make_wsgi_app()
diff --git a/src/service/clmcservice/aggregation/__init__.py b/src/service/clmcservice/aggregation/__init__.py
index e69de29..5a34cbc 100644
--- a/src/service/clmcservice/aggregation/__init__.py
+++ b/src/service/clmcservice/aggregation/__init__.py
@@ -0,0 +1,2 @@
+__all__ = ['aggregator']
+
diff --git a/src/service/clmcservice/aggregation/aggregator.py b/src/service/clmcservice/aggregation/aggregator.py
index f0a00d3..8b66ce8 100644
--- a/src/service/clmcservice/aggregation/aggregator.py
+++ b/src/service/clmcservice/aggregation/aggregator.py
@@ -26,7 +26,7 @@ from threading import Thread, Event
from influxdb import InfluxDBClient
from time import time, sleep
from urllib.parse import urlparse
-from clmcservice.utilities import generate_e2e_delay_report
+from clmcservice.aggregationapi.utilities import generate_e2e_delay_report
import getopt
import logging
diff --git a/src/service/clmcservice/aggregation/influx_data_interface.py b/src/service/clmcservice/aggregation/influx_data_interface.py
index f375469..c6781d0 100644
--- a/src/service/clmcservice/aggregation/influx_data_interface.py
+++ b/src/service/clmcservice/aggregation/influx_data_interface.py
@@ -23,7 +23,7 @@
"""
-from clmcservice.utilities import generate_e2e_delay_report
+from clmcservice.aggregationapi.utilities import generate_e2e_delay_report
"""
A python module which provides auxiliary functions to mimic the behaviour of an InfluxDBClient when unit testing the aggregator.
diff --git a/src/service/clmcservice/aggregationapi/__init__.py b/src/service/clmcservice/aggregationapi/__init__.py
new file mode 100644
index 0000000..1bbd927
--- /dev/null
+++ b/src/service/clmcservice/aggregationapi/__init__.py
@@ -0,0 +1 @@
+__all__ = ['utilities', 'views']
diff --git a/src/service/clmcservice/tests.py b/src/service/clmcservice/aggregationapi/tests.py
similarity index 94%
rename from src/service/clmcservice/tests.py
rename to src/service/clmcservice/aggregationapi/tests.py
index df9fccb..77b4282 100644
--- a/src/service/clmcservice/tests.py
+++ b/src/service/clmcservice/aggregationapi/tests.py
@@ -25,7 +25,7 @@
from pyramid import testing
from pyramid.httpexceptions import HTTPBadRequest
from time import sleep
-from clmcservice.utilities import CONF_FILE_ATTRIBUTE, CONF_OBJECT, AGGREGATOR_CONFIG_SECTION, CONFIG_ATTRIBUTES, PROCESS_ATTRIBUTE, RUNNING_FLAG, MALFORMED_FLAG, URL_REGEX
+from clmcservice.aggregationapi.utilities import CONF_FILE_ATTRIBUTE, CONF_OBJECT, AGGREGATOR_CONFIG_SECTION, CONFIG_ATTRIBUTES, PROCESS_ATTRIBUTE, RUNNING_FLAG, MALFORMED_FLAG, URL_REGEX
import pytest
import os
import signal
@@ -57,7 +57,7 @@ class TestAggregatorAPI(object):
Tests the GET method for the configuration of the aggregator.
"""
- from clmcservice.views import AggregatorConfig # nested import so that importing the class view is part of the test itself
+ from clmcservice.aggregationapi.views import AggregatorConfig # nested import so that importing the class view is part of the test itself
assert int(self.registry.get_settings()[CONF_OBJECT][AGGREGATOR_CONFIG_SECTION].get('aggregator_report_period')) == 5, "Initial report period is 5 seconds."
assert self.registry.get_settings()[CONF_OBJECT][AGGREGATOR_CONFIG_SECTION].get('aggregator_database_name') == 'CLMCMetrics', "Initial database name the aggregator uses is CLMCMetrics."
@@ -102,7 +102,7 @@ class TestAggregatorAPI(object):
:param output_value: the expected output value, None for expecting an Exception
"""
- from clmcservice.views import AggregatorConfig, AggregatorController # nested import so that importing the class view is part of the test itself
+ from clmcservice.aggregationapi.views import AggregatorConfig, AggregatorController # nested import so that importing the class view is part of the test itself
assert not AggregatorController.is_process_running(self.registry.get_settings().get(PROCESS_ATTRIBUTE)), "Initially aggregator is not running."
assert int(self.registry.get_settings()[CONF_OBJECT][AGGREGATOR_CONFIG_SECTION].get('aggregator_report_period')) == 5, "Initial report period is 5 seconds."
@@ -144,7 +144,7 @@ class TestAggregatorAPI(object):
Tests starting the aggregator through an API call.
"""
- from clmcservice.views import AggregatorController # nested import so that importing the class view is part of the test itself
+ from clmcservice.aggregationapi.views import AggregatorController # nested import so that importing the class view is part of the test itself
assert not AggregatorController.is_process_running(self.registry.get_settings().get(PROCESS_ATTRIBUTE)), "Initially aggregator is not running."
assert self.registry.get_settings().get(PROCESS_ATTRIBUTE) is None, "Initially no aggregator process is running."
@@ -167,7 +167,7 @@ class TestAggregatorAPI(object):
Tests stopping the aggregator through an API call.
"""
- from clmcservice.views import AggregatorController # nested import so that importing the class view is part of the test itself
+ from clmcservice.aggregationapi.views import AggregatorController # nested import so that importing the class view is part of the test itself
assert not AggregatorController.is_process_running(self.registry.get_settings().get(PROCESS_ATTRIBUTE)), "Initially aggregator is not running."
assert self.registry.get_settings().get(PROCESS_ATTRIBUTE) is None, "Initially no aggregator process is running."
@@ -207,7 +207,7 @@ class TestAggregatorAPI(object):
Tests restarting the aggregator through an API call.
"""
- from clmcservice.views import AggregatorController # nested import so that importing the class view is part of the test itself
+ from clmcservice.aggregationapi.views import AggregatorController # nested import so that importing the class view is part of the test itself
assert not AggregatorController.is_process_running(self.registry.get_settings().get(PROCESS_ATTRIBUTE)), "Initially aggregator is not running."
assert self.registry.get_settings().get(PROCESS_ATTRIBUTE) is None, "Initially no aggregator process is running."
@@ -250,7 +250,7 @@ class TestAggregatorAPI(object):
Tests sending a malformed type of action to the aggregator through an API call.
"""
- from clmcservice.views import AggregatorController # nested import so that importing the class view is part of the test itself
+ from clmcservice.aggregationapi.views import AggregatorController # nested import so that importing the class view is part of the test itself
assert not AggregatorController.is_process_running(self.registry.get_settings().get(PROCESS_ATTRIBUTE)), "Initially aggregator is not running."
assert self.registry.get_settings().get(PROCESS_ATTRIBUTE) is None, "Initially no aggregator process is running."
@@ -273,7 +273,7 @@ class TestAggregatorAPI(object):
Tests the GET method for the status of the aggregator.
"""
- from clmcservice.views import AggregatorController # nested import so that importing the class view is part of the test itself
+ from clmcservice.aggregationapi.views import AggregatorController # nested import so that importing the class view is part of the test itself
assert not AggregatorController.is_process_running(self.registry.get_settings().get(PROCESS_ATTRIBUTE)), "Initially aggregator is not running."
assert self.registry.get_settings().get(PROCESS_ATTRIBUTE) is None, "Initially no aggregator process is running."
@@ -315,7 +315,7 @@ class TestAggregatorAPI(object):
Tests the behaviour of the malformed configuration flag of the aggregator when doing a sequence of API calls.
"""
- from clmcservice.views import AggregatorController, AggregatorConfig # nested import so that importing the class view is part of the test itself
+ from clmcservice.aggregationapi.views import AggregatorController, AggregatorConfig # nested import so that importing the class view is part of the test itself
assert not AggregatorController.is_process_running(self.registry.get_settings().get(PROCESS_ATTRIBUTE)), "Initially aggregator is not running."
assert not self.registry.get_settings().get(MALFORMED_FLAG), "Initially aggregator is not in a malformed state"
@@ -391,7 +391,7 @@ class TestAggregatorAPI(object):
Tests the behaviour of the service when in unconfigured state.
"""
- from clmcservice.views import AggregatorConfig, AggregatorController
+ from clmcservice.aggregationapi.views import AggregatorConfig, AggregatorController
self.registry.get_settings()[CONF_OBJECT] = None # unconfigured state - conf object is None
diff --git a/src/service/clmcservice/utilities.py b/src/service/clmcservice/aggregationapi/utilities.py
similarity index 100%
rename from src/service/clmcservice/utilities.py
rename to src/service/clmcservice/aggregationapi/utilities.py
diff --git a/src/service/clmcservice/views.py b/src/service/clmcservice/aggregationapi/views.py
similarity index 97%
rename from src/service/clmcservice/views.py
rename to src/service/clmcservice/aggregationapi/views.py
index 906c1c4..1e8c9db 100644
--- a/src/service/clmcservice/views.py
+++ b/src/service/clmcservice/aggregationapi/views.py
@@ -27,7 +27,7 @@ from pyramid.httpexceptions import HTTPBadRequest, HTTPInternalServerError
from influxdb import InfluxDBClient
from urllib.parse import urlparse
from subprocess import Popen
-from clmcservice.utilities import validate_config_content, validate_action_content, validate_round_trip_query_params, \
+from clmcservice.aggregationapi.utilities import validate_config_content, validate_action_content, validate_round_trip_query_params, \
CONF_OBJECT, CONF_FILE_ATTRIBUTE, AGGREGATOR_CONFIG_SECTION, CONFIG_ATTRIBUTES, ROUND_TRIP_ATTRIBUTES, RUNNING_FLAG, PROCESS_ATTRIBUTE, MALFORMED_FLAG, COMMENT_ATTRIBUTE, COMMENT_VALUE
import os
import os.path
@@ -215,11 +215,10 @@ class AggregatorController(object):
:return: the process object of the started aggregator script
"""
- dir_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'aggregation')
python_interpreter = sys.executable
- command = [python_interpreter, 'aggregator.py', '--period', str(config.get('aggregator_report_period')), '--database',
+ command = [python_interpreter, '-m', 'clmcservice.aggregation.aggregator', '--period', str(config.get('aggregator_report_period')), '--database',
config.get('aggregator_database_name'), '--url', config.get('aggregator_database_url')]
- process = Popen(command, cwd=dir_path)
+ process = Popen(command)
log.info("\nStarted aggregator process with PID: {0}\n".format(process.pid))
@@ -254,6 +253,8 @@ class AggregatorController(object):
@view_defaults(route_name='round_trip_time_query', renderer='json')
class RoundTripTimeQuery(object):
+ # TODO This API endpoint has not been tested, neither has the formula used to calculate round trip time.
+
"""
A class-based view for querying the round trip time in a given range.
"""
diff --git a/src/service/clmcservice/whoamiapi/__init__.py b/src/service/clmcservice/whoamiapi/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/src/service/clmcservice/whoamiapi/tests.py b/src/service/clmcservice/whoamiapi/tests.py
new file mode 100644
index 0000000..e69de29
diff --git a/src/service/clmcservice/whoamiapi/views.py b/src/service/clmcservice/whoamiapi/views.py
new file mode 100644
index 0000000..e69de29
diff --git a/src/service/setup.py b/src/service/setup.py
index 802f741..385f0c9 100644
--- a/src/service/setup.py
+++ b/src/service/setup.py
@@ -56,15 +56,15 @@ tests_require = [
]
setup(
- name = "clmcservice",
- version = get_version("_version.py"),
- author = "Michael Boniface",
- author_email = "mjb@it-innovation.soton.ac.uk",
- description = "FLAME CLMC Service Module",
+ name="clmcservice",
+ version=get_version("_version.py"),
+ author="Michael Boniface",
+ author_email="mjb@it-innovation.soton.ac.uk",
+ description="FLAME CLMC Service Module",
long_description="FLAME CLMC Service",
- license = "https://gitlab.it-innovation.soton.ac.uk/FLAME/flame-clmc/blob/integration/LICENSE",
- keywords = "FLAME CLMC service",
- url = 'https://gitlab.it-innovation.soton.ac.uk/FLAME/flame-clmc',
+ license="https://gitlab.it-innovation.soton.ac.uk/FLAME/flame-clmc/blob/integration/LICENSE",
+ keywords="FLAME CLMC service",
+ url='https://gitlab.it-innovation.soton.ac.uk/FLAME/flame-clmc',
packages=find_packages(),
include_package_data=True,
install_requires=requires,
diff --git a/src/test/clmctest/monitoring/test_e2eresults.py b/src/test/clmctest/monitoring/test_e2eresults.py
index 746d922..9c957d6 100644
--- a/src/test/clmctest/monitoring/test_e2eresults.py
+++ b/src/test/clmctest/monitoring/test_e2eresults.py
@@ -23,7 +23,6 @@
"""
import pytest
-import random
import time
import requests
import urllib.parse
--
GitLab