Skip to content
Snippets Groups Projects
Commit 36e2bbf5 authored by Nikolay Stanchev's avatar Nikolay Stanchev
Browse files

Implements file-based configuration of the CLMC service

parent b4c41bbf
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
scripts/clmc-service/install.sh
+ 4
− 0
View file @ 36e2bbf5
...@@ -162,6 +162,10 @@ fi ...@@ -162,6 +162,10 @@ fi
echo "----> Creating CLMC web service log directory" echo "----> Creating CLMC web service log directory"
mkdir -p /var/log/flame/clmc mkdir -p /var/log/flame/clmc
# create directory for CLMC service config
echo "----> Creating CLMC web service config directory"
mkdir -p /etc/flame/clmc
# Install minioclmc as systemctl service # Install minioclmc as systemctl service
# ----------------------------------------------------------------------- # -----------------------------------------------------------------------
mkdir -p /opt/flame/clmc mkdir -p /opt/flame/clmc
......
src/service/clmcservice/__init__.py
+ 6
− 4
View file @ 36e2bbf5
#!/usr/bin/python3
""" """
// © University of Southampton IT Innovation Centre, 2018 // © University of Southampton IT Innovation Centre, 2018
// //
...@@ -22,7 +23,7 @@ ...@@ -22,7 +23,7 @@
""" """
from pyramid.config import Configurator from pyramid.config import Configurator
from clmcservice.utilities import RUNNING_FLAG, MALFORMED_FLAG from clmcservice.utilities import validate_conf_file, RUNNING_FLAG, MALFORMED_FLAG, CONF_FILE_ATTRIBUTE, CONF_OBJECT, AGGREGATOR_CONFIG_SECTION
def main(global_config, **settings): def main(global_config, **settings):
...@@ -30,9 +31,10 @@ def main(global_config, **settings): ...@@ -30,9 +31,10 @@ def main(global_config, **settings):
This function returns a Pyramid WSGI application. This function returns a Pyramid WSGI application.
""" """
# a conversion is necessary so that the configuration values of the aggregator are stored with the right type instead of strings # validate and use (if valid) the configuration file
aggregator_report_period = int(settings.get('aggregator_report_period', 5)) conf_file_path = settings[CONF_FILE_ATTRIBUTE]
settings['aggregator_report_period'] = aggregator_report_period conf = validate_conf_file(conf_file_path) # if None returned here, service is in unconfigured state
settings[CONF_OBJECT] = conf
settings[MALFORMED_FLAG] = False settings[MALFORMED_FLAG] = False
......
src/service/clmcservice/tests.py
+ 124
− 64
View file @ 36e2bbf5
#!/usr/bin/python3
""" """
// © University of Southampton IT Innovation Centre, 2018 // © University of Southampton IT Innovation Centre, 2018
// //
...@@ -21,13 +22,14 @@ ...@@ -21,13 +22,14 @@
// Created for Project : FLAME // Created for Project : FLAME
""" """
import pytest
from pyramid import testing from pyramid import testing
from pyramid.httpexceptions import HTTPBadRequest from pyramid.httpexceptions import HTTPBadRequest
from time import sleep from time import sleep
from clmcservice.utilities import CONFIG_ATTRIBUTES, PROCESS_ATTRIBUTE, RUNNING_FLAG, MALFORMED_FLAG, URL_REGEX from clmcservice.utilities import CONF_FILE_ATTRIBUTE, CONF_OBJECT, AGGREGATOR_CONFIG_SECTION, CONFIG_ATTRIBUTES, PROCESS_ATTRIBUTE, RUNNING_FLAG, MALFORMED_FLAG, URL_REGEX
import pytest
import os import os
import signal import signal
import configparser
class TestAggregatorAPI(object): class TestAggregatorAPI(object):
...@@ -41,9 +43,10 @@ class TestAggregatorAPI(object): ...@@ -41,9 +43,10 @@ class TestAggregatorAPI(object):
A fixture to implement setUp/tearDown functionality for all tests by initializing configuration structure for the web service A fixture to implement setUp/tearDown functionality for all tests by initializing configuration structure for the web service
""" """
self.config = testing.setUp() self.registry = testing.setUp()
self.config.add_settings({'aggregator_running': False, 'malformed': False, 'aggregator_report_period': 5, config = configparser.ConfigParser()
'aggregator_database_name': 'CLMCMetrics', 'aggregator_database_url': "http://172.40.231.51:8086"}) config[AGGREGATOR_CONFIG_SECTION] = {'aggregator_report_period': 5, 'aggregator_database_name': 'CLMCMetrics', 'aggregator_database_url': "http://172.40.231.51:8086"}
self.registry.add_settings({'configuration_object': config, 'aggregator_running': False, 'malformed': False, 'configuration_file_path': "/etc/flame/clmc/service.conf"})
yield yield
...@@ -56,9 +59,9 @@ class TestAggregatorAPI(object): ...@@ -56,9 +59,9 @@ class TestAggregatorAPI(object):
from clmcservice.views import AggregatorConfig # nested import so that importing the class view is part of the test itself from clmcservice.views import AggregatorConfig # nested import so that importing the class view is part of the test itself
assert self.config.get_settings().get('aggregator_report_period') == 5, "Initial report period is 5 seconds." assert int(self.registry.get_settings()[CONF_OBJECT][AGGREGATOR_CONFIG_SECTION].get('aggregator_report_period')) == 5, "Initial report period is 5 seconds."
assert self.config.get_settings().get('aggregator_database_name') == 'CLMCMetrics', "Initial database name the aggregator uses is CLMCMetrics." assert self.registry.get_settings()[CONF_OBJECT][AGGREGATOR_CONFIG_SECTION].get('aggregator_database_name') == 'CLMCMetrics', "Initial database name the aggregator uses is CLMCMetrics."
assert self.config.get_settings().get('aggregator_database_url') == "http://172.40.231.51:8086", "Initial aggregator url is http://172.40.231.51:8086" assert self.registry.get_settings()[CONF_OBJECT][AGGREGATOR_CONFIG_SECTION].get('aggregator_database_url') == "http://172.40.231.51:8086", "Initial aggregator url is http://172.40.231.51:8086"
request = testing.DummyRequest() request = testing.DummyRequest()
response = AggregatorConfig(request).get() response = AggregatorConfig(request).get()
...@@ -67,9 +70,9 @@ class TestAggregatorAPI(object): ...@@ -67,9 +70,9 @@ class TestAggregatorAPI(object):
'aggregator_database_name': 'CLMCMetrics', 'aggregator_database_name': 'CLMCMetrics',
'aggregator_database_url': "http://172.40.231.51:8086"}, "Response must be a dictionary representing a JSON object with the correct configuration data of the aggregator." 'aggregator_database_url': "http://172.40.231.51:8086"}, "Response must be a dictionary representing a JSON object with the correct configuration data of the aggregator."
assert self.config.get_settings().get('aggregator_report_period') == 5, "A GET request must not modify the aggregator configuration data." assert int(self.registry.get_settings()[CONF_OBJECT][AGGREGATOR_CONFIG_SECTION].get('aggregator_report_period')) == 5, "A GET request must not modify the aggregator configuration data."
assert self.config.get_settings().get('aggregator_database_name') == 'CLMCMetrics', "A GET request must not modify the aggregator configuration data." assert self.registry.get_settings()[CONF_OBJECT][AGGREGATOR_CONFIG_SECTION].get('aggregator_database_name') == 'CLMCMetrics', "A GET request must not modify the aggregator configuration data."
assert self.config.get_settings().get('aggregator_database_url') == "http://172.40.231.51:8086", "A GET request must not modify the aggregator configuration data." assert self.registry.get_settings()[CONF_OBJECT][AGGREGATOR_CONFIG_SECTION].get('aggregator_database_url') == "http://172.40.231.51:8086", "A GET request must not modify the aggregator configuration data."
@pytest.mark.parametrize("input_body, output_value", [ @pytest.mark.parametrize("input_body, output_value", [
('{"aggregator_report_period": 10, "aggregator_database_name": "CLMCMetrics", "aggregator_database_url": "http://171.40.231.51:8086"}', ('{"aggregator_report_period": 10, "aggregator_database_name": "CLMCMetrics", "aggregator_database_url": "http://171.40.231.51:8086"}',
...@@ -101,10 +104,10 @@ class TestAggregatorAPI(object): ...@@ -101,10 +104,10 @@ class TestAggregatorAPI(object):
from clmcservice.views import AggregatorConfig, AggregatorController # nested import so that importing the class view is part of the test itself from clmcservice.views import AggregatorConfig, AggregatorController # nested import so that importing the class view is part of the test itself
assert not AggregatorController.is_process_running(self.config.get_settings().get(PROCESS_ATTRIBUTE)), "Initially aggregator is not running." assert not AggregatorController.is_process_running(self.registry.get_settings().get(PROCESS_ATTRIBUTE)), "Initially aggregator is not running."
assert self.config.get_settings().get('aggregator_report_period') == 5, "Initial report period is 5 seconds." assert int(self.registry.get_settings()[CONF_OBJECT][AGGREGATOR_CONFIG_SECTION].get('aggregator_report_period')) == 5, "Initial report period is 5 seconds."
assert self.config.get_settings().get('aggregator_database_name') == 'CLMCMetrics', "Initial database name the aggregator uses is CLMCMetrics." assert self.registry.get_settings()[CONF_OBJECT][AGGREGATOR_CONFIG_SECTION].get('aggregator_database_name') == 'CLMCMetrics', "Initial database name the aggregator uses is CLMCMetrics."
assert self.config.get_settings().get('aggregator_database_url') == "http://172.40.231.51:8086", "Initial aggregator url is http://172.40.231.51:8086" assert self.registry.get_settings()[CONF_OBJECT][AGGREGATOR_CONFIG_SECTION].get('aggregator_database_url') == "http://172.40.231.51:8086", "Initial aggregator url is http://172.40.231.51:8086"
request = testing.DummyRequest() request = testing.DummyRequest()
request.body = input_body.encode(request.charset) request.body = input_body.encode(request.charset)
...@@ -114,9 +117,19 @@ class TestAggregatorAPI(object): ...@@ -114,9 +117,19 @@ class TestAggregatorAPI(object):
assert response == output_value, "Response of PUT request must include the new configuration of the aggregator" assert response == output_value, "Response of PUT request must include the new configuration of the aggregator"
for attribute in CONFIG_ATTRIBUTES: for attribute in CONFIG_ATTRIBUTES:
assert self.config.get_settings().get(attribute) == output_value.get(attribute), "Aggregator settings configuration is not updated." assert self.registry.get_settings()[CONF_OBJECT][AGGREGATOR_CONFIG_SECTION][attribute] == str(output_value[attribute]), "Aggregator settings configuration is not updated."
assert not AggregatorController.is_process_running(self.registry.get_settings().get(PROCESS_ATTRIBUTE)), "Aggregator running status should not be updated after a configuration update."
# assert that the conf file is updated
updated_conf = configparser.ConfigParser()
conf_file = self.registry.get_settings().get(CONF_FILE_ATTRIBUTE)
assert updated_conf.read(conf_file) == [conf_file]
assert AGGREGATOR_CONFIG_SECTION in updated_conf.sections()
for attribute in CONFIG_ATTRIBUTES:
assert self.registry.get_settings()[CONF_OBJECT][AGGREGATOR_CONFIG_SECTION][attribute] == updated_conf[AGGREGATOR_CONFIG_SECTION][attribute], "Aggregator settings configuration is not updated."
assert not AggregatorController.is_process_running(self.config.get_settings().get(PROCESS_ATTRIBUTE)), "Aggregator running status should not be updated after a configuration update."
else: else:
error_raised = False error_raised = False
try: try:
...@@ -133,8 +146,8 @@ class TestAggregatorAPI(object): ...@@ -133,8 +146,8 @@ class TestAggregatorAPI(object):
from clmcservice.views import AggregatorController # nested import so that importing the class view is part of the test itself from clmcservice.views import AggregatorController # nested import so that importing the class view is part of the test itself
assert not AggregatorController.is_process_running(self.config.get_settings().get(PROCESS_ATTRIBUTE)), "Initially aggregator is not running." assert not AggregatorController.is_process_running(self.registry.get_settings().get(PROCESS_ATTRIBUTE)), "Initially aggregator is not running."
assert self.config.get_settings().get(PROCESS_ATTRIBUTE) is None, "Initially no aggregator process is running." assert self.registry.get_settings().get(PROCESS_ATTRIBUTE) is None, "Initially no aggregator process is running."
request = testing.DummyRequest() request = testing.DummyRequest()
input_body = '{"action": "start"}' input_body = '{"action": "start"}'
...@@ -142,8 +155,8 @@ class TestAggregatorAPI(object): ...@@ -142,8 +155,8 @@ class TestAggregatorAPI(object):
response = AggregatorController(request).put() response = AggregatorController(request).put()
assert response == {RUNNING_FLAG: True}, "The aggregator should have been started." assert response == {RUNNING_FLAG: True}, "The aggregator should have been started."
assert AggregatorController.is_process_running(self.config.get_settings().get(PROCESS_ATTRIBUTE)), "The aggregator should have been started." assert AggregatorController.is_process_running(self.registry.get_settings().get(PROCESS_ATTRIBUTE)), "The aggregator should have been started."
assert self.config.get_settings().get(PROCESS_ATTRIBUTE) is not None, "Aggregator process should have been initialized." assert self.registry.get_settings().get(PROCESS_ATTRIBUTE) is not None, "Aggregator process should have been initialized."
# kill the started process after the test is over # kill the started process after the test is over
pid = request.registry.settings[PROCESS_ATTRIBUTE].pid pid = request.registry.settings[PROCESS_ATTRIBUTE].pid
...@@ -156,16 +169,16 @@ class TestAggregatorAPI(object): ...@@ -156,16 +169,16 @@ class TestAggregatorAPI(object):
from clmcservice.views import AggregatorController # nested import so that importing the class view is part of the test itself from clmcservice.views import AggregatorController # nested import so that importing the class view is part of the test itself
assert not AggregatorController.is_process_running(self.config.get_settings().get(PROCESS_ATTRIBUTE)), "Initially aggregator is not running." assert not AggregatorController.is_process_running(self.registry.get_settings().get(PROCESS_ATTRIBUTE)), "Initially aggregator is not running."
assert self.config.get_settings().get(PROCESS_ATTRIBUTE) is None, "Initially no aggregator process is running." assert self.registry.get_settings().get(PROCESS_ATTRIBUTE) is None, "Initially no aggregator process is running."
# send a start request to trigger the aggregator # send a start request to trigger the aggregator
request = testing.DummyRequest() request = testing.DummyRequest()
input_body = '{"action": "start"}' input_body = '{"action": "start"}'
request.body = input_body.encode(request.charset) request.body = input_body.encode(request.charset)
AggregatorController(request).put() AggregatorController(request).put()
assert self.config.get_settings().get(PROCESS_ATTRIBUTE) is not None, "Aggregator process should have been initialized." assert self.registry.get_settings().get(PROCESS_ATTRIBUTE) is not None, "Aggregator process should have been initialized."
assert AggregatorController.is_process_running(self.config.get_settings().get(PROCESS_ATTRIBUTE)), "Aggregator process should have been initialized." assert AggregatorController.is_process_running(self.registry.get_settings().get(PROCESS_ATTRIBUTE)), "Aggregator process should have been initialized."
# test stopping the aggregator process when it is running # test stopping the aggregator process when it is running
request = testing.DummyRequest() request = testing.DummyRequest()
...@@ -174,8 +187,8 @@ class TestAggregatorAPI(object): ...@@ -174,8 +187,8 @@ class TestAggregatorAPI(object):
response = AggregatorController(request).put() response = AggregatorController(request).put()
assert response == {RUNNING_FLAG: False}, "The aggregator should have been stopped." assert response == {RUNNING_FLAG: False}, "The aggregator should have been stopped."
assert not AggregatorController.is_process_running(self.config.get_settings().get(PROCESS_ATTRIBUTE)), "The aggregator should have been stopped." assert not AggregatorController.is_process_running(self.registry.get_settings().get(PROCESS_ATTRIBUTE)), "The aggregator should have been stopped."
assert self.config.get_settings().get(PROCESS_ATTRIBUTE) is None, "Aggregator process should have been terminated." assert self.registry.get_settings().get(PROCESS_ATTRIBUTE) is None, "Aggregator process should have been terminated."
sleep(2) # put a 2 seconds timeout so that the aggregator process can terminate sleep(2) # put a 2 seconds timeout so that the aggregator process can terminate
...@@ -186,8 +199,8 @@ class TestAggregatorAPI(object): ...@@ -186,8 +199,8 @@ class TestAggregatorAPI(object):
response = AggregatorController(request).put() response = AggregatorController(request).put()
assert response == {RUNNING_FLAG: False}, "The aggregator should have been stopped." assert response == {RUNNING_FLAG: False}, "The aggregator should have been stopped."
assert not AggregatorController.is_process_running(self.config.get_settings().get(PROCESS_ATTRIBUTE)), "The aggregator should have been stopped." assert not AggregatorController.is_process_running(self.registry.get_settings().get(PROCESS_ATTRIBUTE)), "The aggregator should have been stopped."
assert self.config.get_settings().get(PROCESS_ATTRIBUTE) is None, "Aggregator process should have been terminated." assert self.registry.get_settings().get(PROCESS_ATTRIBUTE) is None, "Aggregator process should have been terminated."
def test_restart(self): def test_restart(self):
""" """
...@@ -196,8 +209,8 @@ class TestAggregatorAPI(object): ...@@ -196,8 +209,8 @@ class TestAggregatorAPI(object):
from clmcservice.views import AggregatorController # nested import so that importing the class view is part of the test itself from clmcservice.views import AggregatorController # nested import so that importing the class view is part of the test itself
assert not AggregatorController.is_process_running(self.config.get_settings().get(PROCESS_ATTRIBUTE)), "Initially aggregator is not running." assert not AggregatorController.is_process_running(self.registry.get_settings().get(PROCESS_ATTRIBUTE)), "Initially aggregator is not running."
assert self.config.get_settings().get(PROCESS_ATTRIBUTE) is None, "Initially no aggregator process is running." assert self.registry.get_settings().get(PROCESS_ATTRIBUTE) is None, "Initially no aggregator process is running."
# test restarting the aggregator process when it is stopped # test restarting the aggregator process when it is stopped
request = testing.DummyRequest() request = testing.DummyRequest()
...@@ -206,8 +219,8 @@ class TestAggregatorAPI(object): ...@@ -206,8 +219,8 @@ class TestAggregatorAPI(object):
response = AggregatorController(request).put() response = AggregatorController(request).put()
assert response == {RUNNING_FLAG: True}, "The aggregator should have been restarted." assert response == {RUNNING_FLAG: True}, "The aggregator should have been restarted."
assert AggregatorController.is_process_running(self.config.get_settings().get(PROCESS_ATTRIBUTE)), "The aggregator should have been restarted." assert AggregatorController.is_process_running(self.registry.get_settings().get(PROCESS_ATTRIBUTE)), "The aggregator should have been restarted."
assert self.config.get_settings().get(PROCESS_ATTRIBUTE), "The aggregator process should have been reinitialised." assert self.registry.get_settings().get(PROCESS_ATTRIBUTE), "The aggregator process should have been reinitialised."
# test restarting the aggregator process when it is running # test restarting the aggregator process when it is running
request = testing.DummyRequest() request = testing.DummyRequest()
...@@ -216,8 +229,8 @@ class TestAggregatorAPI(object): ...@@ -216,8 +229,8 @@ class TestAggregatorAPI(object):
response = AggregatorController(request).put() response = AggregatorController(request).put()
assert response == {RUNNING_FLAG: True}, "The aggregator should have been restarted." assert response == {RUNNING_FLAG: True}, "The aggregator should have been restarted."
assert AggregatorController.is_process_running(self.config.get_settings().get(PROCESS_ATTRIBUTE)), "The aggregator should have been restarted." assert AggregatorController.is_process_running(self.registry.get_settings().get(PROCESS_ATTRIBUTE)), "The aggregator should have been restarted."
assert self.config.get_settings().get(PROCESS_ATTRIBUTE), "The aggregator process should have been reinitialised." assert self.registry.get_settings().get(PROCESS_ATTRIBUTE), "The aggregator process should have been reinitialised."
# kill the started process after the test is over # kill the started process after the test is over
pid = request.registry.settings[PROCESS_ATTRIBUTE].pid pid = request.registry.settings[PROCESS_ATTRIBUTE].pid
...@@ -239,8 +252,8 @@ class TestAggregatorAPI(object): ...@@ -239,8 +252,8 @@ class TestAggregatorAPI(object):
from clmcservice.views import AggregatorController # nested import so that importing the class view is part of the test itself from clmcservice.views import AggregatorController # nested import so that importing the class view is part of the test itself
assert not AggregatorController.is_process_running(self.config.get_settings().get(PROCESS_ATTRIBUTE)), "Initially aggregator is not running." assert not AggregatorController.is_process_running(self.registry.get_settings().get(PROCESS_ATTRIBUTE)), "Initially aggregator is not running."
assert self.config.get_settings().get(PROCESS_ATTRIBUTE) is None, "Initially no aggregator process is running." assert self.registry.get_settings().get(PROCESS_ATTRIBUTE) is None, "Initially no aggregator process is running."
# test restarting the aggregator process when it is running # test restarting the aggregator process when it is running
request = testing.DummyRequest() request = testing.DummyRequest()
...@@ -262,16 +275,16 @@ class TestAggregatorAPI(object): ...@@ -262,16 +275,16 @@ class TestAggregatorAPI(object):
from clmcservice.views import AggregatorController # nested import so that importing the class view is part of the test itself from clmcservice.views import AggregatorController # nested import so that importing the class view is part of the test itself
assert not AggregatorController.is_process_running(self.config.get_settings().get(PROCESS_ATTRIBUTE)), "Initially aggregator is not running." assert not AggregatorController.is_process_running(self.registry.get_settings().get(PROCESS_ATTRIBUTE)), "Initially aggregator is not running."
assert self.config.get_settings().get(PROCESS_ATTRIBUTE) is None, "Initially no aggregator process is running." assert self.registry.get_settings().get(PROCESS_ATTRIBUTE) is None, "Initially no aggregator process is running."
request = testing.DummyRequest() request = testing.DummyRequest()
response = AggregatorController(request).get() response = AggregatorController(request).get()
assert response == {'aggregator_running': False}, "Response must be a dictionary representing a JSON object with the correct status data of the aggregator." assert response == {'aggregator_running': False}, "Response must be a dictionary representing a JSON object with the correct status data of the aggregator."
assert not AggregatorController.is_process_running(self.config.get_settings().get(PROCESS_ATTRIBUTE)), "A GET request must not modify the aggregator status flag." assert not AggregatorController.is_process_running(self.registry.get_settings().get(PROCESS_ATTRIBUTE)), "A GET request must not modify the aggregator status flag."
assert self.config.get_settings().get(PROCESS_ATTRIBUTE) is None, "A GET request must not start the aggregator process." assert self.registry.get_settings().get(PROCESS_ATTRIBUTE) is None, "A GET request must not start the aggregator process."
# test status with malformed configuration # test status with malformed configuration
# start the aggregator # start the aggregator
...@@ -279,7 +292,7 @@ class TestAggregatorAPI(object): ...@@ -279,7 +292,7 @@ class TestAggregatorAPI(object):
input_body = '{"action": "start"}' input_body = '{"action": "start"}'
request.body = input_body.encode(request.charset) request.body = input_body.encode(request.charset)
AggregatorController(request).put() AggregatorController(request).put()
self.config.get_settings()[MALFORMED_FLAG] = True self.registry.get_settings()[MALFORMED_FLAG] = True
request = testing.DummyRequest() request = testing.DummyRequest()
response = AggregatorController(request).get() response = AggregatorController(request).get()
...@@ -289,9 +302,9 @@ class TestAggregatorAPI(object): ...@@ -289,9 +302,9 @@ class TestAggregatorAPI(object):
'comment': 'Aggregator is running in a malformed state - it uses an old version of the configuration. Please, restart it so that the updated configuration is used.'}, \ 'comment': 'Aggregator is running in a malformed state - it uses an old version of the configuration. Please, restart it so that the updated configuration is used.'}, \
"Response must be a dictionary representing a JSON object with the correct status data of the aggregator." "Response must be a dictionary representing a JSON object with the correct status data of the aggregator."
assert AggregatorController.is_process_running(self.config.get_settings().get(PROCESS_ATTRIBUTE)), "A GET request must not modify the aggregator status flag." assert AggregatorController.is_process_running(self.registry.get_settings().get(PROCESS_ATTRIBUTE)), "A GET request must not modify the aggregator status flag."
assert self.config.get_settings().get(MALFORMED_FLAG), "A GET request must not modify the aggregator malformed flag." assert self.registry.get_settings().get(MALFORMED_FLAG), "A GET request must not modify the aggregator malformed flag."
assert self.config.get_settings().get(PROCESS_ATTRIBUTE) is not None, "A GET request must not stop the aggregator process." assert self.registry.get_settings().get(PROCESS_ATTRIBUTE) is not None, "A GET request must not stop the aggregator process."
# kill the started process after the test is over # kill the started process after the test is over
pid = request.registry.settings[PROCESS_ATTRIBUTE].pid pid = request.registry.settings[PROCESS_ATTRIBUTE].pid
...@@ -304,12 +317,12 @@ class TestAggregatorAPI(object): ...@@ -304,12 +317,12 @@ class TestAggregatorAPI(object):
from clmcservice.views import AggregatorController, AggregatorConfig # nested import so that importing the class view is part of the test itself from clmcservice.views import AggregatorController, AggregatorConfig # nested import so that importing the class view is part of the test itself
assert not AggregatorController.is_process_running(self.config.get_settings().get(PROCESS_ATTRIBUTE)), "Initially aggregator is not running." assert not AggregatorController.is_process_running(self.registry.get_settings().get(PROCESS_ATTRIBUTE)), "Initially aggregator is not running."
assert not self.config.get_settings().get(MALFORMED_FLAG), "Initially aggregator is not in a malformed state" assert not self.registry.get_settings().get(MALFORMED_FLAG), "Initially aggregator is not in a malformed state"
assert self.config.get_settings().get(PROCESS_ATTRIBUTE) is None, "Initially no aggregator process is running." assert self.registry.get_settings().get(PROCESS_ATTRIBUTE) is None, "Initially no aggregator process is running."
assert self.config.get_settings().get('aggregator_report_period') == 5, "Initial report period is 5 seconds." assert int(self.registry.get_settings()[CONF_OBJECT][AGGREGATOR_CONFIG_SECTION].get('aggregator_report_period')) == 5, "Initial report period is 5 seconds."
assert self.config.get_settings().get('aggregator_database_name') == 'CLMCMetrics', "Initial database name the aggregator uses is CLMCMetrics." assert self.registry.get_settings()[CONF_OBJECT][AGGREGATOR_CONFIG_SECTION].get('aggregator_database_name') == 'CLMCMetrics', "Initial database name the aggregator uses is CLMCMetrics."
assert self.config.get_settings().get('aggregator_database_url') == "http://172.40.231.51:8086", "Initial aggregator url is http://172.40.231.51:8086" assert self.registry.get_settings()[CONF_OBJECT][AGGREGATOR_CONFIG_SECTION].get('aggregator_database_url') == "http://172.40.231.51:8086", "Initial aggregator url is http://172.40.231.51:8086"
# start the aggregator with the default configuration # start the aggregator with the default configuration
request = testing.DummyRequest() request = testing.DummyRequest()
...@@ -328,9 +341,9 @@ class TestAggregatorAPI(object): ...@@ -328,9 +341,9 @@ class TestAggregatorAPI(object):
response = AggregatorConfig(request).put() response = AggregatorConfig(request).put()
assert response == output_body, "Response of PUT request must include the new configuration of the aggregator" assert response == output_body, "Response of PUT request must include the new configuration of the aggregator"
assert AggregatorController.is_process_running(self.config.get_settings().get(PROCESS_ATTRIBUTE)), "The aggregator shouldn't be stopped when the configuration is updated." assert AggregatorController.is_process_running(self.registry.get_settings().get(PROCESS_ATTRIBUTE)), "The aggregator shouldn't be stopped when the configuration is updated."
assert self.config.get_settings().get(MALFORMED_FLAG), "The malformed flag should be set when the configuration is updated while the process is running." assert self.registry.get_settings().get(MALFORMED_FLAG), "The malformed flag should be set when the configuration is updated while the process is running."
assert self.config.get_settings().get(PROCESS_ATTRIBUTE) is not None, "The aggregator shouldn't be stopped when the configuration is updated." assert self.registry.get_settings().get(PROCESS_ATTRIBUTE) is not None, "The aggregator shouldn't be stopped when the configuration is updated."
# check that the malformed flag has been updated through a GET call # check that the malformed flag has been updated through a GET call
request = testing.DummyRequest() request = testing.DummyRequest()
...@@ -346,9 +359,9 @@ class TestAggregatorAPI(object): ...@@ -346,9 +359,9 @@ class TestAggregatorAPI(object):
request.body = input_body.encode(request.charset) request.body = input_body.encode(request.charset)
response = AggregatorController(request).put() response = AggregatorController(request).put()
assert response == {RUNNING_FLAG: True}, "The aggregator should have been restarted." assert response == {RUNNING_FLAG: True}, "The aggregator should have been restarted."
assert AggregatorController.is_process_running(self.config.get_settings().get(PROCESS_ATTRIBUTE)), "The aggregator should have been restarted." assert AggregatorController.is_process_running(self.registry.get_settings().get(PROCESS_ATTRIBUTE)), "The aggregator should have been restarted."
assert not self.config.get_settings().get(MALFORMED_FLAG), "The malformed flag should have been reset to False." assert not self.registry.get_settings().get(MALFORMED_FLAG), "The malformed flag should have been reset to False."
assert self.config.get_settings().get(PROCESS_ATTRIBUTE) is not None, "The aggregator should have been restarted." assert self.registry.get_settings().get(PROCESS_ATTRIBUTE) is not None, "The aggregator should have been restarted."
# update the configuration again while the aggregator is running # update the configuration again while the aggregator is running
config_body = '{"aggregator_report_period": 30, "aggregator_database_name": "CLMCMetrics", "aggregator_database_url": "http://172.50.231.51:8086"}' config_body = '{"aggregator_report_period": 30, "aggregator_database_name": "CLMCMetrics", "aggregator_database_url": "http://172.50.231.51:8086"}'
...@@ -359,20 +372,67 @@ class TestAggregatorAPI(object): ...@@ -359,20 +372,67 @@ class TestAggregatorAPI(object):
response = AggregatorConfig(request).put() response = AggregatorConfig(request).put()
assert response == output_body, "Response of PUT request must include the new configuration of the aggregator" assert response == output_body, "Response of PUT request must include the new configuration of the aggregator"
assert AggregatorController.is_process_running(self.config.get_settings().get(PROCESS_ATTRIBUTE)), "The aggregator shouldn't be stopped when the configuration is updated." assert AggregatorController.is_process_running(self.registry.get_settings().get(PROCESS_ATTRIBUTE)), "The aggregator shouldn't be stopped when the configuration is updated."
assert self.config.get_settings().get(MALFORMED_FLAG), "The malformed flag should be set when the configuration is updated while the process is running." assert self.registry.get_settings().get(MALFORMED_FLAG), "The malformed flag should be set when the configuration is updated while the process is running."
assert self.config.get_settings().get(PROCESS_ATTRIBUTE) is not None, "The aggregator shouldn't be stopped when the configuration is updated." assert self.registry.get_settings().get(PROCESS_ATTRIBUTE) is not None, "The aggregator shouldn't be stopped when the configuration is updated."
# stop the aggregator - this should also reset the malformed status flag # stop the aggregator - this should also reset the malformed status flag
# restart the aggregator with the new configuration
request = testing.DummyRequest() request = testing.DummyRequest()
input_body = '{"action": "stop"}' input_body = '{"action": "stop"}'
request.body = input_body.encode(request.charset) request.body = input_body.encode(request.charset)
response = AggregatorController(request).put() response = AggregatorController(request).put()
assert response == {RUNNING_FLAG: False}, "The aggregator should have been stopped." assert response == {RUNNING_FLAG: False}, "The aggregator should have been stopped."
assert not AggregatorController.is_process_running(self.config.get_settings().get(PROCESS_ATTRIBUTE)), "The aggregator should have been stopped." assert not AggregatorController.is_process_running(self.registry.get_settings().get(PROCESS_ATTRIBUTE)), "The aggregator should have been stopped."
assert not self.config.get_settings().get(MALFORMED_FLAG), "The malformed flag should have been reset to False." assert not self.registry.get_settings().get(MALFORMED_FLAG), "The malformed flag should have been reset to False."
assert self.config.get_settings().get(PROCESS_ATTRIBUTE) is None, "The aggregator should have been stopped." assert self.registry.get_settings().get(PROCESS_ATTRIBUTE) is None, "The aggregator should have been stopped."
def test_unconfigured_state(self):
"""
Tests the behaviour of the service when in unconfigured state.
"""
from clmcservice.views import AggregatorConfig, AggregatorController
self.registry.get_settings()[CONF_OBJECT] = None # unconfigured state - conf object is None
# when doing a GET for the configuration we expect a bad request if the service is in unconfigured state
bad_request = False
bad_request_msg = None
try:
request = testing.DummyRequest()
AggregatorConfig(request).get()
except HTTPBadRequest as err:
bad_request = True
bad_request_msg = err.message
assert bad_request
assert bad_request_msg == "Aggregator has not been configured, yet. Send a PUT request to /aggregator/config with a JSON body of the configuration."
# when doing a PUT for the aggregator to start/stop/restart we expect a bad request if the service is in unconfigured state
for action in ('start', 'stop', 'restart'):
bad_request = False
bad_request_msg = None
try:
request = testing.DummyRequest()
request.body = ('{"action": "' + action + '"}').encode(request.charset)
AggregatorController(request).put()
except HTTPBadRequest as err:
bad_request = True
bad_request_msg = err.message
assert bad_request
assert bad_request_msg == "You must configure the aggregator before controlling it. Send a PUT request to /aggregator/config with a JSON body of the configuration."
# configure the aggregator
input_body = '{"aggregator_report_period": 10, "aggregator_database_name": "CLMCMetrics", "aggregator_database_url": "http://171.40.231.51:8086"}'
output_body = {'aggregator_report_period': 10, 'aggregator_database_name': "CLMCMetrics", 'aggregator_database_url': "http://171.40.231.51:8086"}
request = testing.DummyRequest()
request.body = input_body.encode(request.charset)
response = AggregatorConfig(request).put()
assert response == output_body
request = testing.DummyRequest()
assert AggregatorConfig(request).get() == output_body
class TestRegexURL(object): class TestRegexURL(object):
......
src/service/clmcservice/utilities.py
+ 36
− 0
View file @ 36e2bbf5
#!/usr/bin/python3
""" """
// © University of Southampton IT Innovation Centre, 2018 // © University of Southampton IT Innovation Centre, 2018
// //
...@@ -23,7 +24,12 @@ ...@@ -23,7 +24,12 @@
from json import loads from json import loads
from re import compile, IGNORECASE from re import compile, IGNORECASE
from configparser import ConfigParser
CONF_FILE_ATTRIBUTE = 'configuration_file_path' # the attribute pointing to the configuration file path
CONF_OBJECT = 'configuration_object' # the attribute, which stores the service configuration object
AGGREGATOR_CONFIG_SECTION = "AGGREGATOR" # the section in the configuration holding all the configuration attributes declared below
CONFIG_ATTRIBUTES = ('aggregator_report_period', 'aggregator_database_name', 'aggregator_database_url') # all of the configuration attributes - to be used as dictionary keys CONFIG_ATTRIBUTES = ('aggregator_report_period', 'aggregator_database_name', 'aggregator_database_url') # all of the configuration attributes - to be used as dictionary keys
RUNNING_FLAG = 'aggregator_running' # Attribute for storing the flag, which shows whether the aggregator is running or not - to be used as a dictionary key RUNNING_FLAG = 'aggregator_running' # Attribute for storing the flag, which shows whether the aggregator is running or not - to be used as a dictionary key
...@@ -121,6 +127,36 @@ def validate_round_trip_query_params(params): ...@@ -121,6 +127,36 @@ def validate_round_trip_query_params(params):
return params return params
def validate_conf_file(conf_file_path):
"""
Validates the aggregator's configuration file - checks for existence of the file path, whether it can be parsed as a configuration file and
whether it contains the required configuration attributes.
:param conf_file_path: the configuration file path to check
:return: the parsed configuration if valid, None otherwise
"""
global AGGREGATOR_CONFIG_SECTION, CONFIG_ATTRIBUTES
conf = ConfigParser()
result = conf.read(conf_file_path)
# if result doesn't contain one element, namely the conf_file_path,
# then the configuration file cannot be parsed for some reason (doesn't exist, cannot be opened, invalid, etc.)
if len(result) == 0:
return None
if AGGREGATOR_CONFIG_SECTION not in conf.sections():
return None # the config should include a section called AGGREGATOR
for key in CONFIG_ATTRIBUTES:
if key not in conf[AGGREGATOR_CONFIG_SECTION]:
return None # the configuration must include each configuration attribute
return conf
def generate_e2e_delay_report(path_id, source_sfr, target_sfr, endpoint, sf_instance, delay_forward, delay_reverse, delay_service, avg_request_size, avg_response_size, avg_bandwidth, time): def generate_e2e_delay_report(path_id, source_sfr, target_sfr, endpoint, sf_instance, delay_forward, delay_reverse, delay_service, avg_request_size, avg_response_size, avg_bandwidth, time):
""" """
Generates a combined averaged measurement about the e2e delay and its contributing parts Generates a combined averaged measurement about the e2e delay and its contributing parts
......
src/service/clmcservice/views.py
+ 69
− 27
View file @ 36e2bbf5
#!/usr/bin/python3
""" """
// © University of Southampton IT Innovation Centre, 2018 // © University of Southampton IT Innovation Centre, 2018
// //
...@@ -27,11 +28,12 @@ from influxdb import InfluxDBClient ...@@ -27,11 +28,12 @@ from influxdb import InfluxDBClient
from urllib.parse import urlparse from urllib.parse import urlparse
from subprocess import Popen from subprocess import Popen
from clmcservice.utilities import validate_config_content, validate_action_content, validate_round_trip_query_params, \ from clmcservice.utilities import validate_config_content, validate_action_content, validate_round_trip_query_params, \
CONFIG_ATTRIBUTES, ROUND_TRIP_ATTRIBUTES, RUNNING_FLAG, PROCESS_ATTRIBUTE, MALFORMED_FLAG, COMMENT_ATTRIBUTE, COMMENT_VALUE CONF_OBJECT, CONF_FILE_ATTRIBUTE, AGGREGATOR_CONFIG_SECTION, CONFIG_ATTRIBUTES, ROUND_TRIP_ATTRIBUTES, RUNNING_FLAG, PROCESS_ATTRIBUTE, MALFORMED_FLAG, COMMENT_ATTRIBUTE, COMMENT_VALUE
import os import os
import os.path import os.path
import sys import sys
import logging import logging
import configparser
log = logging.getLogger('service_logger') log = logging.getLogger('service_logger')
...@@ -59,8 +61,12 @@ class AggregatorConfig(object): ...@@ -59,8 +61,12 @@ class AggregatorConfig(object):
:return: A JSON response with the configuration of the aggregator. :return: A JSON response with the configuration of the aggregator.
""" """
aggregator_data = self.request.registry.settings aggregator_config_data = self.request.registry.settings[CONF_OBJECT] # fetch the configuration object
config = {key: aggregator_data.get(key) for key in CONFIG_ATTRIBUTES} if aggregator_config_data is None:
raise HTTPBadRequest("Aggregator has not been configured, yet. Send a PUT request to /aggregator/config with a JSON body of the configuration.")
config = {key: aggregator_config_data[AGGREGATOR_CONFIG_SECTION][key] for key in CONFIG_ATTRIBUTES} # extract a json value containing the config attributes
config['aggregator_report_period'] = int(config['aggregator_report_period'])
return config return config
...@@ -72,27 +78,51 @@ class AggregatorConfig(object): ...@@ -72,27 +78,51 @@ class AggregatorConfig(object):
:raises HTTPBadRequest: if request body is not a valid JSON for the configurator :raises HTTPBadRequest: if request body is not a valid JSON for the configurator
""" """
old_config = {attribute: self.request.registry.settings.get(attribute) for attribute in CONFIG_ATTRIBUTES}
new_config = self.request.body.decode(self.request.charset)
try: try:
new_config = validate_config_content(new_config) new_config = self.request.body.decode(self.request.charset)
new_config = validate_config_content(new_config) # validate the content and receive a json dictionary object
for attribute in CONFIG_ATTRIBUTES: except AssertionError as e:
self.request.registry.settings[attribute] = new_config.get(attribute) raise HTTPBadRequest("Bad request content. Configuration format is incorrect: {0}".format(e.args))
# if configuration is not already malformed, check whether the configuration is updated conf = self.request.registry.settings[CONF_OBJECT]
if not self.request.registry.settings[MALFORMED_FLAG]: if conf is None:
malformed = old_config != new_config and AggregatorController.is_process_running(self.request.registry.settings.get(PROCESS_ATTRIBUTE)) conf = configparser.ConfigParser()
self.request.registry.settings[MALFORMED_FLAG] = malformed conf[AGGREGATOR_CONFIG_SECTION] = {}
if malformed: self.request.registry.settings[CONF_OBJECT] = conf
new_config[MALFORMED_FLAG] = True old_config = {}
new_config[COMMENT_ATTRIBUTE] = COMMENT_VALUE else:
# save the old configuration before updating so that it can be compared to the new one and checked for malformed state
old_config = {attribute: conf[AGGREGATOR_CONFIG_SECTION][attribute] for attribute in CONFIG_ATTRIBUTES}
old_config['aggregator_report_period'] = int(old_config['aggregator_report_period'])
for attribute in CONFIG_ATTRIBUTES:
conf[AGGREGATOR_CONFIG_SECTION][attribute] = str(new_config.get(attribute)) # update the configuration attributes
# if configuration is not already malformed, check whether the configuration is updated (changed in any way), if so (and the aggregator is running), malformed state is detected
if not self.request.registry.settings[MALFORMED_FLAG]:
malformed = old_config != new_config and AggregatorController.is_process_running(self.request.registry.settings.get(PROCESS_ATTRIBUTE))
self.request.registry.settings[MALFORMED_FLAG] = malformed
if malformed:
new_config[MALFORMED_FLAG] = True
new_config[COMMENT_ATTRIBUTE] = COMMENT_VALUE
self._write_conf_file() # save the updated configuration to conf file
return new_config
def _write_conf_file(self):
"""
Writes the configuration settings of the aggregator to a file with path stored at CONF_FILE_ATTRIBUTE
"""
return new_config conf = self.request.registry.settings[CONF_OBJECT]
conf_file_path = self.request.registry.settings[CONF_FILE_ATTRIBUTE]
os.makedirs(os.path.dirname(conf_file_path), exist_ok=True)
except AssertionError: log.info("Saving configuration to file {0}.".format(conf_file_path))
raise HTTPBadRequest("Bad request content - configuration format is incorrect.") with open(conf_file_path, 'w') as configfile:
log.info("Opened configuration file {0}.".format(conf_file_path))
conf.write(configfile)
log.info("Successfully saved configuration to file {0}.".format(conf_file_path))
@view_defaults(route_name='aggregator_controller', renderer='json') @view_defaults(route_name='aggregator_controller', renderer='json')
...@@ -143,16 +173,22 @@ class AggregatorController(object): ...@@ -143,16 +173,22 @@ class AggregatorController(object):
try: try:
content = validate_action_content(content) content = validate_action_content(content)
config = {attribute: self.request.registry.settings.get(attribute) for attribute in CONFIG_ATTRIBUTES} conf = self.request.registry.settings[CONF_OBJECT]
if conf is None:
raise HTTPBadRequest("You must configure the aggregator before controlling it. Send a PUT request to /aggregator/config with a JSON body of the configuration.")
aggregator_config = {attribute: conf[AGGREGATOR_CONFIG_SECTION][attribute] for attribute in CONFIG_ATTRIBUTES}
aggregator_config['aggregator_report_period'] = int(aggregator_config['aggregator_report_period'])
action = content['action'] action = content['action']
aggregator_running = self.is_process_running(self.request.registry.settings.get(PROCESS_ATTRIBUTE)) aggregator_running = self.is_process_running(self.request.registry.settings.get(PROCESS_ATTRIBUTE))
if action == 'start': if action == 'start':
if not aggregator_running: if not aggregator_running:
process = self.start_aggregator(config) process = self.start_aggregator(aggregator_config)
aggregator_running = True aggregator_running = True
self.request.registry.settings[PROCESS_ATTRIBUTE] = process self.request.registry.settings[PROCESS_ATTRIBUTE] = process
self.request.registry.settings[MALFORMED_FLAG] = False
elif action == 'stop': elif action == 'stop':
self.stop_aggregator(self.request.registry.settings.get(PROCESS_ATTRIBUTE)) self.stop_aggregator(self.request.registry.settings.get(PROCESS_ATTRIBUTE))
aggregator_running = False aggregator_running = False
...@@ -160,7 +196,7 @@ class AggregatorController(object): ...@@ -160,7 +196,7 @@ class AggregatorController(object):
self.request.registry.settings[MALFORMED_FLAG] = False self.request.registry.settings[MALFORMED_FLAG] = False
elif action == 'restart': elif action == 'restart':
self.stop_aggregator(self.request.registry.settings.get(PROCESS_ATTRIBUTE)) self.stop_aggregator(self.request.registry.settings.get(PROCESS_ATTRIBUTE))
process = self.start_aggregator(config) process = self.start_aggregator(aggregator_config)
aggregator_running = True aggregator_running = True
self.request.registry.settings[PROCESS_ATTRIBUTE] = process self.request.registry.settings[PROCESS_ATTRIBUTE] = process
self.request.registry.settings[MALFORMED_FLAG] = False self.request.registry.settings[MALFORMED_FLAG] = False
...@@ -245,13 +281,19 @@ class RoundTripTimeQuery(object): ...@@ -245,13 +281,19 @@ class RoundTripTimeQuery(object):
try: try:
params = validate_round_trip_query_params(params) params = validate_round_trip_query_params(params)
config_data = {config_attribute: self.request.registry.settings.get(config_attribute) for config_attribute in CONFIG_ATTRIBUTES}
conf = self.request.registry.settings[CONF_OBJECT]
if conf is None:
raise HTTPBadRequest("You must configure the aggregator before making a round trip time query. Send a PUT request to /aggregator/config with a JSON body of the configuration.")
aggregator_config_data = {config_attribute: conf[AGGREGATOR_CONFIG_SECTION][config_attribute] for config_attribute in CONFIG_ATTRIBUTES}
aggregator_config_data['aggregator_report_period'] = int(aggregator_config_data['aggregator_report_period'])
media_service = params.get(ROUND_TRIP_ATTRIBUTES[0]) media_service = params.get(ROUND_TRIP_ATTRIBUTES[0])
start_timestamp = params.get(ROUND_TRIP_ATTRIBUTES[1]) start_timestamp = params.get(ROUND_TRIP_ATTRIBUTES[1])
end_timestamp = params.get(ROUND_TRIP_ATTRIBUTES[2]) end_timestamp = params.get(ROUND_TRIP_ATTRIBUTES[2])
influx_db_name = config_data.get(CONFIG_ATTRIBUTES[1]) influx_db_name = aggregator_config_data.get(CONFIG_ATTRIBUTES[1])
influx_db_url = config_data.get(CONFIG_ATTRIBUTES[2]) influx_db_url = aggregator_config_data.get(CONFIG_ATTRIBUTES[2])
url_object = urlparse(influx_db_url) url_object = urlparse(influx_db_url)
try: try:
......
src/service/development.ini
+ 3
− 4
View file @ 36e2bbf5
...@@ -14,10 +14,9 @@ pyramid.default_locale_name = en ...@@ -14,10 +14,9 @@ pyramid.default_locale_name = en
pyramid.includes = pyramid_debugtoolbar pyramid_exclog pyramid.includes = pyramid_debugtoolbar pyramid_exclog
exclog.ignore = exclog.ignore =
## Aggregator default configuration ## Configuration file path
aggregator_report_period = 5 configuration_file_path = /etc/flame/clmc/service.conf
aggregator_database_name = CLMCMetrics
aggregator_database_url = http://172.40.231.51:8086
# By default, the toolbar only appears for clients from IP addresses # By default, the toolbar only appears for clients from IP addresses
# '127.0.0.1' and '::1'. # '127.0.0.1' and '::1'.
......
src/service/production.ini
+ 3
− 4
View file @ 36e2bbf5
...@@ -14,10 +14,9 @@ pyramid.default_locale_name = en ...@@ -14,10 +14,9 @@ pyramid.default_locale_name = en
pyramid.includes = pyramid_exclog pyramid.includes = pyramid_exclog
exclog.ignore = exclog.ignore =
## Aggregator default configuration ## Configuration file path
aggregator_report_period = 5 configuration_file_path = /etc/flame/clmc/service.conf
aggregator_database_name = CLMCMetrics
aggregator_database_url = http://172.40.231.51:8086
### ###
# wsgi server configuration # wsgi server configuration
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment