From 36e2bbf5976189ebb9651cf0902cf14aff7a272f Mon Sep 17 00:00:00 2001
From: Nikolay Stanchev <ns17@it-innovation.soton.ac.uk>
Date: Mon, 18 Jun 2018 15:05:46 +0100
Subject: [PATCH] Implements file-based configuration of the CLMC service
---
scripts/clmc-service/install.sh | 4 +
src/service/clmcservice/__init__.py | 10 +-
src/service/clmcservice/tests.py | 188 ++++++++++++++++++---------
src/service/clmcservice/utilities.py | 36 +++++
src/service/clmcservice/views.py | 96 ++++++++++----
src/service/development.ini | 7 +-
src/service/production.ini | 7 +-
7 files changed, 245 insertions(+), 103 deletions(-)
diff --git a/scripts/clmc-service/install.sh b/scripts/clmc-service/install.sh
index 8f1132b..cee52f1 100755
--- a/scripts/clmc-service/install.sh
+++ b/scripts/clmc-service/install.sh
@@ -162,6 +162,10 @@ fi
echo "----> Creating CLMC web service log directory"
mkdir -p /var/log/flame/clmc
+# create directory for CLMC service config
+echo "----> Creating CLMC web service config directory"
+mkdir -p /etc/flame/clmc
+
# Install minioclmc as systemctl service
# -----------------------------------------------------------------------
mkdir -p /opt/flame/clmc
diff --git a/src/service/clmcservice/__init__.py b/src/service/clmcservice/__init__.py
index 3473f9c..a3e6c20 100644
--- a/src/service/clmcservice/__init__.py
+++ b/src/service/clmcservice/__init__.py
@@ -1,3 +1,4 @@
+#!/usr/bin/python3
"""
// © University of Southampton IT Innovation Centre, 2018
//
@@ -22,7 +23,7 @@
"""
from pyramid.config import Configurator
-from clmcservice.utilities import RUNNING_FLAG, MALFORMED_FLAG
+from clmcservice.utilities import validate_conf_file, RUNNING_FLAG, MALFORMED_FLAG, CONF_FILE_ATTRIBUTE, CONF_OBJECT, AGGREGATOR_CONFIG_SECTION
def main(global_config, **settings):
@@ -30,9 +31,10 @@ def main(global_config, **settings):
This function returns a Pyramid WSGI application.
"""
- # a conversion is necessary so that the configuration values of the aggregator are stored with the right type instead of strings
- aggregator_report_period = int(settings.get('aggregator_report_period', 5))
- settings['aggregator_report_period'] = aggregator_report_period
+ # validate and use (if valid) the configuration file
+ conf_file_path = settings[CONF_FILE_ATTRIBUTE]
+ conf = validate_conf_file(conf_file_path) # if None returned here, service is in unconfigured state
+ settings[CONF_OBJECT] = conf
settings[MALFORMED_FLAG] = False
diff --git a/src/service/clmcservice/tests.py b/src/service/clmcservice/tests.py
index d9980cb..df9fccb 100644
--- a/src/service/clmcservice/tests.py
+++ b/src/service/clmcservice/tests.py
@@ -1,3 +1,4 @@
+#!/usr/bin/python3
"""
// © University of Southampton IT Innovation Centre, 2018
//
@@ -21,13 +22,14 @@
// Created for Project : FLAME
"""
-import pytest
from pyramid import testing
from pyramid.httpexceptions import HTTPBadRequest
from time import sleep
-from clmcservice.utilities import CONFIG_ATTRIBUTES, PROCESS_ATTRIBUTE, RUNNING_FLAG, MALFORMED_FLAG, URL_REGEX
+from clmcservice.utilities import CONF_FILE_ATTRIBUTE, CONF_OBJECT, AGGREGATOR_CONFIG_SECTION, CONFIG_ATTRIBUTES, PROCESS_ATTRIBUTE, RUNNING_FLAG, MALFORMED_FLAG, URL_REGEX
+import pytest
import os
import signal
+import configparser
class TestAggregatorAPI(object):
@@ -41,9 +43,10 @@ class TestAggregatorAPI(object):
A fixture to implement setUp/tearDown functionality for all tests by initializing configuration structure for the web service
"""
- self.config = testing.setUp()
- self.config.add_settings({'aggregator_running': False, 'malformed': False, 'aggregator_report_period': 5,
- 'aggregator_database_name': 'CLMCMetrics', 'aggregator_database_url': "http://172.40.231.51:8086"})
+ self.registry = testing.setUp()
+ config = configparser.ConfigParser()
+ config[AGGREGATOR_CONFIG_SECTION] = {'aggregator_report_period': 5, 'aggregator_database_name': 'CLMCMetrics', 'aggregator_database_url': "http://172.40.231.51:8086"}
+ self.registry.add_settings({'configuration_object': config, 'aggregator_running': False, 'malformed': False, 'configuration_file_path': "/etc/flame/clmc/service.conf"})
yield
@@ -56,9 +59,9 @@ class TestAggregatorAPI(object):
from clmcservice.views import AggregatorConfig # nested import so that importing the class view is part of the test itself
- assert self.config.get_settings().get('aggregator_report_period') == 5, "Initial report period is 5 seconds."
- assert self.config.get_settings().get('aggregator_database_name') == 'CLMCMetrics', "Initial database name the aggregator uses is CLMCMetrics."
- assert self.config.get_settings().get('aggregator_database_url') == "http://172.40.231.51:8086", "Initial aggregator url is http://172.40.231.51:8086"
+ assert int(self.registry.get_settings()[CONF_OBJECT][AGGREGATOR_CONFIG_SECTION].get('aggregator_report_period')) == 5, "Initial report period is 5 seconds."
+ assert self.registry.get_settings()[CONF_OBJECT][AGGREGATOR_CONFIG_SECTION].get('aggregator_database_name') == 'CLMCMetrics', "Initial database name the aggregator uses is CLMCMetrics."
+ assert self.registry.get_settings()[CONF_OBJECT][AGGREGATOR_CONFIG_SECTION].get('aggregator_database_url') == "http://172.40.231.51:8086", "Initial aggregator url is http://172.40.231.51:8086"
request = testing.DummyRequest()
response = AggregatorConfig(request).get()
@@ -67,9 +70,9 @@ class TestAggregatorAPI(object):
'aggregator_database_name': 'CLMCMetrics',
'aggregator_database_url': "http://172.40.231.51:8086"}, "Response must be a dictionary representing a JSON object with the correct configuration data of the aggregator."
- assert self.config.get_settings().get('aggregator_report_period') == 5, "A GET request must not modify the aggregator configuration data."
- assert self.config.get_settings().get('aggregator_database_name') == 'CLMCMetrics', "A GET request must not modify the aggregator configuration data."
- assert self.config.get_settings().get('aggregator_database_url') == "http://172.40.231.51:8086", "A GET request must not modify the aggregator configuration data."
+ assert int(self.registry.get_settings()[CONF_OBJECT][AGGREGATOR_CONFIG_SECTION].get('aggregator_report_period')) == 5, "A GET request must not modify the aggregator configuration data."
+ assert self.registry.get_settings()[CONF_OBJECT][AGGREGATOR_CONFIG_SECTION].get('aggregator_database_name') == 'CLMCMetrics', "A GET request must not modify the aggregator configuration data."
+ assert self.registry.get_settings()[CONF_OBJECT][AGGREGATOR_CONFIG_SECTION].get('aggregator_database_url') == "http://172.40.231.51:8086", "A GET request must not modify the aggregator configuration data."
@pytest.mark.parametrize("input_body, output_value", [
('{"aggregator_report_period": 10, "aggregator_database_name": "CLMCMetrics", "aggregator_database_url": "http://171.40.231.51:8086"}',
@@ -101,10 +104,10 @@ class TestAggregatorAPI(object):
from clmcservice.views import AggregatorConfig, AggregatorController # nested import so that importing the class view is part of the test itself
- assert not AggregatorController.is_process_running(self.config.get_settings().get(PROCESS_ATTRIBUTE)), "Initially aggregator is not running."
- assert self.config.get_settings().get('aggregator_report_period') == 5, "Initial report period is 5 seconds."
- assert self.config.get_settings().get('aggregator_database_name') == 'CLMCMetrics', "Initial database name the aggregator uses is CLMCMetrics."
- assert self.config.get_settings().get('aggregator_database_url') == "http://172.40.231.51:8086", "Initial aggregator url is http://172.40.231.51:8086"
+ assert not AggregatorController.is_process_running(self.registry.get_settings().get(PROCESS_ATTRIBUTE)), "Initially aggregator is not running."
+ assert int(self.registry.get_settings()[CONF_OBJECT][AGGREGATOR_CONFIG_SECTION].get('aggregator_report_period')) == 5, "Initial report period is 5 seconds."
+ assert self.registry.get_settings()[CONF_OBJECT][AGGREGATOR_CONFIG_SECTION].get('aggregator_database_name') == 'CLMCMetrics', "Initial database name the aggregator uses is CLMCMetrics."
+ assert self.registry.get_settings()[CONF_OBJECT][AGGREGATOR_CONFIG_SECTION].get('aggregator_database_url') == "http://172.40.231.51:8086", "Initial aggregator url is http://172.40.231.51:8086"
request = testing.DummyRequest()
request.body = input_body.encode(request.charset)
@@ -114,9 +117,19 @@ class TestAggregatorAPI(object):
assert response == output_value, "Response of PUT request must include the new configuration of the aggregator"
for attribute in CONFIG_ATTRIBUTES:
- assert self.config.get_settings().get(attribute) == output_value.get(attribute), "Aggregator settings configuration is not updated."
+ assert self.registry.get_settings()[CONF_OBJECT][AGGREGATOR_CONFIG_SECTION][attribute] == str(output_value[attribute]), "Aggregator settings configuration is not updated."
+
+ assert not AggregatorController.is_process_running(self.registry.get_settings().get(PROCESS_ATTRIBUTE)), "Aggregator running status should not be updated after a configuration update."
+
+ # assert that the conf file is updated
+ updated_conf = configparser.ConfigParser()
+ conf_file = self.registry.get_settings().get(CONF_FILE_ATTRIBUTE)
+ assert updated_conf.read(conf_file) == [conf_file]
+ assert AGGREGATOR_CONFIG_SECTION in updated_conf.sections()
+
+ for attribute in CONFIG_ATTRIBUTES:
+ assert self.registry.get_settings()[CONF_OBJECT][AGGREGATOR_CONFIG_SECTION][attribute] == updated_conf[AGGREGATOR_CONFIG_SECTION][attribute], "Aggregator settings configuration is not updated."
- assert not AggregatorController.is_process_running(self.config.get_settings().get(PROCESS_ATTRIBUTE)), "Aggregator running status should not be updated after a configuration update."
else:
error_raised = False
try:
@@ -133,8 +146,8 @@ class TestAggregatorAPI(object):
from clmcservice.views import AggregatorController # nested import so that importing the class view is part of the test itself
- assert not AggregatorController.is_process_running(self.config.get_settings().get(PROCESS_ATTRIBUTE)), "Initially aggregator is not running."
- assert self.config.get_settings().get(PROCESS_ATTRIBUTE) is None, "Initially no aggregator process is running."
+ assert not AggregatorController.is_process_running(self.registry.get_settings().get(PROCESS_ATTRIBUTE)), "Initially aggregator is not running."
+ assert self.registry.get_settings().get(PROCESS_ATTRIBUTE) is None, "Initially no aggregator process is running."
request = testing.DummyRequest()
input_body = '{"action": "start"}'
@@ -142,8 +155,8 @@ class TestAggregatorAPI(object):
response = AggregatorController(request).put()
assert response == {RUNNING_FLAG: True}, "The aggregator should have been started."
- assert AggregatorController.is_process_running(self.config.get_settings().get(PROCESS_ATTRIBUTE)), "The aggregator should have been started."
- assert self.config.get_settings().get(PROCESS_ATTRIBUTE) is not None, "Aggregator process should have been initialized."
+ assert AggregatorController.is_process_running(self.registry.get_settings().get(PROCESS_ATTRIBUTE)), "The aggregator should have been started."
+ assert self.registry.get_settings().get(PROCESS_ATTRIBUTE) is not None, "Aggregator process should have been initialized."
# kill the started process after the test is over
pid = request.registry.settings[PROCESS_ATTRIBUTE].pid
@@ -156,16 +169,16 @@ class TestAggregatorAPI(object):
from clmcservice.views import AggregatorController # nested import so that importing the class view is part of the test itself
- assert not AggregatorController.is_process_running(self.config.get_settings().get(PROCESS_ATTRIBUTE)), "Initially aggregator is not running."
- assert self.config.get_settings().get(PROCESS_ATTRIBUTE) is None, "Initially no aggregator process is running."
+ assert not AggregatorController.is_process_running(self.registry.get_settings().get(PROCESS_ATTRIBUTE)), "Initially aggregator is not running."
+ assert self.registry.get_settings().get(PROCESS_ATTRIBUTE) is None, "Initially no aggregator process is running."
# send a start request to trigger the aggregator
request = testing.DummyRequest()
input_body = '{"action": "start"}'
request.body = input_body.encode(request.charset)
AggregatorController(request).put()
- assert self.config.get_settings().get(PROCESS_ATTRIBUTE) is not None, "Aggregator process should have been initialized."
- assert AggregatorController.is_process_running(self.config.get_settings().get(PROCESS_ATTRIBUTE)), "Aggregator process should have been initialized."
+ assert self.registry.get_settings().get(PROCESS_ATTRIBUTE) is not None, "Aggregator process should have been initialized."
+ assert AggregatorController.is_process_running(self.registry.get_settings().get(PROCESS_ATTRIBUTE)), "Aggregator process should have been initialized."
# test stopping the aggregator process when it is running
request = testing.DummyRequest()
@@ -174,8 +187,8 @@ class TestAggregatorAPI(object):
response = AggregatorController(request).put()
assert response == {RUNNING_FLAG: False}, "The aggregator should have been stopped."
- assert not AggregatorController.is_process_running(self.config.get_settings().get(PROCESS_ATTRIBUTE)), "The aggregator should have been stopped."
- assert self.config.get_settings().get(PROCESS_ATTRIBUTE) is None, "Aggregator process should have been terminated."
+ assert not AggregatorController.is_process_running(self.registry.get_settings().get(PROCESS_ATTRIBUTE)), "The aggregator should have been stopped."
+ assert self.registry.get_settings().get(PROCESS_ATTRIBUTE) is None, "Aggregator process should have been terminated."
sleep(2) # put a 2 seconds timeout so that the aggregator process can terminate
@@ -186,8 +199,8 @@ class TestAggregatorAPI(object):
response = AggregatorController(request).put()
assert response == {RUNNING_FLAG: False}, "The aggregator should have been stopped."
- assert not AggregatorController.is_process_running(self.config.get_settings().get(PROCESS_ATTRIBUTE)), "The aggregator should have been stopped."
- assert self.config.get_settings().get(PROCESS_ATTRIBUTE) is None, "Aggregator process should have been terminated."
+ assert not AggregatorController.is_process_running(self.registry.get_settings().get(PROCESS_ATTRIBUTE)), "The aggregator should have been stopped."
+ assert self.registry.get_settings().get(PROCESS_ATTRIBUTE) is None, "Aggregator process should have been terminated."
def test_restart(self):
"""
@@ -196,8 +209,8 @@ class TestAggregatorAPI(object):
from clmcservice.views import AggregatorController # nested import so that importing the class view is part of the test itself
- assert not AggregatorController.is_process_running(self.config.get_settings().get(PROCESS_ATTRIBUTE)), "Initially aggregator is not running."
- assert self.config.get_settings().get(PROCESS_ATTRIBUTE) is None, "Initially no aggregator process is running."
+ assert not AggregatorController.is_process_running(self.registry.get_settings().get(PROCESS_ATTRIBUTE)), "Initially aggregator is not running."
+ assert self.registry.get_settings().get(PROCESS_ATTRIBUTE) is None, "Initially no aggregator process is running."
# test restarting the aggregator process when it is stopped
request = testing.DummyRequest()
@@ -206,8 +219,8 @@ class TestAggregatorAPI(object):
response = AggregatorController(request).put()
assert response == {RUNNING_FLAG: True}, "The aggregator should have been restarted."
- assert AggregatorController.is_process_running(self.config.get_settings().get(PROCESS_ATTRIBUTE)), "The aggregator should have been restarted."
- assert self.config.get_settings().get(PROCESS_ATTRIBUTE), "The aggregator process should have been reinitialised."
+ assert AggregatorController.is_process_running(self.registry.get_settings().get(PROCESS_ATTRIBUTE)), "The aggregator should have been restarted."
+ assert self.registry.get_settings().get(PROCESS_ATTRIBUTE), "The aggregator process should have been reinitialised."
# test restarting the aggregator process when it is running
request = testing.DummyRequest()
@@ -216,8 +229,8 @@ class TestAggregatorAPI(object):
response = AggregatorController(request).put()
assert response == {RUNNING_FLAG: True}, "The aggregator should have been restarted."
- assert AggregatorController.is_process_running(self.config.get_settings().get(PROCESS_ATTRIBUTE)), "The aggregator should have been restarted."
- assert self.config.get_settings().get(PROCESS_ATTRIBUTE), "The aggregator process should have been reinitialised."
+ assert AggregatorController.is_process_running(self.registry.get_settings().get(PROCESS_ATTRIBUTE)), "The aggregator should have been restarted."
+ assert self.registry.get_settings().get(PROCESS_ATTRIBUTE), "The aggregator process should have been reinitialised."
# kill the started process after the test is over
pid = request.registry.settings[PROCESS_ATTRIBUTE].pid
@@ -239,8 +252,8 @@ class TestAggregatorAPI(object):
from clmcservice.views import AggregatorController # nested import so that importing the class view is part of the test itself
- assert not AggregatorController.is_process_running(self.config.get_settings().get(PROCESS_ATTRIBUTE)), "Initially aggregator is not running."
- assert self.config.get_settings().get(PROCESS_ATTRIBUTE) is None, "Initially no aggregator process is running."
+ assert not AggregatorController.is_process_running(self.registry.get_settings().get(PROCESS_ATTRIBUTE)), "Initially aggregator is not running."
+ assert self.registry.get_settings().get(PROCESS_ATTRIBUTE) is None, "Initially no aggregator process is running."
# test restarting the aggregator process when it is running
request = testing.DummyRequest()
@@ -262,16 +275,16 @@ class TestAggregatorAPI(object):
from clmcservice.views import AggregatorController # nested import so that importing the class view is part of the test itself
- assert not AggregatorController.is_process_running(self.config.get_settings().get(PROCESS_ATTRIBUTE)), "Initially aggregator is not running."
- assert self.config.get_settings().get(PROCESS_ATTRIBUTE) is None, "Initially no aggregator process is running."
+ assert not AggregatorController.is_process_running(self.registry.get_settings().get(PROCESS_ATTRIBUTE)), "Initially aggregator is not running."
+ assert self.registry.get_settings().get(PROCESS_ATTRIBUTE) is None, "Initially no aggregator process is running."
request = testing.DummyRequest()
response = AggregatorController(request).get()
assert response == {'aggregator_running': False}, "Response must be a dictionary representing a JSON object with the correct status data of the aggregator."
- assert not AggregatorController.is_process_running(self.config.get_settings().get(PROCESS_ATTRIBUTE)), "A GET request must not modify the aggregator status flag."
- assert self.config.get_settings().get(PROCESS_ATTRIBUTE) is None, "A GET request must not start the aggregator process."
+ assert not AggregatorController.is_process_running(self.registry.get_settings().get(PROCESS_ATTRIBUTE)), "A GET request must not modify the aggregator status flag."
+ assert self.registry.get_settings().get(PROCESS_ATTRIBUTE) is None, "A GET request must not start the aggregator process."
# test status with malformed configuration
# start the aggregator
@@ -279,7 +292,7 @@ class TestAggregatorAPI(object):
input_body = '{"action": "start"}'
request.body = input_body.encode(request.charset)
AggregatorController(request).put()
- self.config.get_settings()[MALFORMED_FLAG] = True
+ self.registry.get_settings()[MALFORMED_FLAG] = True
request = testing.DummyRequest()
response = AggregatorController(request).get()
@@ -289,9 +302,9 @@ class TestAggregatorAPI(object):
'comment': 'Aggregator is running in a malformed state - it uses an old version of the configuration. Please, restart it so that the updated configuration is used.'}, \
"Response must be a dictionary representing a JSON object with the correct status data of the aggregator."
- assert AggregatorController.is_process_running(self.config.get_settings().get(PROCESS_ATTRIBUTE)), "A GET request must not modify the aggregator status flag."
- assert self.config.get_settings().get(MALFORMED_FLAG), "A GET request must not modify the aggregator malformed flag."
- assert self.config.get_settings().get(PROCESS_ATTRIBUTE) is not None, "A GET request must not stop the aggregator process."
+ assert AggregatorController.is_process_running(self.registry.get_settings().get(PROCESS_ATTRIBUTE)), "A GET request must not modify the aggregator status flag."
+ assert self.registry.get_settings().get(MALFORMED_FLAG), "A GET request must not modify the aggregator malformed flag."
+ assert self.registry.get_settings().get(PROCESS_ATTRIBUTE) is not None, "A GET request must not stop the aggregator process."
# kill the started process after the test is over
pid = request.registry.settings[PROCESS_ATTRIBUTE].pid
@@ -304,12 +317,12 @@ class TestAggregatorAPI(object):
from clmcservice.views import AggregatorController, AggregatorConfig # nested import so that importing the class view is part of the test itself
- assert not AggregatorController.is_process_running(self.config.get_settings().get(PROCESS_ATTRIBUTE)), "Initially aggregator is not running."
- assert not self.config.get_settings().get(MALFORMED_FLAG), "Initially aggregator is not in a malformed state"
- assert self.config.get_settings().get(PROCESS_ATTRIBUTE) is None, "Initially no aggregator process is running."
- assert self.config.get_settings().get('aggregator_report_period') == 5, "Initial report period is 5 seconds."
- assert self.config.get_settings().get('aggregator_database_name') == 'CLMCMetrics', "Initial database name the aggregator uses is CLMCMetrics."
- assert self.config.get_settings().get('aggregator_database_url') == "http://172.40.231.51:8086", "Initial aggregator url is http://172.40.231.51:8086"
+ assert not AggregatorController.is_process_running(self.registry.get_settings().get(PROCESS_ATTRIBUTE)), "Initially aggregator is not running."
+ assert not self.registry.get_settings().get(MALFORMED_FLAG), "Initially aggregator is not in a malformed state"
+ assert self.registry.get_settings().get(PROCESS_ATTRIBUTE) is None, "Initially no aggregator process is running."
+ assert int(self.registry.get_settings()[CONF_OBJECT][AGGREGATOR_CONFIG_SECTION].get('aggregator_report_period')) == 5, "Initial report period is 5 seconds."
+ assert self.registry.get_settings()[CONF_OBJECT][AGGREGATOR_CONFIG_SECTION].get('aggregator_database_name') == 'CLMCMetrics', "Initial database name the aggregator uses is CLMCMetrics."
+ assert self.registry.get_settings()[CONF_OBJECT][AGGREGATOR_CONFIG_SECTION].get('aggregator_database_url') == "http://172.40.231.51:8086", "Initial aggregator url is http://172.40.231.51:8086"
# start the aggregator with the default configuration
request = testing.DummyRequest()
@@ -328,9 +341,9 @@ class TestAggregatorAPI(object):
response = AggregatorConfig(request).put()
assert response == output_body, "Response of PUT request must include the new configuration of the aggregator"
- assert AggregatorController.is_process_running(self.config.get_settings().get(PROCESS_ATTRIBUTE)), "The aggregator shouldn't be stopped when the configuration is updated."
- assert self.config.get_settings().get(MALFORMED_FLAG), "The malformed flag should be set when the configuration is updated while the process is running."
- assert self.config.get_settings().get(PROCESS_ATTRIBUTE) is not None, "The aggregator shouldn't be stopped when the configuration is updated."
+ assert AggregatorController.is_process_running(self.registry.get_settings().get(PROCESS_ATTRIBUTE)), "The aggregator shouldn't be stopped when the configuration is updated."
+ assert self.registry.get_settings().get(MALFORMED_FLAG), "The malformed flag should be set when the configuration is updated while the process is running."
+ assert self.registry.get_settings().get(PROCESS_ATTRIBUTE) is not None, "The aggregator shouldn't be stopped when the configuration is updated."
# check that the malformed flag has been updated through a GET call
request = testing.DummyRequest()
@@ -346,9 +359,9 @@ class TestAggregatorAPI(object):
request.body = input_body.encode(request.charset)
response = AggregatorController(request).put()
assert response == {RUNNING_FLAG: True}, "The aggregator should have been restarted."
- assert AggregatorController.is_process_running(self.config.get_settings().get(PROCESS_ATTRIBUTE)), "The aggregator should have been restarted."
- assert not self.config.get_settings().get(MALFORMED_FLAG), "The malformed flag should have been reset to False."
- assert self.config.get_settings().get(PROCESS_ATTRIBUTE) is not None, "The aggregator should have been restarted."
+ assert AggregatorController.is_process_running(self.registry.get_settings().get(PROCESS_ATTRIBUTE)), "The aggregator should have been restarted."
+ assert not self.registry.get_settings().get(MALFORMED_FLAG), "The malformed flag should have been reset to False."
+ assert self.registry.get_settings().get(PROCESS_ATTRIBUTE) is not None, "The aggregator should have been restarted."
# update the configuration again while the aggregator is running
config_body = '{"aggregator_report_period": 30, "aggregator_database_name": "CLMCMetrics", "aggregator_database_url": "http://172.50.231.51:8086"}'
@@ -359,20 +372,67 @@ class TestAggregatorAPI(object):
response = AggregatorConfig(request).put()
assert response == output_body, "Response of PUT request must include the new configuration of the aggregator"
- assert AggregatorController.is_process_running(self.config.get_settings().get(PROCESS_ATTRIBUTE)), "The aggregator shouldn't be stopped when the configuration is updated."
- assert self.config.get_settings().get(MALFORMED_FLAG), "The malformed flag should be set when the configuration is updated while the process is running."
- assert self.config.get_settings().get(PROCESS_ATTRIBUTE) is not None, "The aggregator shouldn't be stopped when the configuration is updated."
+ assert AggregatorController.is_process_running(self.registry.get_settings().get(PROCESS_ATTRIBUTE)), "The aggregator shouldn't be stopped when the configuration is updated."
+ assert self.registry.get_settings().get(MALFORMED_FLAG), "The malformed flag should be set when the configuration is updated while the process is running."
+ assert self.registry.get_settings().get(PROCESS_ATTRIBUTE) is not None, "The aggregator shouldn't be stopped when the configuration is updated."
# stop the aggregator - this should also reset the malformed status flag
- # restart the aggregator with the new configuration
request = testing.DummyRequest()
input_body = '{"action": "stop"}'
request.body = input_body.encode(request.charset)
response = AggregatorController(request).put()
assert response == {RUNNING_FLAG: False}, "The aggregator should have been stopped."
- assert not AggregatorController.is_process_running(self.config.get_settings().get(PROCESS_ATTRIBUTE)), "The aggregator should have been stopped."
- assert not self.config.get_settings().get(MALFORMED_FLAG), "The malformed flag should have been reset to False."
- assert self.config.get_settings().get(PROCESS_ATTRIBUTE) is None, "The aggregator should have been stopped."
+ assert not AggregatorController.is_process_running(self.registry.get_settings().get(PROCESS_ATTRIBUTE)), "The aggregator should have been stopped."
+ assert not self.registry.get_settings().get(MALFORMED_FLAG), "The malformed flag should have been reset to False."
+ assert self.registry.get_settings().get(PROCESS_ATTRIBUTE) is None, "The aggregator should have been stopped."
+
+ def test_unconfigured_state(self):
+ """
+ Tests the behaviour of the service when in unconfigured state.
+ """
+
+ from clmcservice.views import AggregatorConfig, AggregatorController
+
+ self.registry.get_settings()[CONF_OBJECT] = None # unconfigured state - conf object is None
+
+ # when doing a GET for the configuration we expect a bad request if the service is in unconfigured state
+ bad_request = False
+ bad_request_msg = None
+ try:
+ request = testing.DummyRequest()
+ AggregatorConfig(request).get()
+ except HTTPBadRequest as err:
+ bad_request = True
+ bad_request_msg = err.message
+
+ assert bad_request
+ assert bad_request_msg == "Aggregator has not been configured, yet. Send a PUT request to /aggregator/config with a JSON body of the configuration."
+
+ # when doing a PUT for the aggregator to start/stop/restart we expect a bad request if the service is in unconfigured state
+ for action in ('start', 'stop', 'restart'):
+ bad_request = False
+ bad_request_msg = None
+ try:
+ request = testing.DummyRequest()
+ request.body = ('{"action": "' + action + '"}').encode(request.charset)
+ AggregatorController(request).put()
+ except HTTPBadRequest as err:
+ bad_request = True
+ bad_request_msg = err.message
+
+ assert bad_request
+ assert bad_request_msg == "You must configure the aggregator before controlling it. Send a PUT request to /aggregator/config with a JSON body of the configuration."
+
+ # configure the aggregator
+ input_body = '{"aggregator_report_period": 10, "aggregator_database_name": "CLMCMetrics", "aggregator_database_url": "http://171.40.231.51:8086"}'
+ output_body = {'aggregator_report_period': 10, 'aggregator_database_name': "CLMCMetrics", 'aggregator_database_url': "http://171.40.231.51:8086"}
+ request = testing.DummyRequest()
+ request.body = input_body.encode(request.charset)
+ response = AggregatorConfig(request).put()
+ assert response == output_body
+
+ request = testing.DummyRequest()
+ assert AggregatorConfig(request).get() == output_body
class TestRegexURL(object):
diff --git a/src/service/clmcservice/utilities.py b/src/service/clmcservice/utilities.py
index 44ccffe..14fd1b4 100644
--- a/src/service/clmcservice/utilities.py
+++ b/src/service/clmcservice/utilities.py
@@ -1,3 +1,4 @@
+#!/usr/bin/python3
"""
// © University of Southampton IT Innovation Centre, 2018
//
@@ -23,7 +24,12 @@
from json import loads
from re import compile, IGNORECASE
+from configparser import ConfigParser
+CONF_FILE_ATTRIBUTE = 'configuration_file_path' # the attribute pointing to the configuration file path
+CONF_OBJECT = 'configuration_object' # the attribute, which stores the service configuration object
+
+AGGREGATOR_CONFIG_SECTION = "AGGREGATOR" # the section in the configuration holding all the configuration attributes declared below
CONFIG_ATTRIBUTES = ('aggregator_report_period', 'aggregator_database_name', 'aggregator_database_url') # all of the configuration attributes - to be used as dictionary keys
RUNNING_FLAG = 'aggregator_running' # Attribute for storing the flag, which shows whether the aggregator is running or not - to be used as a dictionary key
@@ -121,6 +127,36 @@ def validate_round_trip_query_params(params):
return params
+def validate_conf_file(conf_file_path):
+ """
+ Validates the aggregator's configuration file - checks for existence of the file path, whether it can be parsed as a configuration file and
+ whether it contains the required configuration attributes.
+
+ :param conf_file_path: the configuration file path to check
+
+ :return: the parsed configuration if valid, None otherwise
+ """
+
+ global AGGREGATOR_CONFIG_SECTION, CONFIG_ATTRIBUTES
+
+ conf = ConfigParser()
+ result = conf.read(conf_file_path)
+
+ # if result doesn't contain one element, namely the conf_file_path,
+ # then the configuration file cannot be parsed for some reason (doesn't exist, cannot be opened, invalid, etc.)
+ if len(result) == 0:
+ return None
+
+ if AGGREGATOR_CONFIG_SECTION not in conf.sections():
+ return None # the config should include a section called AGGREGATOR
+
+ for key in CONFIG_ATTRIBUTES:
+ if key not in conf[AGGREGATOR_CONFIG_SECTION]:
+ return None # the configuration must include each configuration attribute
+
+ return conf
+
+
def generate_e2e_delay_report(path_id, source_sfr, target_sfr, endpoint, sf_instance, delay_forward, delay_reverse, delay_service, avg_request_size, avg_response_size, avg_bandwidth, time):
"""
Generates a combined averaged measurement about the e2e delay and its contributing parts
diff --git a/src/service/clmcservice/views.py b/src/service/clmcservice/views.py
index b033675..906c1c4 100644
--- a/src/service/clmcservice/views.py
+++ b/src/service/clmcservice/views.py
@@ -1,3 +1,4 @@
+#!/usr/bin/python3
"""
// © University of Southampton IT Innovation Centre, 2018
//
@@ -27,11 +28,12 @@ from influxdb import InfluxDBClient
from urllib.parse import urlparse
from subprocess import Popen
from clmcservice.utilities import validate_config_content, validate_action_content, validate_round_trip_query_params, \
- CONFIG_ATTRIBUTES, ROUND_TRIP_ATTRIBUTES, RUNNING_FLAG, PROCESS_ATTRIBUTE, MALFORMED_FLAG, COMMENT_ATTRIBUTE, COMMENT_VALUE
+ CONF_OBJECT, CONF_FILE_ATTRIBUTE, AGGREGATOR_CONFIG_SECTION, CONFIG_ATTRIBUTES, ROUND_TRIP_ATTRIBUTES, RUNNING_FLAG, PROCESS_ATTRIBUTE, MALFORMED_FLAG, COMMENT_ATTRIBUTE, COMMENT_VALUE
import os
import os.path
import sys
import logging
+import configparser
log = logging.getLogger('service_logger')
@@ -59,8 +61,12 @@ class AggregatorConfig(object):
:return: A JSON response with the configuration of the aggregator.
"""
- aggregator_data = self.request.registry.settings
- config = {key: aggregator_data.get(key) for key in CONFIG_ATTRIBUTES}
+ aggregator_config_data = self.request.registry.settings[CONF_OBJECT] # fetch the configuration object
+ if aggregator_config_data is None:
+ raise HTTPBadRequest("Aggregator has not been configured, yet. Send a PUT request to /aggregator/config with a JSON body of the configuration.")
+
+ config = {key: aggregator_config_data[AGGREGATOR_CONFIG_SECTION][key] for key in CONFIG_ATTRIBUTES} # extract a json value containing the config attributes
+ config['aggregator_report_period'] = int(config['aggregator_report_period'])
return config
@@ -72,27 +78,51 @@ class AggregatorConfig(object):
:raises HTTPBadRequest: if request body is not a valid JSON for the configurator
"""
- old_config = {attribute: self.request.registry.settings.get(attribute) for attribute in CONFIG_ATTRIBUTES}
- new_config = self.request.body.decode(self.request.charset)
-
try:
- new_config = validate_config_content(new_config)
-
- for attribute in CONFIG_ATTRIBUTES:
- self.request.registry.settings[attribute] = new_config.get(attribute)
-
- # if configuration is not already malformed, check whether the configuration is updated
- if not self.request.registry.settings[MALFORMED_FLAG]:
- malformed = old_config != new_config and AggregatorController.is_process_running(self.request.registry.settings.get(PROCESS_ATTRIBUTE))
- self.request.registry.settings[MALFORMED_FLAG] = malformed
- if malformed:
- new_config[MALFORMED_FLAG] = True
- new_config[COMMENT_ATTRIBUTE] = COMMENT_VALUE
+ new_config = self.request.body.decode(self.request.charset)
+ new_config = validate_config_content(new_config) # validate the content and receive a json dictionary object
+ except AssertionError as e:
+ raise HTTPBadRequest("Bad request content. Configuration format is incorrect: {0}".format(e.args))
+
+ conf = self.request.registry.settings[CONF_OBJECT]
+ if conf is None:
+ conf = configparser.ConfigParser()
+ conf[AGGREGATOR_CONFIG_SECTION] = {}
+ self.request.registry.settings[CONF_OBJECT] = conf
+ old_config = {}
+ else:
+ # save the old configuration before updating so that it can be compared to the new one and checked for malformed state
+ old_config = {attribute: conf[AGGREGATOR_CONFIG_SECTION][attribute] for attribute in CONFIG_ATTRIBUTES}
+ old_config['aggregator_report_period'] = int(old_config['aggregator_report_period'])
+
+ for attribute in CONFIG_ATTRIBUTES:
+ conf[AGGREGATOR_CONFIG_SECTION][attribute] = str(new_config.get(attribute)) # update the configuration attributes
+
+ # if configuration is not already malformed, check whether the configuration is updated (changed in any way), if so (and the aggregator is running), malformed state is detected
+ if not self.request.registry.settings[MALFORMED_FLAG]:
+ malformed = old_config != new_config and AggregatorController.is_process_running(self.request.registry.settings.get(PROCESS_ATTRIBUTE))
+ self.request.registry.settings[MALFORMED_FLAG] = malformed
+ if malformed:
+ new_config[MALFORMED_FLAG] = True
+ new_config[COMMENT_ATTRIBUTE] = COMMENT_VALUE
+
+ self._write_conf_file() # save the updated configuration to conf file
+ return new_config
+
+ def _write_conf_file(self):
+ """
+ Writes the configuration settings of the aggregator to a file with path stored at CONF_FILE_ATTRIBUTE
+ """
- return new_config
+ conf = self.request.registry.settings[CONF_OBJECT]
+ conf_file_path = self.request.registry.settings[CONF_FILE_ATTRIBUTE]
+ os.makedirs(os.path.dirname(conf_file_path), exist_ok=True)
- except AssertionError:
- raise HTTPBadRequest("Bad request content - configuration format is incorrect.")
+ log.info("Saving configuration to file {0}.".format(conf_file_path))
+ with open(conf_file_path, 'w') as configfile:
+ log.info("Opened configuration file {0}.".format(conf_file_path))
+ conf.write(configfile)
+ log.info("Successfully saved configuration to file {0}.".format(conf_file_path))
@view_defaults(route_name='aggregator_controller', renderer='json')
@@ -143,16 +173,22 @@ class AggregatorController(object):
try:
content = validate_action_content(content)
- config = {attribute: self.request.registry.settings.get(attribute) for attribute in CONFIG_ATTRIBUTES}
+ conf = self.request.registry.settings[CONF_OBJECT]
+ if conf is None:
+ raise HTTPBadRequest("You must configure the aggregator before controlling it. Send a PUT request to /aggregator/config with a JSON body of the configuration.")
+
+ aggregator_config = {attribute: conf[AGGREGATOR_CONFIG_SECTION][attribute] for attribute in CONFIG_ATTRIBUTES}
+ aggregator_config['aggregator_report_period'] = int(aggregator_config['aggregator_report_period'])
action = content['action']
aggregator_running = self.is_process_running(self.request.registry.settings.get(PROCESS_ATTRIBUTE))
if action == 'start':
if not aggregator_running:
- process = self.start_aggregator(config)
+ process = self.start_aggregator(aggregator_config)
aggregator_running = True
self.request.registry.settings[PROCESS_ATTRIBUTE] = process
+ self.request.registry.settings[MALFORMED_FLAG] = False
elif action == 'stop':
self.stop_aggregator(self.request.registry.settings.get(PROCESS_ATTRIBUTE))
aggregator_running = False
@@ -160,7 +196,7 @@ class AggregatorController(object):
self.request.registry.settings[MALFORMED_FLAG] = False
elif action == 'restart':
self.stop_aggregator(self.request.registry.settings.get(PROCESS_ATTRIBUTE))
- process = self.start_aggregator(config)
+ process = self.start_aggregator(aggregator_config)
aggregator_running = True
self.request.registry.settings[PROCESS_ATTRIBUTE] = process
self.request.registry.settings[MALFORMED_FLAG] = False
@@ -245,13 +281,19 @@ class RoundTripTimeQuery(object):
try:
params = validate_round_trip_query_params(params)
- config_data = {config_attribute: self.request.registry.settings.get(config_attribute) for config_attribute in CONFIG_ATTRIBUTES}
+
+ conf = self.request.registry.settings[CONF_OBJECT]
+ if conf is None:
+ raise HTTPBadRequest("You must configure the aggregator before making a round trip time query. Send a PUT request to /aggregator/config with a JSON body of the configuration.")
+
+ aggregator_config_data = {config_attribute: conf[AGGREGATOR_CONFIG_SECTION][config_attribute] for config_attribute in CONFIG_ATTRIBUTES}
+ aggregator_config_data['aggregator_report_period'] = int(aggregator_config_data['aggregator_report_period'])
media_service = params.get(ROUND_TRIP_ATTRIBUTES[0])
start_timestamp = params.get(ROUND_TRIP_ATTRIBUTES[1])
end_timestamp = params.get(ROUND_TRIP_ATTRIBUTES[2])
- influx_db_name = config_data.get(CONFIG_ATTRIBUTES[1])
- influx_db_url = config_data.get(CONFIG_ATTRIBUTES[2])
+ influx_db_name = aggregator_config_data.get(CONFIG_ATTRIBUTES[1])
+ influx_db_url = aggregator_config_data.get(CONFIG_ATTRIBUTES[2])
url_object = urlparse(influx_db_url)
try:
diff --git a/src/service/development.ini b/src/service/development.ini
index 9345526..390e2a5 100644
--- a/src/service/development.ini
+++ b/src/service/development.ini
@@ -14,10 +14,9 @@ pyramid.default_locale_name = en
pyramid.includes = pyramid_debugtoolbar pyramid_exclog
exclog.ignore =
-## Aggregator default configuration
-aggregator_report_period = 5
-aggregator_database_name = CLMCMetrics
-aggregator_database_url = http://172.40.231.51:8086
+## Configuration file path
+configuration_file_path = /etc/flame/clmc/service.conf
+
# By default, the toolbar only appears for clients from IP addresses
# '127.0.0.1' and '::1'.
diff --git a/src/service/production.ini b/src/service/production.ini
index 2e1cfcf..3e8876b 100644
--- a/src/service/production.ini
+++ b/src/service/production.ini
@@ -14,10 +14,9 @@ pyramid.default_locale_name = en
pyramid.includes = pyramid_exclog
exclog.ignore =
-## Aggregator default configuration
-aggregator_report_period = 5
-aggregator_database_name = CLMCMetrics
-aggregator_database_url = http://172.40.231.51:8086
+## Configuration file path
+configuration_file_path = /etc/flame/clmc/service.conf
+
###
# wsgi server configuration
--
GitLab