Implements file-based configuration of the CLMC service

scripts/clmc-service/install.sh

@@ -162,6 +162,10 @@ fi
 echo "----> Creating CLMC web service log directory"
 mkdir -p /var/log/flame/clmc
 
+# create directory for CLMC service config
+echo "----> Creating CLMC web service config directory"
+mkdir -p /etc/flame/clmc
+
 # Install minioclmc as systemctl service
 # -----------------------------------------------------------------------
 mkdir -p /opt/flame/clmc

src/service/clmcservice/__init__.py

+#!/usr/bin/python3
 """
 // © University of Southampton IT Innovation Centre, 2018
 //
@@ -22,7 +23,7 @@
 """
 
 from pyramid.config import Configurator
-from clmcservice.utilities import RUNNING_FLAG, MALFORMED_FLAG
+from clmcservice.utilities import validate_conf_file, RUNNING_FLAG, MALFORMED_FLAG, CONF_FILE_ATTRIBUTE, CONF_OBJECT, AGGREGATOR_CONFIG_SECTION
 
 
 def main(global_config, **settings):
@@ -30,9 +31,10 @@ def main(global_config, **settings):
     This function returns a Pyramid WSGI application.
     """
 
-    # a conversion is necessary so that the configuration values of the aggregator are stored with the right type instead of strings
-    aggregator_report_period = int(settings.get('aggregator_report_period', 5))
-    settings['aggregator_report_period'] = aggregator_report_period
+    # validate and use (if valid) the configuration file
+    conf_file_path = settings[CONF_FILE_ATTRIBUTE]
+    conf = validate_conf_file(conf_file_path)  # if None returned here, service is in unconfigured state
+    settings[CONF_OBJECT] = conf
 
     settings[MALFORMED_FLAG] = False
 

src/service/clmcservice/utilities.py

+#!/usr/bin/python3
 """
 // © University of Southampton IT Innovation Centre, 2018
 //
@@ -23,7 +24,12 @@
 
 from json import loads
 from re import compile, IGNORECASE
+from configparser import ConfigParser
 
+CONF_FILE_ATTRIBUTE = 'configuration_file_path'  # the attribute pointing to the configuration file path
+CONF_OBJECT = 'configuration_object'  # the attribute, which stores the service configuration object
+
+AGGREGATOR_CONFIG_SECTION = "AGGREGATOR"  # the section in the configuration holding all the configuration attributes declared below
 CONFIG_ATTRIBUTES = ('aggregator_report_period', 'aggregator_database_name', 'aggregator_database_url')  # all of the configuration attributes - to be used as dictionary keys
 
 RUNNING_FLAG = 'aggregator_running'  # Attribute for storing the flag, which shows whether the aggregator is running or not - to be used as a dictionary key
@@ -121,6 +127,36 @@ def validate_round_trip_query_params(params):
     return params
 
 
+def validate_conf_file(conf_file_path):
+    """
+    Validates the aggregator's configuration file - checks for existence of the file path, whether it can be parsed as a configuration file and
+    whether it contains the required configuration attributes.
+
+    :param conf_file_path: the configuration file path to check
+
+    :return: the parsed configuration if valid, None otherwise
+    """
+
+    global AGGREGATOR_CONFIG_SECTION, CONFIG_ATTRIBUTES
+
+    conf = ConfigParser()
+    result = conf.read(conf_file_path)
+
+    # if result doesn't contain one element, namely the conf_file_path,
+    # then the configuration file cannot be parsed for some reason (doesn't exist, cannot be opened, invalid, etc.)
+    if len(result) == 0:
+        return None
+
+    if AGGREGATOR_CONFIG_SECTION not in conf.sections():
+        return None  # the config should include a section called AGGREGATOR
+
+    for key in CONFIG_ATTRIBUTES:
+        if key not in conf[AGGREGATOR_CONFIG_SECTION]:
+            return None  # the configuration must include each configuration attribute
+
+    return conf
+
+
 def generate_e2e_delay_report(path_id, source_sfr, target_sfr, endpoint, sf_instance, delay_forward, delay_reverse, delay_service, avg_request_size, avg_response_size, avg_bandwidth, time):
     """
     Generates a combined averaged measurement about the e2e delay and its contributing parts

src/service/clmcservice/views.py

+#!/usr/bin/python3
 """
 // © University of Southampton IT Innovation Centre, 2018
 //
@@ -27,11 +28,12 @@ from influxdb import InfluxDBClient
 from urllib.parse import urlparse
 from subprocess import Popen
 from clmcservice.utilities import validate_config_content, validate_action_content, validate_round_trip_query_params, \
-    CONFIG_ATTRIBUTES, ROUND_TRIP_ATTRIBUTES, RUNNING_FLAG, PROCESS_ATTRIBUTE, MALFORMED_FLAG, COMMENT_ATTRIBUTE, COMMENT_VALUE
+    CONF_OBJECT, CONF_FILE_ATTRIBUTE, AGGREGATOR_CONFIG_SECTION, CONFIG_ATTRIBUTES, ROUND_TRIP_ATTRIBUTES, RUNNING_FLAG, PROCESS_ATTRIBUTE, MALFORMED_FLAG, COMMENT_ATTRIBUTE, COMMENT_VALUE
 import os
 import os.path
 import sys
 import logging
+import configparser
 
 
 log = logging.getLogger('service_logger')
@@ -59,8 +61,12 @@ class AggregatorConfig(object):
         :return: A JSON response with the configuration of the aggregator.
         """
 
-        aggregator_data = self.request.registry.settings
-        config = {key: aggregator_data.get(key) for key in CONFIG_ATTRIBUTES}
+        aggregator_config_data = self.request.registry.settings[CONF_OBJECT]  # fetch the configuration object
+        if aggregator_config_data is None:
+            raise HTTPBadRequest("Aggregator has not been configured, yet. Send a PUT request to /aggregator/config with a JSON body of the configuration.")
+
+        config = {key: aggregator_config_data[AGGREGATOR_CONFIG_SECTION][key] for key in CONFIG_ATTRIBUTES}  # extract a json value containing the config attributes
+        config['aggregator_report_period'] = int(config['aggregator_report_period'])
 
         return config
 
@@ -72,27 +78,51 @@ class AggregatorConfig(object):
         :raises HTTPBadRequest: if request body is not a valid JSON for the configurator
         """
 
-        old_config = {attribute: self.request.registry.settings.get(attribute) for attribute in CONFIG_ATTRIBUTES}
-        new_config = self.request.body.decode(self.request.charset)
-
         try:
-            new_config = validate_config_content(new_config)
-
-            for attribute in CONFIG_ATTRIBUTES:
-                self.request.registry.settings[attribute] = new_config.get(attribute)
-
-            # if configuration is not already malformed, check whether the configuration is updated
-            if not self.request.registry.settings[MALFORMED_FLAG]:
-                malformed = old_config != new_config and AggregatorController.is_process_running(self.request.registry.settings.get(PROCESS_ATTRIBUTE))
-                self.request.registry.settings[MALFORMED_FLAG] = malformed
-                if malformed:
-                    new_config[MALFORMED_FLAG] = True
-                    new_config[COMMENT_ATTRIBUTE] = COMMENT_VALUE
+            new_config = self.request.body.decode(self.request.charset)
+            new_config = validate_config_content(new_config)  # validate the content and receive a json dictionary object
+        except AssertionError as e:
+            raise HTTPBadRequest("Bad request content. Configuration format is incorrect: {0}".format(e.args))
+
+        conf = self.request.registry.settings[CONF_OBJECT]
+        if conf is None:
+            conf = configparser.ConfigParser()
+            conf[AGGREGATOR_CONFIG_SECTION] = {}
+            self.request.registry.settings[CONF_OBJECT] = conf
+            old_config = {}
+        else:
+            # save the old configuration before updating so that it can be compared to the new one and checked for malformed state
+            old_config = {attribute: conf[AGGREGATOR_CONFIG_SECTION][attribute] for attribute in CONFIG_ATTRIBUTES}
+            old_config['aggregator_report_period'] = int(old_config['aggregator_report_period'])
+
+        for attribute in CONFIG_ATTRIBUTES:
+            conf[AGGREGATOR_CONFIG_SECTION][attribute] = str(new_config.get(attribute))  # update the configuration attributes
+
+        # if configuration is not already malformed, check whether the configuration is updated (changed in any way), if so (and the aggregator is running), malformed state is detected
+        if not self.request.registry.settings[MALFORMED_FLAG]:
+            malformed = old_config != new_config and AggregatorController.is_process_running(self.request.registry.settings.get(PROCESS_ATTRIBUTE))
+            self.request.registry.settings[MALFORMED_FLAG] = malformed
+            if malformed:
+                new_config[MALFORMED_FLAG] = True
+                new_config[COMMENT_ATTRIBUTE] = COMMENT_VALUE
+
+        self._write_conf_file()  # save the updated configuration to conf file
+        return new_config
+
+    def _write_conf_file(self):
+        """
+        Writes the configuration settings of the aggregator to a file with path stored at CONF_FILE_ATTRIBUTE
+        """
 
-            return new_config
+        conf = self.request.registry.settings[CONF_OBJECT]
+        conf_file_path = self.request.registry.settings[CONF_FILE_ATTRIBUTE]
+        os.makedirs(os.path.dirname(conf_file_path), exist_ok=True)
 
-        except AssertionError:
-            raise HTTPBadRequest("Bad request content - configuration format is incorrect.")
+        log.info("Saving configuration to file {0}.".format(conf_file_path))
+        with open(conf_file_path, 'w') as configfile:
+            log.info("Opened configuration file {0}.".format(conf_file_path))
+            conf.write(configfile)
+        log.info("Successfully saved configuration to file {0}.".format(conf_file_path))
 
 
 @view_defaults(route_name='aggregator_controller', renderer='json')
@@ -143,16 +173,22 @@ class AggregatorController(object):
         try:
             content = validate_action_content(content)
 
-            config = {attribute: self.request.registry.settings.get(attribute) for attribute in CONFIG_ATTRIBUTES}
+            conf = self.request.registry.settings[CONF_OBJECT]
+            if conf is None:
+                raise HTTPBadRequest("You must configure the aggregator before controlling it. Send a PUT request to /aggregator/config with a JSON body of the configuration.")
+
+            aggregator_config = {attribute: conf[AGGREGATOR_CONFIG_SECTION][attribute] for attribute in CONFIG_ATTRIBUTES}
+            aggregator_config['aggregator_report_period'] = int(aggregator_config['aggregator_report_period'])
 
             action = content['action']
 
             aggregator_running = self.is_process_running(self.request.registry.settings.get(PROCESS_ATTRIBUTE))
             if action == 'start':
                 if not aggregator_running:
-                    process = self.start_aggregator(config)
+                    process = self.start_aggregator(aggregator_config)
                     aggregator_running = True
                     self.request.registry.settings[PROCESS_ATTRIBUTE] = process
+                    self.request.registry.settings[MALFORMED_FLAG] = False
             elif action == 'stop':
                 self.stop_aggregator(self.request.registry.settings.get(PROCESS_ATTRIBUTE))
                 aggregator_running = False
@@ -160,7 +196,7 @@ class AggregatorController(object):
                 self.request.registry.settings[MALFORMED_FLAG] = False
             elif action == 'restart':
                 self.stop_aggregator(self.request.registry.settings.get(PROCESS_ATTRIBUTE))
-                process = self.start_aggregator(config)
+                process = self.start_aggregator(aggregator_config)
                 aggregator_running = True
                 self.request.registry.settings[PROCESS_ATTRIBUTE] = process
                 self.request.registry.settings[MALFORMED_FLAG] = False
@@ -245,13 +281,19 @@ class RoundTripTimeQuery(object):
 
         try:
             params = validate_round_trip_query_params(params)
-            config_data = {config_attribute: self.request.registry.settings.get(config_attribute) for config_attribute in CONFIG_ATTRIBUTES}
+
+            conf = self.request.registry.settings[CONF_OBJECT]
+            if conf is None:
+                raise HTTPBadRequest("You must configure the aggregator before making a round trip time query. Send a PUT request to /aggregator/config with a JSON body of the configuration.")
+
+            aggregator_config_data = {config_attribute: conf[AGGREGATOR_CONFIG_SECTION][config_attribute] for config_attribute in CONFIG_ATTRIBUTES}
+            aggregator_config_data['aggregator_report_period'] = int(aggregator_config_data['aggregator_report_period'])
 
             media_service = params.get(ROUND_TRIP_ATTRIBUTES[0])
             start_timestamp = params.get(ROUND_TRIP_ATTRIBUTES[1])
             end_timestamp = params.get(ROUND_TRIP_ATTRIBUTES[2])
-            influx_db_name = config_data.get(CONFIG_ATTRIBUTES[1])
-            influx_db_url = config_data.get(CONFIG_ATTRIBUTES[2])
+            influx_db_name = aggregator_config_data.get(CONFIG_ATTRIBUTES[1])
+            influx_db_url = aggregator_config_data.get(CONFIG_ATTRIBUTES[2])
 
             url_object = urlparse(influx_db_url)
             try:

src/service/development.ini

@@ -14,10 +14,9 @@ pyramid.default_locale_name = en
 pyramid.includes = pyramid_debugtoolbar pyramid_exclog
 exclog.ignore =
 
-## Aggregator default configuration
-aggregator_report_period = 5
-aggregator_database_name = CLMCMetrics
-aggregator_database_url = http://172.40.231.51:8086
+## Configuration file path
+configuration_file_path = /etc/flame/clmc/service.conf
+
 
 # By default, the toolbar only appears for clients from IP addresses
 # '127.0.0.1' and '::1'.

src/service/production.ini

@@ -14,10 +14,9 @@ pyramid.default_locale_name = en
 pyramid.includes = pyramid_exclog
 exclog.ignore =
 
-## Aggregator default configuration
-aggregator_report_period = 5
-aggregator_database_name = CLMCMetrics
-aggregator_database_url = http://172.40.231.51:8086
+## Configuration file path
+configuration_file_path = /etc/flame/clmc/service.conf
+
 
 ###
 # wsgi server configuration