Implements integration test for CLMC alerts

src/test/clmctest/alerts/alert_handler_server.py

+#!/usr/bin/python3
+"""
+## © University of Southampton IT Innovation Centre, 2018
+##
+## Copyright in this software belongs to University of Southampton
+## IT Innovation Centre of Gamma House, Enterprise Road,
+## Chilworth Science Park, Southampton, SO16 7NS, UK.
+##
+## This software may not be used, sold, licensed, transferred, copied
+## or reproduced in whole or in part in any manner or form or in or
+## on any media by any person other than in accordance with the terms
+## of the Licence Agreement supplied with the software, or otherwise
+## without the prior written consent of the copyright owners.
+##
+## This software is distributed WITHOUT ANY WARRANTY, without even the
+## implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+## PURPOSE, except where stated in the Licence Agreement supplied with
+## the software.
+##
+##      Created By :            Nikolay Stanchev
+##      Created Date :          22-08-2018
+##      Created for Project :   FLAME
+"""
+
+
+from http.server import HTTPServer, BaseHTTPRequestHandler
+from json import loads, dump
+from os.path import join
+
+
+LOG_TEST_FOLDER_PATH = "/var/log/flame/clmc/alerts"
+
+
+class CustomHTTPHandler(BaseHTTPRequestHandler):
+    """
+    An http handler used in the integration test of CLMC alerts.
+    """
+
+    def _set_headers(self):
+        """
+        Sets up headers used to send back a response to a received request
+        """
+
+        self.send_response(200)
+        self.send_header('Content-type', 'application/json')
+        self.end_headers()
+
+    def do_POST(self):
+        """
+        This method handles any POST requests made to the server - used to handle the incoming POST requests from Kapacitor.
+        """
+
+        global LOG_TEST_FOLDER_PATH
+
+        # read post data and message ID - the topic from Kapacitor
+        content_length = int(self.headers['Content-Length'])
+        post_data = self.rfile.read(content_length)
+        post_data = post_data.decode(self.headers.get('Accept-Charset', "utf-8"))
+        post_data = loads(post_data)  # load to json
+        msg_id = post_data["id"]
+
+        # write data to log file named after the message ID
+        with open(join(LOG_TEST_FOLDER_PATH, "alert-{0}.log".format(msg_id)), "w+") as fh:
+            dump(fp=fh, obj=post_data)
+
+        # send back a response (needed to mimic the behaviour of an actual http server)
+        self._set_headers()
+        self.wfile.write(b"{\"msg\": \"accepted\"}")
+
+
+def run(server_class=HTTPServer, handler_class=BaseHTTPRequestHandler):
+    """
+    Starts the server on port 9999
+
+    :param server_class: defaults to HTTPServer (standard lib)
+    :param handler_class: defaults to the Base HTTP request handler (standard lib)
+    """
+
+    server_address = ('', 9999)
+    httpd = server_class(server_address, handler_class)
+    httpd.serve_forever()
+
+
+if __name__ == "__main__":
+    run(handler_class=CustomHTTPHandler)  # run server with the custom http handler

src/test/clmctest/alerts/alerts_test_config.yaml

+tosca_definitions_version: tosca_simple_profile_for_nfv_1_0_0
+
+description: TOSCA Alerts Configuration document
+
+imports:
+- flame_clmc_alerts_definitions.yaml
+
+metadata:
+  sfc: CLMCMetrics
+  sfci: MS_I1
+
+topology_template:
+
+  policies:
+    - scale_nginx_policy:
+        type: eu.ict-flame.policies.StateChange
+        triggers:
+          high_requests:
+            description: |
+              This event triggers when the number of requests for a given service function
+              exceeds a given threshold.
+            event_type: threshold
+            metric: nginx.requests
+            condition:
+              threshold: 5
+              granularity: 5
+              aggregation_method: mean
+              resource_type:
+                location: DC1
+              comparison_operator: gte
+            action:
+              implementation:
+                - http://172.40.231.200:9999/
+          high_cpu_usage:
+            description: This event triggers when the cpu system usage is too high.
+            event_type: threshold
+            metric: cpu.usage_system
+            condition:
+              threshold: 0.2
+              granularity: 10
+              aggregation_method: mean
+              resource_type:
+                location: DC1
+                sf: nginx
+              comparison_operator: gte
+            action:
+              implementation:
+                - http://172.40.231.200:9999/
+          increase_in_active_requests:
+            description: This event triggers when the cpu system usage is too high.
+            event_type: relative
+            metric: nginx.accepts
+            condition:
+              threshold: 5
+              granularity: 10
+              resource_type:
+                location: DC1
+                sf: nginx
+              comparison_operator: gte
+            action:
+              implementation:
+                - http://172.40.231.200:9999/
+    - deadman_policies:
+        type: eu.ict-flame.policies.StateChange
+        triggers:
+          no_measurements:
+            description: |
+              This event triggers when RTT measurements are missing for more than 12 seconds.
+            event_type: deadman
+            metric: clcm.rtt
+            condition:
+              threshold: 0
+              granularity: 5
+              resource_type:
+                sf: nginx
+                host: DC1
+                location: DC1
+            action:
+              implementation:
+                - http://172.40.231.200:9999/
\ No newline at end of file

src/test/clmctest/alerts/conftest.py

+#!/usr/bin/python3
+"""
+## © University of Southampton IT Innovation Centre, 2018
+##
+## Copyright in this software belongs to University of Southampton
+## IT Innovation Centre of Gamma House, Enterprise Road,
+## Chilworth Science Park, Southampton, SO16 7NS, UK.
+##
+## This software may not be used, sold, licensed, transferred, copied
+## or reproduced in whole or in part in any manner or form or in or
+## on any media by any person other than in accordance with the terms
+## of the Licence Agreement supplied with the software, or otherwise
+## without the prior written consent of the copyright owners.
+##
+## This software is distributed WITHOUT ANY WARRANTY, without even the
+## implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+## PURPOSE, except where stated in the Licence Agreement supplied with
+## the software.
+##
+##      Created By :            Nikolay Stanchev
+##      Created Date :          22-08-2018
+##      Created for Project :   FLAME
+"""
+
+from time import sleep
+from pytest import fixture
+from subprocess import Popen, DEVNULL
+from os.path import join, dirname, exists
+from os import kill, makedirs
+from shutil import rmtree
+from signal import SIGKILL
+from json import load
+from pkg_resources import resource_filename
+from clmctest.alerts.alert_handler_server import LOG_TEST_FOLDER_PATH
+
+
+@fixture(scope="module")
+def rspec_config():
+    """
+    Reads the service configuration deployed for the integration tests.
+
+    :return: the python object representing the read JSON file
+    """
+
+    rspec = resource_filename('clmctest', 'rspec.json')
+    print("\nrspec file: {0}".format(rspec))
+
+    with open(rspec, 'r') as stream:
+        data_loaded = load(stream)
+    return data_loaded
+
+
+@fixture(autouse=True, scope="module")
+def set_up_tear_down_fixture():
+    """
+    Set up/tear down fixture for the alerts integration test.
+    """
+
+    if exists(LOG_TEST_FOLDER_PATH):
+        rmtree(LOG_TEST_FOLDER_PATH)  # clean out the log directory
+    makedirs(LOG_TEST_FOLDER_PATH)  # create the log directory
+
+    print("\nStarting alert handler HTTP server...")
+    http_server_file = join(dirname(__file__), "alert_handler_server.py")
+    p = Popen(["python3", http_server_file], stdout=DEVNULL, stderr=DEVNULL)
+    process_id = p.pid
+    sleep(1)
+    print("Server started with PID {0}".format(process_id))
+
+    yield
+
+    print("\nKilling process with PID {0}".format(process_id))
+    kill(process_id, SIGKILL)
+    if exists(LOG_TEST_FOLDER_PATH):
+        rmtree(LOG_TEST_FOLDER_PATH)

src/test/clmctest/alerts/test_alerts.py

+#!/usr/bin/python3
+"""
+## © University of Southampton IT Innovation Centre, 2018
+##
+## Copyright in this software belongs to University of Southampton
+## IT Innovation Centre of Gamma House, Enterprise Road,
+## Chilworth Science Park, Southampton, SO16 7NS, UK.
+##
+## This software may not be used, sold, licensed, transferred, copied
+## or reproduced in whole or in part in any manner or form or in or
+## on any media by any person other than in accordance with the terms
+## of the Licence Agreement supplied with the software, or otherwise
+## without the prior written consent of the copyright owners.
+##
+## This software is distributed WITHOUT ANY WARRANTY, without even the
+## implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+## PURPOSE, except where stated in the Licence Agreement supplied with
+## the software.
+##
+##      Created By :            Nikolay Stanchev
+##      Created Date :          22-08-2018
+##      Created for Project :   FLAME
+"""
+
+from time import sleep
+from requests import post, get
+from os import listdir
+from os.path import join, dirname
+from clmctest.alerts.alert_handler_server import LOG_TEST_FOLDER_PATH
+
+
+CLMC_SERVICE_PORT = 9080
+NGINX_PORT = 80
+
+
+class TestAlerts(object):
+
+    def test_alert_triggers(self, rspec_config):
+        """
+        Test is implemented using the following steps:
+            * Send clmc service a TOSCA alert spec. file
+            * Wait 15 seconds for Kapacitor to configure and start executing the defined tasks
+            * Send some test requests to nginx to increase the load
+            * Wait 20 seconds for alerts to be triggered
+            * Check that 4 log files have been created - one for each alert defined in the alert spec.
+
+        :param rspec_config: fixture from conftest.py
+        """
+
+        global CLMC_SERVICE_PORT, NGINX_PORT
+
+        clmc_service_host, nginx_host = None, None
+        for host in rspec_config:
+            if host["name"] == "clmc-service":
+                clmc_service_host = host["ip_address"]
+            elif host["name"] == "nginx":
+                nginx_host = host["ip_address"]
+
+            if clmc_service_host is not None and nginx_host is not None:
+                break
+
+        print("Sending alerts specification to clmc service...")
+        alerts_spec = join(dirname(__file__), "alerts_test_config.yaml")
+
+        with open(alerts_spec, 'rb') as fh:
+            files = {'alert-spec': fh}
+            response = post("http://{0}:{1}/alerts".format(clmc_service_host, CLMC_SERVICE_PORT), files=files)
+            assert response.status_code == 200
+        print("Alert spec sent successfully")
+
+        print("Wait 10 seconds for Kapacitor stream/batch tasks to start working...")
+        sleep(10)
+
+        print("Sending test requests to nginx...")
+        for i in range(20):
+            response = get("http://{0}:{1}/".format(nginx_host, NGINX_PORT))
+            assert response.status_code == 200
+
+        print("Wait 20 seconds for Kapacitor to trigger alerts...")
+        sleep(20)
+
+        assert len(listdir(LOG_TEST_FOLDER_PATH)) == 4, "4 log files must have been created - one for each alert defined in the specification."