Updates graph_monitor API endpoint to also save the request identifier and the...

Updates graph_monitor API endpoint to also save the request identifier and the pipeline process identifier

scripts/clmc-service/graph-pipeline.sh

@@ -37,11 +37,6 @@ read query_period db_name results_measurement <<< ${fields}
 JSON_CONFIG=$(echo ${JSON_CONFIG} | jq 'del(.query_period, .results_measurement_name)')
 
 
-# TODO this shouldn't be in the script maybe move it to the clmc isntall script
-echo "Building network subgraph..."
-response=$(curl -s -X POST http://${CLMC_IP}/clmc-service/graph/network)
-echo ${response}
-
 while true
 do
     echo "Building temporal graph..."

src/service/clmcservice/graphapi/tests.py

@@ -23,11 +23,12 @@
 """
 
 from json import dumps
-import pytest
 from unittest.mock import patch, Mock, PropertyMock
+import pytest
 from pyramid import testing
-from clmcservice.graphapi.views import GraphAPI
 from pyramid.httpexceptions import HTTPBadRequest, HTTPNotFound, HTTPInternalServerError
+from clmcservice.graphapi.views import GraphAPI
+from clmcservice.models import MonitoringProcess
 
 
 class TestGraphAPI(object):
@@ -533,6 +534,10 @@ class TestGraphAPI(object):
         pid_property_mock.assert_called_once_with()  # assert that the process ID attribute was called and saved
         returncode_property_mock.assert_called_once_with()  # assert that the process return code attribute was called to check if the process has started successfully
 
+        # check that the process ID was saved
+        assert MonitoringProcess.exists(uuid_mock.return_value), "Request identifier was not saved during the request processing"
+        assert MonitoringProcess.get(uuid_mock.return_value) == pid_property_mock.return_value, "Incorrect PID was saved during the request processing"
+
         # check erroneous behaviour
         returncode_property_mock.return_value = -1
         request = testing.DummyRequest()
@@ -548,7 +553,6 @@ class TestGraphAPI(object):
         pid_property_mock.assert_called_with()  # assert that the process ID attribute was called and saved
         returncode_property_mock.assert_called_with()  # assert that the process return code attribute was called to check if the process has started successfully
 
-
     @staticmethod
     def check_exist_relationship(relationships_tuple, graph, uuid):
         """

src/service/clmcservice/graphapi/views.py

@@ -25,6 +25,7 @@
 
 from clmcservice.graphapi.utilities import validate_build_request_body, validate_monitor_request_body, RTT_CYPHER_QUERY_TEMPLATE, \
     build_network_graph, delete_network_graph, build_temporal_subgraph, delete_temporal_subgraph, validate_graph_rtt_params, find_node_with_possible_types
+from clmcservice.models import MonitoringProcess
 from influxdb import InfluxDBClient
 from py2neo import Graph
 from pyramid.httpexceptions import HTTPBadRequest, HTTPNotFound, HTTPServiceUnavailable, HTTPNotImplemented, HTTPInternalServerError
@@ -372,6 +373,9 @@ class GraphAPI(object):
 
         if process_return_code is None:  # process has started running
             log.info("Started a graph pipeline process for SFC {0} with PID {1}".format(sfc, process_pid))
+
+            MonitoringProcess.add({"request_id": request_uuid, "process_id": process_pid})
+
             return {"database": database_name, "uuid": request_uuid}
         else:  # a valid returned code was returned, hence the process has terminated one way or another - we do not expect this since the pipeline script must be continuously running
             log.warning("Graph pipeline process for SFC {0} with PID {1} has finished executing unexpectedly with return code {2}".format(sfc, process_pid, process_return_code))

src/service/clmcservice/models/__init__.py

 from .meta import DBSession
+
+from .graphapi_models import MonitoringProcess
+# NOTE: all ORM models defined in this package must be imported here (in the __init__.py file) - Pyramid and SQLAlchemy specific approach
 from .whoami_models import ServiceFunctionEndpoint

src/service/clmcservice/models/graphapi_models.py

+#!/usr/bin/python3
+"""
+// © University of Southampton IT Innovation Centre, 2018
+//
+// Copyright in this software belongs to University of Southampton
+// IT Innovation Centre of Gamma House, Enterprise Road,
+// Chilworth Science Park, Southampton, SO16 7NS, UK.
+//
+// This software may not be used, sold, licensed, transferred, copied
+// or reproduced in whole or in part in any manner or form or in or
+// on any media by any person other than in accordance with the terms
+// of the Licence Agreement supplied with the software, or otherwise
+// without the prior written consent of the copyright owners.
+//
+// This software is distributed WITHOUT ANY WARRANTY, without even the
+// implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE, except where stated in the Licence Agreement supplied with
+// the software.
+//
+//      Created By :            Nikolay Stanchev
+//      Created Date :          02-07-2018
+//      Created for Project :   FLAME
+"""
+
+
+class _MonitoringProcess:
+    """
+    Container-like class used to store key value pairs between graph pipelines request identifiers and process identifiers.
+    The class is declared as "pseudo-private" and is ought to be initialised only in this module and exposed as a singleton.
+    """
+
+    def __init__(self):
+
+        self.__pipelines = {}
+
+    def add(self, instance):
+        """
+        Adds a new key-value pair to the container representing a request ID mapped to a PID.
+
+        :param instance: JSON representation of key value pair - e.g. {"request_id": <reequest uuid>, "process_id": <process ID>}
+        """
+
+        self.__pipelines[instance["request_id"]] = instance["process_id"]
+
+    def get(self, request_id):
+        """
+        Gets the data associated with a request identifier - currently the data is just the process identifier.
+
+        :param request_id: the request identifier to check
+
+        :return: the PID for this request ID or None if it doesn't exist
+        """
+
+        return self.__pipelines.get(request_id)
+
+    def exists(self, request_id):
+        """
+        Checks if a request ID is associated to a given process ID
+
+        :param request_id: the request identifier to check
+
+        :return: True or False
+        """
+
+        return request_id in self.__pipelines
+
+
+# Only initialise the class here, this instance is imported in models/__init__.py so that it behaves like the other ORM models,
+# the container's interface mimics the interface of the other ORM models
+MonitoringProcess = _MonitoringProcess()