From bb5a4d1aad26e62b025c134fba3bb1952f771130 Mon Sep 17 00:00:00 2001
From: Nikolay Stanchev <ns17@it-innovation.soton.ac.uk>
Date: Mon, 25 Feb 2019 13:45:32 +0000
Subject: [PATCH] Updates graph_monitor API endpoint to also save the request
identifier and the pipeline process identifier
---
scripts/clmc-service/graph-pipeline.sh | 5 --
src/service/clmcservice/graphapi/tests.py | 10 ++-
src/service/clmcservice/graphapi/views.py | 4 ++
src/service/clmcservice/models/__init__.py | 3 +
.../clmcservice/models/graphapi_models.py | 70 +++++++++++++++++++
5 files changed, 84 insertions(+), 8 deletions(-)
create mode 100644 src/service/clmcservice/models/graphapi_models.py
diff --git a/scripts/clmc-service/graph-pipeline.sh b/scripts/clmc-service/graph-pipeline.sh
index 1f03150..dfd348f 100644
--- a/scripts/clmc-service/graph-pipeline.sh
+++ b/scripts/clmc-service/graph-pipeline.sh
@@ -37,11 +37,6 @@ read query_period db_name results_measurement <<< ${fields}
JSON_CONFIG=$(echo ${JSON_CONFIG} | jq 'del(.query_period, .results_measurement_name)')
-# TODO this shouldn't be in the script maybe move it to the clmc isntall script
-echo "Building network subgraph..."
-response=$(curl -s -X POST http://${CLMC_IP}/clmc-service/graph/network)
-echo ${response}
-
while true
do
echo "Building temporal graph..."
diff --git a/src/service/clmcservice/graphapi/tests.py b/src/service/clmcservice/graphapi/tests.py
index 78ec6b5..521bbe9 100644
--- a/src/service/clmcservice/graphapi/tests.py
+++ b/src/service/clmcservice/graphapi/tests.py
@@ -23,11 +23,12 @@
"""
from json import dumps
-import pytest
from unittest.mock import patch, Mock, PropertyMock
+import pytest
from pyramid import testing
-from clmcservice.graphapi.views import GraphAPI
from pyramid.httpexceptions import HTTPBadRequest, HTTPNotFound, HTTPInternalServerError
+from clmcservice.graphapi.views import GraphAPI
+from clmcservice.models import MonitoringProcess
class TestGraphAPI(object):
@@ -533,6 +534,10 @@ class TestGraphAPI(object):
pid_property_mock.assert_called_once_with() # assert that the process ID attribute was called and saved
returncode_property_mock.assert_called_once_with() # assert that the process return code attribute was called to check if the process has started successfully
+ # check that the process ID was saved
+ assert MonitoringProcess.exists(uuid_mock.return_value), "Request identifier was not saved during the request processing"
+ assert MonitoringProcess.get(uuid_mock.return_value) == pid_property_mock.return_value, "Incorrect PID was saved during the request processing"
+
# check erroneous behaviour
returncode_property_mock.return_value = -1
request = testing.DummyRequest()
@@ -548,7 +553,6 @@ class TestGraphAPI(object):
pid_property_mock.assert_called_with() # assert that the process ID attribute was called and saved
returncode_property_mock.assert_called_with() # assert that the process return code attribute was called to check if the process has started successfully
-
@staticmethod
def check_exist_relationship(relationships_tuple, graph, uuid):
"""
diff --git a/src/service/clmcservice/graphapi/views.py b/src/service/clmcservice/graphapi/views.py
index e562bdf..fd95327 100644
--- a/src/service/clmcservice/graphapi/views.py
+++ b/src/service/clmcservice/graphapi/views.py
@@ -25,6 +25,7 @@
from clmcservice.graphapi.utilities import validate_build_request_body, validate_monitor_request_body, RTT_CYPHER_QUERY_TEMPLATE, \
build_network_graph, delete_network_graph, build_temporal_subgraph, delete_temporal_subgraph, validate_graph_rtt_params, find_node_with_possible_types
+from clmcservice.models import MonitoringProcess
from influxdb import InfluxDBClient
from py2neo import Graph
from pyramid.httpexceptions import HTTPBadRequest, HTTPNotFound, HTTPServiceUnavailable, HTTPNotImplemented, HTTPInternalServerError
@@ -372,6 +373,9 @@ class GraphAPI(object):
if process_return_code is None: # process has started running
log.info("Started a graph pipeline process for SFC {0} with PID {1}".format(sfc, process_pid))
+
+ MonitoringProcess.add({"request_id": request_uuid, "process_id": process_pid})
+
return {"database": database_name, "uuid": request_uuid}
else: # a valid returned code was returned, hence the process has terminated one way or another - we do not expect this since the pipeline script must be continuously running
log.warning("Graph pipeline process for SFC {0} with PID {1} has finished executing unexpectedly with return code {2}".format(sfc, process_pid, process_return_code))
diff --git a/src/service/clmcservice/models/__init__.py b/src/service/clmcservice/models/__init__.py
index bdf3774..345922d 100644
--- a/src/service/clmcservice/models/__init__.py
+++ b/src/service/clmcservice/models/__init__.py
@@ -1,2 +1,5 @@
from .meta import DBSession
+
+from .graphapi_models import MonitoringProcess
+# NOTE: all ORM models defined in this package must be imported here (in the __init__.py file) - Pyramid and SQLAlchemy specific approach
from .whoami_models import ServiceFunctionEndpoint
diff --git a/src/service/clmcservice/models/graphapi_models.py b/src/service/clmcservice/models/graphapi_models.py
new file mode 100644
index 0000000..33e1b5c
--- /dev/null
+++ b/src/service/clmcservice/models/graphapi_models.py
@@ -0,0 +1,70 @@
+#!/usr/bin/python3
+"""
+// © University of Southampton IT Innovation Centre, 2018
+//
+// Copyright in this software belongs to University of Southampton
+// IT Innovation Centre of Gamma House, Enterprise Road,
+// Chilworth Science Park, Southampton, SO16 7NS, UK.
+//
+// This software may not be used, sold, licensed, transferred, copied
+// or reproduced in whole or in part in any manner or form or in or
+// on any media by any person other than in accordance with the terms
+// of the Licence Agreement supplied with the software, or otherwise
+// without the prior written consent of the copyright owners.
+//
+// This software is distributed WITHOUT ANY WARRANTY, without even the
+// implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE, except where stated in the Licence Agreement supplied with
+// the software.
+//
+// Created By : Nikolay Stanchev
+// Created Date : 02-07-2018
+// Created for Project : FLAME
+"""
+
+
+class _MonitoringProcess:
+ """
+ Container-like class used to store key value pairs between graph pipelines request identifiers and process identifiers.
+ The class is declared as "pseudo-private" and is ought to be initialised only in this module and exposed as a singleton.
+ """
+
+ def __init__(self):
+
+ self.__pipelines = {}
+
+ def add(self, instance):
+ """
+ Adds a new key-value pair to the container representing a request ID mapped to a PID.
+
+ :param instance: JSON representation of key value pair - e.g. {"request_id": <reequest uuid>, "process_id": <process ID>}
+ """
+
+ self.__pipelines[instance["request_id"]] = instance["process_id"]
+
+ def get(self, request_id):
+ """
+ Gets the data associated with a request identifier - currently the data is just the process identifier.
+
+ :param request_id: the request identifier to check
+
+ :return: the PID for this request ID or None if it doesn't exist
+ """
+
+ return self.__pipelines.get(request_id)
+
+ def exists(self, request_id):
+ """
+ Checks if a request ID is associated to a given process ID
+
+ :param request_id: the request identifier to check
+
+ :return: True or False
+ """
+
+ return request_id in self.__pipelines
+
+
+# Only initialise the class here, this instance is imported in models/__init__.py so that it behaves like the other ORM models,
+# the container's interface mimics the interface of the other ORM models
+MonitoringProcess = _MonitoringProcess()
--
GitLab