Merge remote-tracking branch 'origin' into clmcservice

Vagrantfile

@@ -91,7 +91,7 @@ Vagrant.configure("2") do |config|
         when 'test-runner'
           instance_config.vm.provision :shell, :path => "clmctest/services/pytest/install.sh"
         when 'clmc-service'
-          instance_config.vm.provision :shell, :path => "scripts/clmc-service/install.sh"       
+          instance_config.vm.provision :shell, :path => "scripts/clmc-service/install.sh", :args => "#{host["influxdb_url"]} #{host["database_name"]} #{host["report_period"]}"       
         else
           # specific service install
           instance_config.vm.provision :shell, :path => "clmctest/services/#{host["service_name"]}/install.sh", env: {"REPO_ROOT" => "/vagrant"}

clmctest/monitoring/E2ESim.py

 #!/usr/bin/python3
 """
-## © University of Southampton IT Innovation Centre, 2018
+## Copyright University of Southampton IT Innovation Centre, 2018
 ##
 ## Copyright in this software belongs to University of Southampton
 ## IT Innovation Centre of Gamma House, Enterprise Road,
@@ -37,8 +37,8 @@ class Simulator(object):
     Simulator used to generate E2E measurements.
     """
 
-    DATABASE = 'E2EMetrics'  # default database name
-    DATABASE_URL = 'http://203.0.113.100:8086'  # default database url
+    DATABASE = 'CLMCMetrics'  # default database name
+    DATABASE_URL = 'http://172.40.231.51:8086'  # default database url
 
     TICK = 1  # a simulation tick represents 1s
     SIMULATION_LENGTH = 120  # simulation time in seconds

clmctest/monitoring/rspec.yml

@@ -34,6 +34,9 @@ hosts:
       - guest: 9080
         host: 9080
     ip_address: "172.40.231.51"
+    influxdb_url: "http://172.40.231.51:8086"
+    database_name: "CLMCMetrics"
+    report_period: 25    
   - name: ipendpoint1
     cpus: 1
     memory: 2048

clmctest/services/pytest/install.sh

@@ -24,7 +24,8 @@
 #//
 #/////////////////////////////////////////////////////////////////////////
 apt-get update
-apt-get -y install python3 python3-pip python-influxdb
+apt-get -y install python3 python3-pip python-influxdb  
 update-alternatives --install /usr/bin/python python /usr/bin/python3 10
+
 pip3 install pytest pyyaml
 pip3 install --upgrade influxdb

docs/clmc-information-model.md

@@ -392,7 +392,7 @@ A simple example and some measurement rows for an endpoint configuration states
 
 Each sample period is 1 second.
 
-First sample period reports the VM being in state __unpalced__ for 0.7s, then changing state to __placing__ for 0.3 seconds. __placing__ is not 
+First sample period reports the VM being in state __unplaced__ for 0.7s, then changing state to __placing__ for 0.3 seconds. __placing__ is not 
 reported since it is not a **completed state**. The mean state time value for __unplaced__ is the same as the sum value because the VM has only been once in this state.
 
 Then the VM is reported to be in current state __placing__ for the whole sample period (1s) for 9 consecutive times. Only the 'current_state' tag value and the 'current_state_time'

scripts/clmc-service/install.sh

@@ -27,6 +27,20 @@
 # Force fail on command fail (off for now as virtualenvwrapper install fails)
 # set -euo pipefail
 
+
+echo "Configuring CLMC service"
+
+# Get command line parameters
+if [ "$#" -ne 3 ]; then
+    echo "Error: illegal number of arguments: "$#
+    echo "Usage: install.sh INFLUX_URL DATABASE_NAME REPORT_PERIOD"
+    exit 1 
+fi
+
+INFLUX_URL=$1
+DATABASE_NAME=$2
+REPORT_PERIOD=$3
+
 # Define tickstack software versions
 INFLUX_VERSION=1.5.2
 INFLUX_CHECKSUM=42fede7b497bdf30d4eb5138db218d1add986fca4fce4a8bcd9c7d6dabaf572a
@@ -121,7 +135,7 @@ fi
 
 # navigate to the clmc-webservice - and check
 echo "----> Moving to CLMC webservice"
-cd /vagrant/src/clmc-webservice
+cd /vagrant/src/clmcwebservice
 if [ $? -ne 0 ] ; then
         echo "Failed: could not find clmc-webservice"
 		exit 1
@@ -129,7 +143,7 @@ fi
 
 # install the service
 echo "----> Installing CLMC web service"
-pip3 install -e .
+pip3 install  .
 if [ $? -ne 0 ] ; then
         echo "Failed: installing clmc-webservice"
 		exit 1
@@ -137,10 +151,23 @@ fi
 
 # start the service
 echo "----> Starting CLMC web service"
-pserve development.ini --reload
+nohup pserve production.ini > /dev/null 2>&1 &
 if [ $? -ne 0 ] ; then
         echo "Failed: starting clmc-webservice"
 		exit 1
 else
 	echo "CLMC service started."
-fi
\ No newline at end of file
+fi
+
+while ! nc -z localhost 9080
+do
+  echo "Waiting for clmc service port 9080 to be ready on localhost..."
+  sleep 5
+done
+
+JSON="{\"aggregator_report_period\": ${REPORT_PERIOD}, \"aggregator_database_name\": \"${DATABASE_NAME}\", \"aggregator_database_url\": \"${INFLUX_URL}\"}"
+curl -H 'Content-Type: application/json' -X PUT -d "${JSON}" http://localhost:9080/aggregator/config
+
+JSON="{\"action\": \"start\"}"
+curl -H 'Content-Type: application/json' -X PUT -d "${JSON}" http://localhost:9080/aggregator/control
+

src/clmcwebservice/clmcservice/aggregator.py

@@ -51,6 +51,7 @@ class Aggregator(object):
         """
 
         # initialise a database client using the database url and the database name
+        print("Creating InfluxDB Connection")
         url_object = urlparse(database_url)
         while True:
             try:
@@ -82,6 +83,8 @@ class Aggregator(object):
         Performs the functionality of the aggregator - query data from both measurements merge that data and post it back in influx every 5 seconds.
         """
 
+        print("Running aggregator")
+
         current_time = int(time())
         while not self._stop_flag.is_set():
 
@@ -94,12 +97,16 @@ class Aggregator(object):
             network_delays = {}
 
             while True:
+
                 try:
+                    print("Query for network delays")
                     result = self.db_client.query(
-                        'SELECT mean(latency) as "net_latency", mean(bandwidth) as "net_bandwidth" FROM "E2EMetrics"."autogen"."network_delays" WHERE time >= {0} and time < {1} GROUP BY path, source, target'.format(
+                        'SELECT mean(latency) as "net_latency", mean(bandwidth) as "net_bandwidth" FROM "{0}"."autogen"."network_delays" WHERE time >= {1} and time < {2} GROUP BY path, source, target'.format(self.db_name,
                             boundary_time_nano, current_time_nano))
                     break
-                except:
+                except Exception as e:
+                    print("Exception getting network delay")
+                    print(e)
                     sleep(self.RETRY_PERIOD)
 
             for item in result.items():
@@ -116,11 +123,14 @@ class Aggregator(object):
 
             while True:
                 try:
+                    print("Query for service delays")                    
                     result = self.db_client.query(
-                        'SELECT mean(response_time) as "response_time", mean(request_size) as "request_size", mean(response_size) as "response_size" FROM "E2EMetrics"."autogen"."service_delays" WHERE time >= {0} and time < {1} GROUP BY endpoint, sf_instance, sfr'.format(
+                        'SELECT mean(response_time) as "response_time", mean(request_size) as "request_size", mean(response_size) as "response_size" FROM "{0}"."autogen"."service_delays" WHERE time >= {1} and time < {2} GROUP BY endpoint, sf_instance, sfr'.format(self.db_name,
                             boundary_time_nano, current_time_nano))
                     break
-                except:
+                except Exception as e:
+                    print("Exception getting service delay")
+                    print(e)
                     sleep(self.RETRY_PERIOD)
 
             for item in result.items():

src/clmcwebservice/production.ini

@@ -11,9 +11,11 @@ pyramid.debug_authorization = false
 pyramid.debug_notfound = false
 pyramid.debug_routematch = false
 pyramid.default_locale_name = en
+
+## Aggregator configuration
 aggregator_running = false
 aggregator_report_period = 5
-aggregator_database_name = E2EMetrics
+aggregator_database_name = CLMCMetrics
 aggregator_database_url = http://172.40.231.51:8086
 
 ###