Merge remote-tracking branch 'origin' into clmcservice

.gitignore

@@ -13,3 +13,4 @@ ubuntu-xenial-16.04-cloudimg-console.log
 .tox
 *$py.class
 **/.pytest_cache/
+build/

.gitlab-ci.yml

@@ -18,7 +18,6 @@
 ##      Created By :            Michael Boniface
 ##      Created Date :          21-03-2018
 ##      Created for Project :   FLAME
-
 stages:  
   - build
   - test
@@ -28,11 +27,15 @@ build:tests:
   stage: build
   only:
     - schedules    
-  script: 
-    - python setup.py sdist --dist-dir=build
+  script:  
+    - cd src/test
+    - python setup.py sdist --dist-dir=../../build
+    - cd ../../src/service
+    - python setup.py sdist --dist-dir=../../build
   artifacts:
     paths:
     - build/clmctest-SNAPSHOT.tar.gz
+    - build/clmcservice-SNAPSHOT.tar.gz
     expire_in: 1 day
 
 test:all:
@@ -42,11 +45,15 @@ test:all:
   dependencies: 
     - build:tests
   script: 
+    - echo "REPO_USER=${REPO_USER}" >> reporc
+    - echo "REPO_PASS=${REPO_PASS}" >> reporc
     - vagrant --fixture=scripts -- up
     - vagrant --fixture=scripts -- ssh test-runner -- -tt "pip3 install /vagrant/build/clmctest-SNAPSHOT.tar.gz"
     - vagrant --fixture=scripts -- ssh test-runner -- -tt "pytest -s --pyargs clmctest.scripts"
     - vagrant --fixture=monitoring -- up
     - vagrant --fixture=monitoring -- ssh test-runner -- -tt "pytest -s --pyargs clmctest.monitoring"  
+    - vagrant --fixture=monitoring -- ssh test-runner -- -tt "pip3 install /vagrant/build/clmcservice-SNAPSHOT.tar.gz"
+    - vagrant --fixture=monitoring -- ssh test-runner -- -tt "pytest -s --pyargs clmcservice.tests"    
     - vagrant --fixture=inputs -- up
     - vagrant --fixture=inputs -- ssh test-runner -- -tt "pytest -s --pyargs clmctest.inputs"        
   when: on_success      

Vagrantfile

@@ -48,8 +48,8 @@ end
 
 # load custom config file
 puts "loading custom infrastructure configuration: #{fixture}"
-puts "custom config file: /clmctest/#{fixture}/rspec.yml"
-host_rspec_file = "clmctest/#{fixture}/rspec.yml"
+puts "custom config file: /src/test/clmctest/#{fixture}/rspec.yml"
+host_rspec_file = "src/test/clmctest/#{fixture}/rspec.yml"
 hosts = YAML.load_file(host_rspec_file)
 
 # Start creating VMS using xenial64 as the base box
@@ -89,12 +89,12 @@ Vagrant.configure("2") do |config|
       puts "Instance name #{instance_name}:"
       case instance_name
         when 'test-runner'
-          instance_config.vm.provision :shell, :path => "clmctest/services/pytest/install.sh"
+          instance_config.vm.provision :shell, :path => "src/test/clmctest/services/pytest/install.sh"
         when 'clmc-service'
-          instance_config.vm.provision :shell, :path => "scripts/clmc-service/install.sh", :args => "#{host["influxdb_url"]} #{host["database_name"]} #{host["report_period"]}"       
+          instance_config.vm.provision :shell, :path => "scripts/clmc-service/install.sh", :args => "#{host["influxdb_url"]} #{host["database_name"]} #{host["report_period"]}", env: {"REPO_ROOT" => "/vagrant"}       
         else
           # specific service install
-          instance_config.vm.provision :shell, :path => "clmctest/services/#{host["service_name"]}/install.sh", env: {"REPO_ROOT" => "/vagrant"}
+          instance_config.vm.provision :shell, :path => "src/test/clmctest/services/#{host["service_name"]}/install.sh", env: {"REPO_ROOT" => "/vagrant"}
     
           # CLMC agent install
           instance_config.vm.provision "file", source: "reporc", destination: "/vagrant/reporc"
@@ -107,7 +107,7 @@ Vagrant.configure("2") do |config|
 
             cp /vagrant/scripts/clmc-agent/telegraf_output.conf /etc/telegraf/telegraf.d/                        
             
-            cp /vagrant/clmctest/services/#{host["service_name"]}/telegraf_#{host["service_name"]}.conf /etc/telegraf/telegraf.d/ 
+            cp /vagrant/src/test/clmctest/services/#{host["service_name"]}/telegraf_#{host["service_name"]}.conf /etc/telegraf/telegraf.d/ 
 
           SHELL
           

clmctest/streaming/__init__.py

-#!/usr/bin/python3
\ No newline at end of file

clmctest/streaming/conftest.py

-#!/usr/bin/python3
-"""
-## © University of Southampton IT Innovation Centre, 2018
-##
-## Copyright in this software belongs to University of Southampton
-## IT Innovation Centre of Gamma House, Enterprise Road, 
-## Chilworth Science Park, Southampton, SO16 7NS, UK.
-##
-## This software may not be used, sold, licensed, transferred, copied
-## or reproduced in whole or in part in any manner or form or in or
-## on any media by any person other than in accordance with the terms
-## of the Licence Agreement supplied with the software, or otherwise
-## without the prior written consent of the copyright owners.
-##
-## This software is distributed WITHOUT ANY WARRANTY, without even the
-## implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
-## PURPOSE, except where stated in the Licence Agreement supplied with
-## the software.
-##
-##      Created By :            Michael Boniface
-##      Created Date :          25-02-2018
-##      Created for Project :   FLAME
-"""
-
-import pytest
-import yaml
-import requests
-import time
-import pkg_resources
-
-
-@pytest.fixture(scope="module")
-def streaming_config():
-    """
-    Reads the service configuration deployed for the streaming simulation test.
-
-    :param request: access the parameters of the fixture
-    :return: the python object representing the read YAML file
-    """
-    rspec = pkg_resources.resource_filename('clmctest.streaming', 'rspec.yml')
-    print("rspec file: {0}".format(rspec))
-
-    with open(rspec, 'r') as stream:
-        data_loaded = yaml.load(stream)
-    return data_loaded
-
-
-@pytest.fixture(scope="module", autouse=True,
-                params=[{'config': {'kapacitor_url': 'http://localhost:8888/chronograf/v1/sources/1/kapacitors', 'kapacitor_file': '/vagrant/test/streaming/kapacitor.json'}}])
-def kapacitor_config(request):
-
-    kapacitor_configuration = request.param['config']['kapacitor_file']
-    with open(kapacitor_configuration, "r") as rule_file:
-        data = "".join(line.strip() for line in rule_file.readlines())
-
-    kapacitor_url = request.param['config']['kapacitor_url']
-    requests.post(url=kapacitor_url, data=data, headers={"Content-Type": "application/json"})
-    time.sleep(1)

clmctest/streaming/dashboard.json

-{"id":1,"cells":[{"i":"396b0b14-1482-4b8a-a359-f144541170a4","x":6,"y":8,"w":6,"h":4,"name":"AdaptiveStreaming_SF_NetworkBytesSentPerSecond","queries":[{"query":"SELECT derivative(mean(\"bytes_sent\"), 1s) AS \"bytes_sent_per_second\" FROM \"CLMCMetrics\".\"autogen\".\"net\" WHERE time \u003e :dashboardTime: AND \"ipendpoint\"='adaptive_streaming_I1_nginx2' GROUP BY :interval:","queryConfig":{"database":"","measurement":"","retentionPolicy":"","fields":[],"tags":{},"groupBy":{"time":"","tags":[]},"areTagsAccepted":false,"rawText":"SELECT derivative(mean(\"bytes_sent\"), 1s) AS \"bytes_sent_per_second\" FROM \"CLMCMetrics\".\"autogen\".\"net\" WHERE time \u003e :dashboardTime: AND \"ipendpoint\"='adaptive_streaming_I1_nginx2' GROUP BY :interval:","range":null}}],"type":"line","links":{"self":"/chronograf/v1/dashboards/1/cells/396b0b14-1482-4b8a-a359-f144541170a4"}},{"i":"480b4037-a816-4e1c-8c84-edb39b0c1f6d","x":0,"y":8,"w":6,"h":4,"name":"AdapativeStreaming_SF_NetworkBytesSentPerSecond","queries":[{"query":"SELECT derivative(mean(\"bytes_sent\"), 1s) AS \"bytes_sent_per_second\" FROM \"CLMCMetrics\".\"autogen\".\"net\" WHERE time \u003e :dashboardTime: AND \"ipendpoint\"='adaptive_streaming_I1_nginx1' GROUP BY :interval:","queryConfig":{"database":"","measurement":"","retentionPolicy":"","fields":[],"tags":{},"groupBy":{"time":"","tags":[]},"areTagsAccepted":false,"rawText":"SELECT derivative(mean(\"bytes_sent\"), 1s) AS \"bytes_sent_per_second\" FROM \"CLMCMetrics\".\"autogen\".\"net\" WHERE time \u003e :dashboardTime: AND \"ipendpoint\"='adaptive_streaming_I1_nginx1' GROUP BY :interval:","range":null}}],"type":"line","links":{"self":"/chronograf/v1/dashboards/1/cells/480b4037-a816-4e1c-8c84-edb39b0c1f6d"}},{"i":"6ad170aa-c5f2-4930-a604-1e88579dffee","x":6,"y":4,"w":6,"h":4,"name":"AdaptiveStreaming_SF2_CPU","queries":[{"query":"SELECT 100-mean(\"usage_idle\") AS \"mean_usage_idle\" FROM \"CLMCMetrics\".\"autogen\".\"cpu\" WHERE time \u003e :dashboardTime: AND \"ipendpoint\"='adaptive_streaming_I1_nginx2' GROUP BY :interval:","queryConfig":{"database":"","measurement":"","retentionPolicy":"","fields":[],"tags":{},"groupBy":{"time":"","tags":[]},"areTagsAccepted":false,"rawText":"SELECT 100-mean(\"usage_idle\") AS \"mean_usage_idle\" FROM \"CLMCMetrics\".\"autogen\".\"cpu\" WHERE time \u003e :dashboardTime: AND \"ipendpoint\"='adaptive_streaming_I1_nginx2' GROUP BY :interval:","range":null}}],"type":"line","links":{"self":"/chronograf/v1/dashboards/1/cells/6ad170aa-c5f2-4930-a604-1e88579dffee"}},{"i":"7e424259-32b8-40be-aa53-477aaf801f0e","x":0,"y":4,"w":6,"h":4,"name":"AdaptiveStreaming_SF1_CPU","queries":[{"query":"SELECT 100-mean(\"usage_idle\") AS \"mean_usage_idle\" FROM \"CLMCMetrics\".\"autogen\".\"cpu\" WHERE time \u003e :dashboardTime: AND \"ipendpoint\"='adaptive_streaming_I1_nginx1' GROUP BY :interval:","queryConfig":{"database":"","measurement":"","retentionPolicy":"","fields":[],"tags":{},"groupBy":{"time":"","tags":[]},"areTagsAccepted":false,"rawText":"SELECT 100-mean(\"usage_idle\") AS \"mean_usage_idle\" FROM \"CLMCMetrics\".\"autogen\".\"cpu\" WHERE time \u003e :dashboardTime: AND \"ipendpoint\"='adaptive_streaming_I1_nginx1' GROUP BY :interval:","range":null}}],"type":"line","links":{"self":"/chronograf/v1/dashboards/1/cells/7e424259-32b8-40be-aa53-477aaf801f0e"}},{"i":"a095c820-8bac-45fe-974d-4030e1bb8770","x":6,"y":0,"w":6,"h":4,"name":"AdaptiveStreaming_SF2_ActiveConnections","queries":[{"query":"SELECT mean(\"active\") AS \"mean_active\" FROM \"CLMCMetrics\".\"autogen\".\"nginx\" WHERE time \u003e :dashboardTime: AND \"ipendpoint\"='adaptive_streaming_I1_nginx2' GROUP BY :interval:","label":"nginx.active","queryConfig":{"database":"CLMCMetrics","measurement":"nginx","retentionPolicy":"autogen","fields":[{"field":"active","funcs":["mean"]}],"tags":{"ipendpoint":["adaptive_streaming_I1_nginx2"]},"groupBy":{"time":"auto","tags":[]},"areTagsAccepted":true,"rawText":null,"range":null}}],"type":"line","links":{"self":"/chronograf/v1/dashboards/1/cells/a095c820-8bac-45fe-974d-4030e1bb8770"}},{"i":"63a7e85a-b411-46be-9478-8479405379a3","x":0,"y":0,"w":6,"h":4,"name":"AdaptiveStreaming_SF1_ActiveConnections","queries":[{"query":"SELECT mean(\"active\") AS \"mean_active\" FROM \"CLMCMetrics\".\"autogen\".\"nginx\" WHERE time \u003e :dashboardTime: AND \"ipendpoint\"='adaptive_streaming_I1_nginx1' GROUP BY :interval:","label":"nginx.active","queryConfig":{"database":"CLMCMetrics","measurement":"nginx","retentionPolicy":"autogen","fields":[{"field":"active","funcs":["mean"]}],"tags":{"ipendpoint":["adaptive_streaming_I1_nginx1"]},"groupBy":{"time":"auto","tags":[]},"areTagsAccepted":true,"rawText":null,"range":null}}],"type":"line","links":{"self":"/chronograf/v1/dashboards/1/cells/63a7e85a-b411-46be-9478-8479405379a3"}}],"templates":[],"name":"Adaptive Streaming Experiment Dashboard","links":{"self":"/chronograf/v1/dashboards/1","cells":"/chronograf/v1/dashboards/1/cells","templates":"/chronograf/v1/dashboards/1/templates"}}

clmctest/streaming/influx.json

-{
-    "id": "1",
-    "name": "Influx 1",
-    "url": "http://localhost:8086",
-    "default": true,
-    "telegraf": "CLMCMetrics"
-}
\ No newline at end of file

clmctest/streaming/kapacitor.json

-{
-    "id": "1",
-    "name": "CLMCKapacitor",
-    "url": "http://localhost:9092",
-    "active": false
-}
\ No newline at end of file

clmctest/streaming/manual.md

-<!--
-// © University of Southampton IT Innovation Centre, 2017
-//
-// Copyright in this software belongs to University of Southampton
-// IT Innovation Centre of Gamma House, Enterprise Road, 
-// Chilworth Science Park, Southampton, SO16 7NS, UK.
-//
-// This software may not be used, sold, licensed, transferred, copied
-// or reproduced in whole or in part in any manner or form or in or
-// on any media by any person other than in accordance with the terms
-// of the Licence Agreement supplied with the software, or otherwise
-// without the prior written consent of the copyright owners.
-//
-// This software is distributed WITHOUT ANY WARRANTY, without even the
-// implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
-// PURPOSE, except where stated in the Licence Agreement supplied with
-// the software.
-//
-//      Created By :            Michael Boniface
-//      Updated By :            Simon Crowle
-//      Created Date :          18-12-2017
-//      Update Date :           14-02-2018
-//      Created for Project :   FLAME
--->
-
-# CLMC Adaptive Streaming Test
-
-This test streams mpeg-dash video using the two nginx servers monitored by Telegraf configured with a default apache plugin and a net_response plugin. The data is stored in the `clmc-service` using database `CLMCMetrics` and measurements `nginx` and `net_response`
-
-The following command brings up the services
-
-`vagrant --fixture=streaming up`
-
-* clmc-service: configured with influx, kapacitor, chornograf
-* nginx1@DC1, nginx2@DC2: configured with nginx and a test video located at http://192.168.50.11:80/test_video/stream.mpd on the internal vbox network and at http://localhost:8081/test_video/stream.mpd if accessing from the host machine
-
-### Run the test set-up
-
-`vagrant --fixture=streaming ssh clmc-service -- "sudo /vagrant/test/streaming/setupCLMC.sh /vagrant/test/streaming"`  
-`vagrant --fixture=streaming ssh nginx1 -- "sudo /vagrant/test/streaming/setupNGINX.sh"`
-
-### Run the automated test
-
-To run the load test using the following command (here, the last parameter '15' refers to the number of VLC player clients to be launched):
-
-`vagrant --fixture=streaming ssh loadtest-streaming -- "sudo /vagrant/test/streaming/run.sh /home/ubuntu/test/streaming http://192.168.50.11/test_video/stream.mpd 15`
-
-This test currently just generates the load and does not have any assertions. It breaks at 1000.
-
-And then point your browser to the Chronograf dashboard:
-
-`http://localhost:8888`
-
-### Run the automated PyTests
-
-SSH into the clmc-service VM:
-
-`vagrant --fixture=streaming ssh clmc-service`
-
-Run the automated tests written in pytest:
-
-`pytest -s /vagrant/test/streaming/` 
-
-### Manual test 
-
-## Manual set-up of Chronograf's CLMC data source
-
-If you __do not__ want to run the automatic set-up, basic entry to the Chronograf dashboard is as follows:
-
-1. Point your browser to: [http://localhost:8888](http://localhost:8888)
-2. Enter your connection string: `http://localhost:8086`
-3. Enter the Name: `Influx 1`
-4. Enter the Telegraf database: `CLMCMetrics`
-
-## Manual test on Windows
-
-### View the video
-Install VLC video client on the host machine, you must use a very recent version otherwise the MPD file cannot we read. At the time of writng the following nighly build was installed:
-
-https://nightlies.videolan.org/build/win32/vlc-3.0.0-rc1-20171201-0326/vlc-3.0.0-20171201-0326-rc1-win32.exe
-
-Start the VLC Player
-
-`Media->Open Network Stream`
-
-The test video is the FLAME project video and it can be viewed at the following location.
-
-`Enter the network URL: http://localhost:8081/test_video/stream.mpd for nginx1 server`
-
-The video should play.
-
-### Query the data
-
-Open Chronograph by entering the following URL into a browser on the host http://localhost:8888. Your CLMC data source, Kapacitor and demonstration dashboard should be ready for you to explore.
-
-Press the Data Explorer in the menu and select the nginx measurement and create a query such as 
-
-`SELECT mean("requests") AS "mean_requests" FROM "CLMCMetrics"."autogen"."nginx" WHERE time > now() - 1h GROUP BY time(10s)`
-
-## KPI triggers
-
-In this demonstrator an example KPI rule has been set up in Kapacitor which fires when the average number of active connections per 5 seconds on the Nginx 1 or Nginx 2 server goes above certain thresholds ( a 'warning' at 10 connections/5 seconds ). The TICKscript specification for this rule is as follows:
-
-```
-dbrp "CLMCMetrics"."autogen"
-
-// Nginx 1 rule
-// -------------
-var n1Data = batch
-    |query(''' SELECT mean("active") AS "mean_active" FROM "CLMCMetrics"."autogen"."nginx" WHERE "ipendpoint"='adaptive_streaming_I1_nginx1' ''')
-        .period(5s)
-        .every(5s)
-
-varn n1Alert = n1Data
-    |alert()
-        .id('{{ .Name }}/adaptive_streaming_I1_nginx1')
-        .message('{{ .ID }} is {{ .Level }} Mean active connections: {{ index .Fields "mean_active" }}')
-        .warn(lambda: "mean_active" > 10)
-        .slack()
-        .log( '/tmp/RPSLoad.log' )
-
-// Nginx 2 rule
-// -------------
-var n2Data = batch
-    |query(''' SELECT mean("active") AS "mean_active" FROM "CLMCMetrics"."autogen"."nginx" WHERE "ipendpoint"='adaptive_streaming_I1_nginx2' ''')
-        .period(5s)
-        .every(5s)
-
-var n2Alert = n2Data
-    |alert()
-        .id('{{ .Name }}/adaptive_streaming_I1_nginx2')
-        .message('{{ .ID }} is {{ .Level }} Mean active connections: {{ index .Fields "mean_active" }}')
-        .warn(lambda: "mean_active" > 10)
-        .slack()
-        .log( '/tmp/RPSLoad.log' )
-```
-
-Alerts are sent to both an internal logging within the CLMC service file system and also to a FLAME demo Slack service:
-
-https://flamedemo-itinnov.slack.com
-
-Alerts can be found under the '#clmc' channel. 
-
-### Kapacitor rules in Chronograf's GUI
-
-Additional rules can be added to this demonstrator either via the Chronograf GUI (see [here](https://docs.influxdata.com/chronograf/v1.4/introduction/getting-started/#4-connect-chronograf-to-kapacitor) for more information) or by using the Kapacitor HTTP API and TICKscript (for an introduction, [look here](https://docs.influxdata.com/kapacitor/v1.4/tick/)).

clmctest/streaming/report.sh

-#!/bin/bash
-## © University of Southampton IT Innovation Centre, 2018
-##
-## Copyright in this software belongs to University of Southampton
-## IT Innovation Centre of Gamma House, Enterprise Road, 
-## Chilworth Science Park, Southampton, SO16 7NS, UK.
-##
-## This software may not be used, sold, licensed, transferred, copied
-## or reproduced in whole or in part in any manner or form or in or
-## on any media by any person other than in accordance with the terms
-## of the Licence Agreement supplied with the software, or otherwise
-## without the prior written consent of the copyright owners.
-##
-## This software is distributed WITHOUT ANY WARRANTY, without even the
-## implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
-## PURPOSE, except where stated in the Licence Agreement supplied with
-## the software.
-##
-##      Created By :            Stephen Phillips
-##      Created Date :          19-02-2018
-##      Created for Project :   FLAME
-
-# This script reads stdin and expects the output of cvlc.
-# It is used by the run.sh script and receives the output of the cvlc client.
-# It counts the number of times the frame "dropping" error is seen and every 10 times it sends a message to telegraf reporting "another 10" errors.
-
-if [ "$#" -ne 1 ]; then
-    echo "Error: illegal number of arguments: "$#
-    echo "Usage: report.sh <client number>"
-    exit 
-fi
-
-COUNTER=$1
-TELEGRAF=http://localhost:8186
-
-ERR_COUNT=0
-while read line; do
-  if [[ $line = *"dropping"* ]]; then
-    ERR_COUNT=$(($ERR_COUNT + 1))
-  fi
-  TEN=$((ERR_COUNT % 10))
-  if [ $TEN -eq 0 ]; then
-    curl -i -XPOST "${TELEGRAF}/write?precision=s" --data-binary "vlc,client=${COUNTER} drop_error=10 $(date +%s)" >& /dev/null
-  fi
-done
\ No newline at end of file

clmctest/streaming/rspec.yml

-## (c) University of Southampton IT Innovation Centre, 2018
-##
-## Copyright in this software belongs to University of Southampton
-## IT Innovation Centre of Gamma House, Enterprise Road, 
-## Chilworth Science Park, Southampton, SO16 7NS, UK.
-##
-## This software may not be used, sold, licensed, transferred, copied
-## or reproduced in whole or in part in any manner or form or in or
-## on any media by any person other than in accordance with the terms
-## of the Licence Agreement supplied with the software, or otherwise
-## without the prior written consent of the copyright owners.
-##
-## This software is distributed WITHOUT ANY WARRANTY, without even the
-## implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
-## PURPOSE, except where stated in the Licence Agreement supplied with
-## the software.
-##
-##      Created By :            Michael Boniface
-##      Created Date :          02-02-2018
-##      Created for Project :   FLAME
-
-hosts:
-  - name: clmc-service
-    cpus: 1
-    memory: 2048
-    disk: "10GB"
-    forward_ports:
-      - guest: 8086
-        host: 8086
-      - guest: 8888
-        host: 8888
-      - guest: 9092
-        host: 9092
-    ip_address: "192.168.50.10"
-  - name: nginx1
-    cpus: 1
-    memory: 2048
-    disk: "10GB"
-    service_name: "nginx"
-    forward_ports:
-      - guest: 80
-        host: 8081
-    ip_address: "192.168.50.11"
-    location: "DC1"
-    sfc_id: "MS_Template_1"
-    sfc_id_instance: "MS_I1"
-    sf_id: "adaptive_streaming"
-    sf_id_instance: "adaptive_streaming_I1"
-    ipendpoint_id: "adaptive_streaming_I1_nginx1"
-    sr_id: "service_router"
-    influxdb_url: "http://192.168.50.10:8086"
-    database_name: "CLMCMetrics"
-  - name: nginx2
-    cpus: 1
-    memory: 2048
-    disk: "10GB"
-    service_name: "nginx"
-    forward_ports:
-      - guest: 80
-        host: 8082
-    ip_address: "192.168.50.12"
-    location: "DC2"
-    sfc_id: "MS_Template_1"
-    sfc_id_instance: "MS_I1"
-    sf_id: "adaptive_streaming"
-    sf_id_instance: "adaptive_streaming_I1"
-    ipendpoint_id: "adaptive_streaming_I1_nginx2"
-    sr_id: "service_router"    
-    influxdb_url: "http://192.168.50.10:8086"
-    database_name: "CLMCMetrics"  
-  - name: loadtest-streaming
-    cpus: 2
-    memory: 4096
-    disk: "10GB"
-    service_name: "loadtest-streaming"
-    forward_ports:
-      - guest: 80
-        host: 8083
-    ip_address: "192.168.50.13"
-    location: "DC1"
-    sfc_id: "MS_Template_1"
-    sfc_id_instance: "MS_I1"
-    sf_id: "adaptive_streaming_client"
-    sf_id_instance: "adaptive_streaming_I1"
-    ipendpoint_id: "adaptive_streaming_I1_client1"
-    sr_id: "service_router"
-    influxdb_url: "http://192.168.50.10:8086"
-    database_name: "CLMCMetrics"    

clmctest/streaming/rules.json

-{
-    "id" : "Request_Rate_Alert_NGINXServers",
-    "type" : "batch",
-    "dbrps" : [{"db": "CLMCMetrics", "rp" : "autogen"}],
-    
-    "script" : "\/\/ NGINX 1 Rule\r\n\/\/ -------------\r\nvar n1Data = batch\r\n    |query(''' SELECT mean(\"active\") AS \"mean_active\" FROM \"CLMCMetrics\".\"autogen\".\"nginx\" WHERE \"ipendpoint\"='adaptive_streaming_I1_nginx1' ''')\r\n        .period(5s)\r\n        .every(5s)\r\n\r\nvar n1Alert = n1Data\r\n    |alert()\r\n        .id('{{ .Name }}\/adaptive_streaming_I1_nginx1')\r\n        .message('{{ .ID }} is {{ .Level }} Mean active connections: {{ index .Fields \"mean_active\" }}')\r\n        .warn(lambda: \"mean_active\" > 10)\r\n        .slack()\r\n        .log( '\/tmp\/RPSLoad.log' )\r\n\r\n\/\/ NGINX 2 Rule\r\n\/\/ -------------\r\nvar n2Data = batch\r\n    |query(''' SELECT mean(\"active\") AS \"mean_active\" FROM \"CLMCMetrics\".\"autogen\".\"nginx\" WHERE \"ipendpoint\"='adaptive_streaming_I1_nginx2' ''')\r\n        .period(5s)\r\n        .every(5s)\r\n\r\nvar n2Alert = n2Data\r\n    |alert()\r\n        .id('{{ .Name }}\/adaptive_streaming_I1_nginx2')\r\n        .message('{{ .ID }} is {{ .Level }} Mean active connections: {{ index .Fields \"mean_active\" }}')\r\n        .warn(lambda: \"mean_active\" > 10)\r\n        .slack()\r\n        .log( '\/tmp\/RPSLoad.log' )",
-    
-    "status" : "enabled"
-}
\ No newline at end of file

clmctest/streaming/run.sh

-#!/bin/bash
-#/////////////////////////////////////////////////////////////////////////
-#//
-#// (c) University of Southampton IT Innovation Centre, 2017
-#//
-#// Copyright in this software belongs to University of Southampton
-#// IT Innovation Centre of Gamma House, Enterprise Road,
-#// Chilworth Science Park, Southampton, SO16 7NS, UK.
-#//
-#// This software may not be used, sold, licensed, transferred, copied
-#// or reproduced in whole or in part in any manner or form or in or
-#// on any media by any person other than in accordance with the terms
-#// of the Licence Agreement supplied with the software, or otherwise
-#// without the prior written consent of the copyright owners.
-#//
-#// This software is distributed WITHOUT ANY WARRANTY, without even the
-#// implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
-#// PURPOSE, except where stated in the Licence Agreement supplied with
-#// the software.
-#//
-#//      Created By :            Michael Boniface
-#//      Created Date :          15/02/2017
-#//      Created for Project :   FLAME
-#//
-#/////////////////////////////////////////////////////////////////////////
-
-if [ "$#" -ne 3 ]; then
-    echo "Error: illegal number of arguments: "$#
-      echo "Usage: run.sh TEST_RUN_DIR STREAM_URI MAX_CLIENTS"
-      exit 
-fi
-
-# create test directories
-TEST_FOLDER=$(date +%Y%m%d%H%M%S) 
-TEST_RUN_DIR=$1
-TEST_DIR=$TEST_RUN_DIR"/streaming/"$TEST_FOLDER
-echo "Test directory: "$TEST_DIR
-mkdir -p  "$TEST_DIR"
-
-# run testplan
-cd $TEST_DIR
-
-#jmeter -n -LDEBUG -t /vagrant/test/streaming/testplan.jmx -l results.jtx -j jmeter.log
-
-# quick bash equivalent in case Jmeter fails
-STREAM_URI=$2
-COUNTER=0
-MAX_CLIENTS=$3
-while [  $COUNTER -lt $MAX_CLIENTS ]; do
-  # run cvlc headless, redirect stderr into stdout, pipe that into the report.sh script
-  cvlc -Vdummy --no-audio $STREAM_URI 2>&1 | /vagrant/test/streaming/report.sh ${COUNTER} &
-  sleep 1
-  let COUNTER=COUNTER+1 
-done
-
-
-

clmctest/streaming/setupCLMC.sh

-#!/bin/bash
-#/////////////////////////////////////////////////////////////////////////
-#//
-#// (c) University of Southampton IT Innovation Centre, 2018
-#//
-#// Copyright in this software belongs to University of Southampton
-#// IT Innovation Centre of Gamma House, Enterprise Road,
-#// Chilworth Science Park, Southampton, SO16 7NS, UK.
-#//
-#// This software may not be used, sold, licensed, transferred, copied
-#// or reproduced in whole or in part in any manner or form or in or
-#// on any media by any person other than in accordance with the terms
-#// of the Licence Agreement supplied with the software, or otherwise
-#// without the prior written consent of the copyright owners.
-#//
-#// This software is distributed WITHOUT ANY WARRANTY, without even the
-#// implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
-#// PURPOSE, except where stated in the Licence Agreement supplied with
-#// the software.
-#//
-#//      Created By :            Simon Crowle
-#//      Created Date :          14/02/2018
-#//      Created for Project :   FLAME
-#//
-#/////////////////////////////////////////////////////////////////////////
-
-TEST_DIR=$1
-
-# copy Kapacitor conf to /etc/kapacitor and restart
-
-systemctl stop kapacitor
-echo $TEST_DIR"/kapacitor.conf"
-cp $TEST_DIR/kapacitor.conf /etc/kapacitor/kapacitor.conf
-systemctl start kapacitor
-
-# Set up Influx data source
-curl -i -X POST -H "Content-Type: application/json" http://localhost:8888/chronograf/v1/sources -d @$TEST_DIR/influx.json
-
-# Set up dashboard
-curl -i -X POST -H "Content-Type: application/json" http://localhost:8888/chronograf/v1/dashboards -d @$TEST_DIR/dashboard.json

clmctest/streaming/setupNGINX.sh

-#!/bin/bash
-#/////////////////////////////////////////////////////////////////////////
-#//
-#// (c) University of Southampton IT Innovation Centre, 2018
-#//
-#// Copyright in this software belongs to University of Southampton
-#// IT Innovation Centre of Gamma House, Enterprise Road,
-#// Chilworth Science Park, Southampton, SO16 7NS, UK.
-#//
-#// This software may not be used, sold, licensed, transferred, copied
-#// or reproduced in whole or in part in any manner or form or in or
-#// on any media by any person other than in accordance with the terms
-#// of the Licence Agreement supplied with the software, or otherwise
-#// without the prior written consent of the copyright owners.
-#//
-#// This software is distributed WITHOUT ANY WARRANTY, without even the
-#// implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
-#// PURPOSE, except where stated in the Licence Agreement supplied with
-#// the software.
-#//
-#//      Created By :            Simon Crowle
-#//      Created Date :          14/02/2018
-#//      Created for Project :   FLAME
-#//
-#/////////////////////////////////////////////////////////////////////////
-
-# NGINX
-DEST_DIR="/usr/share/nginx/html"
-
-TEST_VIDEO="20180212104221flame-project-full.mp4"
-TEST_VIDEO_ARCHIVE=$TEST_VIDEO".gz"
-DEST_FILE=$DEST_DIR"/"$TEST_VIDEO_ARCHIVE
-
-echo "ftp://ftp.it-innovation.soton.ac.uk/testdata/video/"$TEST_VIDEO_ARCHIVE
-
-# Copy files for MPEG-DASH testing
-curl "ftp://ftp.it-innovation.soton.ac.uk/testdata/video/"$TEST_VIDEO_ARCHIVE --user flame-rw:DR8ngj3ogSjd8gl -o $DEST_FILE
-tar -xvf $DEST_FILE -C $DEST_DIR
-
-rm -rf $DEST_FILE
-mv $DEST_DIR"/"$TEST_VIDEO $DEST_DIR"/"test_video

clmctest/streaming/stop.sh

-#!/bin/bash
-"""
-// © University of Southampton IT Innovation Centre, 2018
-//
-// Copyright in this software belongs to University of Southampton
-// IT Innovation Centre of Gamma House, Enterprise Road, 
-// Chilworth Science Park, Southampton, SO16 7NS, UK.
-//
-// This software may not be used, sold, licensed, transferred, copied
-// or reproduced in whole or in part in any manner or form or in or
-// on any media by any person other than in accordance with the terms
-// of the Licence Agreement supplied with the software, or otherwise
-// without the prior written consent of the copyright owners.
-//
-// This software is distributed WITHOUT ANY WARRANTY, without even the
-// implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
-// PURPOSE, except where stated in the Licence Agreement supplied with
-// the software.
-//
-//      Created By :            Michael Boniface
-//      Created Date :          15-02-2018
-//      Created for Project :   FLAME
-"""
-
-for pid in $(ps -ef | grep "/usr/bin/vlc" | awk '{print $2}'); do kill -9 $pid; done
-# TODO: 'killall vlc' should work: need to test though
\ No newline at end of file

clmctest/streaming/test_rspec.py

-#!/usr/bin/python3
-"""
-// © University of Southampton IT Innovation Centre, 2018
-//
-// Copyright in this software belongs to University of Southampton
-// IT Innovation Centre of Gamma House, Enterprise Road, 
-// Chilworth Science Park, Southampton, SO16 7NS, UK.
-//
-// This software may not be used, sold, licensed, transferred, copied
-// or reproduced in whole or in part in any manner or form or in or
-// on any media by any person other than in accordance with the terms
-// of the Licence Agreement supplied with the software, or otherwise
-// without the prior written consent of the copyright owners.
-//
-// This software is distributed WITHOUT ANY WARRANTY, without even the
-// implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
-// PURPOSE, except where stated in the Licence Agreement supplied with
-// the software.
-//
-//      Created By :            Michael Boniface
-//      Created Date :          24-02-2018
-//      Created for Project :   FLAME
-"""
-
-from subprocess import run
-from platform import system
-import pytest
-
-
-@pytest.mark.parametrize("service_name", [
-    'clmc-service',
-    'nginx1',
-    'nginx2',
-    'loadtest-streaming'
-])
-def test_service_names(streaming_config, service_name):
-    """
-    Tests the service names in the configuration.
-
-    :param streaming_config: the configuration fixture collected from conftest.py
-    :param service_name the service name to test
-    """
-
-    assert any(s['name'] == service_name for s in streaming_config['hosts']), "{0} not in list of hosts".format(service_name)
-    print("\nSuccessfully passed configuration test for service name {0}\n".format(service_name))
-
-
-def test_ping(streaming_config):
-    """
-    Pings each service to test for liveliness
-
-    :param streaming_config: the configuration fixture collected from conftest.py
-    """
-
-    print("\n")  # blank line printed for formatting purposes
-
-    ping_count = 1
-    system_dependent_param = "-n" if system().lower() == "windows" else "-c"
-
-    for service in streaming_config['hosts']:
-        command = ["ping", system_dependent_param, str(ping_count), service['ip_address']]
-        assert run(command).returncode == 0, "Service ping test failed for {0} with ip address {1}".format(service['name'], service['ip_address'])
-        print("\nSuccessfully passed ping test for service: {0}\n".format(service['name']))

clmctest/streaming/test_rule1.json

-{
-    "id" : "TestRule1",
-    "type" : "batch",
-    "dbrps" : [{"db": "CLMCMetrics", "rp" : "autogen"}],
-
-    "script" : "var ruleData = batch\r\n    |query(''' SELECT mean(\"handled\") AS \"mean_handled\" FROM \"CLMCMetrics\".\"autogen\".\"nginx\" WHERE \"ipendpoint\"='adaptive_streaming_I1_nginx1' ''')\r\n        .period(5s)\r\n        .every(5s)\r\n\r\nvar ruleAlert = ruleData\r\n    |alert()\r\n        .id('{{ .Name }}\/adaptive_streaming_I1_nginx1')\r\n        .message('{{ .ID }} is {{ .Level }} Mean handled connections: {{ index .Fields \"mean_handled\" }}')\r\n        .warn(lambda: \"mean_handled\" > 10)\r\n        .log( '\/tmp\/TestRule1.log' )",
-
-    "status" : "enabled"
-}
\ No newline at end of file

clmctest/streaming/test_rule2.json

-{
-    "id" : "TestRule2",
-    "type" : "batch",
-    "dbrps" : [{"db": "CLMCMetrics", "rp" : "autogen"}],
-
-    "script" : "var ruleData = batch\r\n    |query(''' SELECT mean(\"waiting\") AS \"mean_waiting\" FROM \"CLMCMetrics\".\"autogen\".\"nginx\" WHERE \"ipendpoint\"='adaptive_streaming_I1_nginx1' ''')\r\n        .period(5s)\r\n        .every(5s)\r\n\r\nvar ruleAlert = ruleData\r\n    |alert()\r\n        .id('{{ .Name }}\/adaptive_streaming_I1_nginx1')\r\n        .message('{{ .ID }} is {{ .Level }} Mean waiting connections: {{ index .Fields \"mean_waiting\" }}')\r\n        .warn(lambda: \"mean_waiting\" > 10)\r\n        .log( '\/tmp\/TestRule2.log' )",
-
-    "status" : "enabled"
-}
\ No newline at end of file