Merge branch 'rp-scripts' into 'integration'

Rp scripts

Closes #48

See merge request FLAME/flame-clmc!17

Vagrantfile

@@ -87,13 +87,14 @@ Vagrant.configure("2") do |config|
       puts "Instance name #{instance_name}:"
       case instance_name
         when 'clmc-service'
-          instance_config.vm.provision :shell, :path => "scripts/clmc-service/install.sh"
+       #   instance_config.vm.provision :shell, :path => "scripts/clmc-service/install.sh"
+          instance_config.vm.provision :shell, :path => "test/services/pytest/install.sh"         
         else
           # specific service install
           instance_config.vm.provision :shell, :path => "test/services/#{host["service_name"]}/install.sh", env: {"REPO_ROOT" => "/vagrant"}
     
           # CLMC agent install
-          instance_config.vm.provision :shell, :path => "scripts/clmc-agent/install.sh", :args => "/vagrant/test/services/#{host["service_name"]}/telegraf_template.conf #{host["location"]} #{host["sfc_id"]} #{host["sfc_id_instance"]} #{host["sf_id"]} #{host["sf_id_instance"]} #{host["ipendpoint_id"]} #{host["influxdb_url"]} #{host["database_name"]}"
+          instance_config.vm.provision :shell, :path => "scripts/clmc-agent/install.sh", :args => "/vagrant/scripts/clmc-agent/telegraf_template.conf /vagrant/test/services/#{host["service_name"]}/telegraf_#{host["service_name"]}.conf #{host["location"]} #{host["sfc_id"]} #{host["sfc_id_instance"]} #{host["sf_id"]} #{host["sf_id_instance"]} #{host["ipendpoint_id"]} #{host["influxdb_url"]} #{host["database_name"]}"
       end
 
       

scripts/clmc-agent/install.sh

@@ -24,34 +24,50 @@
 #//
 #/////////////////////////////////////////////////////////////////////////
 
+# Force fail on command fail
+set -euo pipefail
+
 # Install telegraf
-if [ "$#" -ne 9 ]; then
+if [ "$#" -ne 10 ]; then
     echo "Error: illegal number of arguments: "$#
-      echo "Usage: install-clmc-agent.sh TELEGRAF_CONF_FILE LOCATION SFC_ID SFC_ID_INSTANCE SF_ID SF_ID_INSTANCE IP_ENDPOINT_ID INFLUXDB_URL DATABASE_NAME"
+      echo "Usage: install-clmc-agent.sh TELEGRAF_GENERAL_CONF_FILE TELEGRAF_SERVICE_CONF_FILE LOCATION SFC_ID SFC_ID_INSTANCE SF_ID SF_ID_INSTANCE IP_ENDPOINT_ID INFLUXDB_URL DATABASE_NAME"
       exit 
 fi
 
-TELEGRAF_CONF_FILE=$1
-LOCATION=$2
-SFC_ID=$3
-SFC_ID_INSTANCE=$4
-SF_ID=$5
-SF_ID_INSTANCE=$6
-IP_ENDPOINT_ID=$7
-INFLUXDB_URL=$8
-DATABASE_NAME=$9
+TELEGRAF_GENERAL_CONF_FILE=$1
+TELEGRAF_SERVICE_CONF_FILE=$2
+LOCATION=$3
+SFC_ID=$4
+SFC_ID_INSTANCE=$5
+SF_ID=$6
+SF_ID_INSTANCE=$7
+IP_ENDPOINT_ID=$8
+INFLUXDB_URL=$9
+DATABASE_NAME=${10}
 
-if [ ! -f $TELEGRAF_CONF_FILE ]; then
-    echo "Error: Telegraf conf template file not found: "$TELEGRAF_CONF_FILE
+if [ ! -f $TELEGRAF_GENERAL_CONF_FILE ]; then
+    echo "Error: Telegraf conf template file not found: "$TELEGRAF_GENERAL_CONF_FILE
     exit
 fi
 
 wget https://dl.influxdata.com/telegraf/releases/telegraf_1.3.2-1_amd64.deb
+sha1sum telegraf_1.3.2-1_amd64.deb | grep '73794cc2986fef7c81cfff8bc638cd84d6629da8' &> /dev/null
+if [ $? == 1 ]; then
+	echo "telegraf download failed sha1sum check"
+	exit
+fi
 dpkg -i telegraf_1.3.2-1_amd64.deb
 
 # Copy configuration
-echo "Telegraf config file: " $TELEGRAF_CONF_FILE
-cp $TELEGRAF_CONF_FILE /etc/telegraf/telegraf.conf
+echo "Telegraf general config file: " $TELEGRAF_GENERAL_CONF_FILE
+cp $TELEGRAF_GENERAL_CONF_FILE /etc/telegraf/telegraf.conf
+
+if [ ! -f $TELEGRAF_SERVICE_CONF_FILE ]; then
+    echo "Warning: Telegraf conf service specific file not found: "$TELEGRAF_SERVICE_CONF_FILE
+else
+	echo "Telegraf service config file: " $TELEGRAF_SERVICE_CONF_FILE
+	cp $TELEGRAF_SERVICE_CONF_FILE /etc/telegraf/telegraf.d/
+fi
 
 echo "INFLUXDB_URL: " $INFLUXDB_URL
 echo "DATABASE_NAME: " $DATABASE_NAME

scripts/clmc-agent/telegraf_ipendpoint_template.conf

-# Telegraf configuration
-
-# Telegraf is entirely plugin driven. All metrics are gathered from the
-# declared inputs, and sent to the declared outputs.
-
-# Plugins must be declared in here to be active.
-# To deactivate a plugin, comment out the name and any variables.
-
-# Use 'telegraf -config telegraf.conf -test' to see what metrics a config
-# file would generate.
-
-# Global tags can be specified here in key="value" format.
-[global_tags]
-  # location of the data centre
-  location="{{LOCATION}}"
-  # media service template id
-  sfc="{{SFC_ID}}"
-  # media service instance
-  sfc_i="{{SFC_ID_INSTANCE}}"
-  # service function type
-  sf="{{SF_ID}}"
-  # service function instance id
-  sf_i="{{SF_ID_INSTANCE}}"
-  # ipendpoint id aka surrogate instance
-  ipendpoint="{{IP_ENDPOINT_ID}}"
-
-# Configuration for telegraf agent
-[agent]
-  ## Default data collection interval for all inputs
-  interval = "10s"
-  ## Rounds collection interval to 'interval'
-  ## ie, if interval="10s" then always collect on :00, :10, :20, etc.
-  round_interval = true
-
-  ## Telegraf will cache metric_buffer_limit metrics for each output, and will
-  ## flush this buffer on a successful write.
-  metric_buffer_limit = 1000
-  ## Flush the buffer whenever full, regardless of flush_interval.
-  flush_buffer_when_full = true
-
-  ## Collection jitter is used to jitter the collection by a random amount.
-  ## Each plugin will sleep for a random time within jitter before collecting.
-  ## This can be used to avoid many plugins querying things like sysfs at the
-  ## same time, which can have a measurable effect on the system.
-  collection_jitter = "0s"
-
-  ## Default flushing interval for all outputs. You shouldn't set this below
-  ## interval. Maximum flush_interval will be flush_interval + flush_jitter
-  flush_interval = "10s"
-  ## Jitter the flush interval by a random amount. This is primarily to avoid
-  ## large write spikes for users running a large number of telegraf instances.
-  ## ie, a jitter of 5s and interval 10s means flushes will happen every 10-15s
-  flush_jitter = "0s"
-
-  ## Logging configuration:
-  ## Run telegraf in debug mode
-  debug = false
-  ## Run telegraf in quiet mode
-  quiet = false
-  ## Specify the log file name. The empty string means to log to stdout.
-  logfile = "G:/Telegraf/telegraf.log"
-
-  ## Override default hostname, if empty use os.Hostname()
-  hostname = ""
-
-
-###############################################################################
-#                                  OUTPUTS                                    #
-###############################################################################
-
-# Configuration for influxdb server to send metrics to
-[[outputs.influxdb]]
-  # The full HTTP or UDP endpoint URL for your InfluxDB instance.
-  # Multiple urls can be specified but it is assumed that they are part of the same
-  # cluster, this means that only ONE of the urls will be written to each interval.
-  # urls = ["udp://127.0.0.1:8089"] # UDP endpoint example
-  urls = ["{{INFLUXDB_URL}}"] # required
-  # The target database for metrics (telegraf will create it if not exists)
-  database = "{{DATABASE_NAME}}" # required
-  # Precision of writes, valid values are "ns", "us" (or "µs"), "ms", "s", "m", "h".
-  # note: using second precision greatly helps InfluxDB compression
-  precision = "s"
-
-  ## Write timeout (for the InfluxDB client), formatted as a string.
-  ## If not provided, will default to 5s. 0s means no timeout (not recommended).
-  timeout = "5s"
-  # username = "telegraf"
-  # password = "metricsmetricsmetricsmetrics"
-  # Set the user agent for HTTP POSTs (can be useful for log differentiation)
-  # user_agent = "telegraf"
-  # Set UDP payload size, defaults to InfluxDB UDP Client default (512 bytes)
-  # udp_payload = 512
-
-
-###############################################################################
-#                                  INPUTS                                     #
-###############################################################################
-# # Influx HTTP write listener
-[[inputs.http_listener]]
-  ## Address and port to host HTTP listener on
-  service_address = ":8186"
-
-  ## timeouts
-  read_timeout = "10s"
-  write_timeout = "10s"
-
-  ## HTTPS
-  #tls_cert= "/etc/telegraf/cert.pem"
-  #tls_key = "/etc/telegraf/key.pem"
-
-  ## MTLS
-  #tls_allowed_cacerts = ["/etc/telegraf/clientca.pem"]

test/services/apache/telegraf_template.conf → scripts/clmc-agent/telegraf_template.conf

@@ -172,27 +172,6 @@
   ## MTLS
   #tls_allowed_cacerts = ["/etc/telegraf/clientca.pem"]
 
-# Read Apache status information (mod_status)
-[[inputs.apache]]
-  ## An array of URLs to gather from, must be directed at the machine
-  ## readable version of the mod_status page including the auto query string.
-  ## Default is "http://localhost/server-status?auto".
-  urls = ["http://localhost/server-status?auto"]
-
-  ## Credentials for basic HTTP authentication.
-  # username = "myuser"
-  # password = "mypassword"
-
-  ## Maximum time to receive response.
-  # response_timeout = "5s"
-
-  ## Optional SSL Config
-  # ssl_ca = "/etc/telegraf/ca.pem"
-  # ssl_cert = "/etc/telegraf/cert.pem"
-  # ssl_key = "/etc/telegraf/key.pem"
-  ## Use SSL but skip chain & host verification
-  # insecure_skip_verify = false
-
 [[inputs.net_response]]
   ## Protocol, must be "tcp" or "udp"
   ## NOTE: because the "udp" protocol does not respond to requests, it requires

scripts/clmc-service/install.sh

@@ -24,20 +24,39 @@
 #//
 #/////////////////////////////////////////////////////////////////////////
 
+# Force fail on command fail
+set -euo pipefail
+
+
 # install python for the simulator
 apt-get update
 apt-get -y install python
 
 # install influx
 wget https://dl.influxdata.com/influxdb/releases/influxdb_1.2.4_amd64.deb
+sha1sum influxdb_1.2.4_amd64.deb | grep 'e77522b65a582787b0d61b90d355284bb2683258' &> /dev/null
+if [ $? == 1 ]; then
+	echo "influx download failed sha1sum check"
+	exit
+fi
 dpkg -i influxdb_1.2.4_amd64.deb
 
 # install kapacitor
 wget https://dl.influxdata.com/kapacitor/releases/kapacitor_1.3.1_amd64.deb
+sha1sum kapacitor_1.3.1_amd64.deb | grep '5ae1ead9904ea651e72b181848c2e84338eb88b4' &> /dev/null
+if [ $? == 1 ]; then
+	echo "Kapacitor download failed sha1sum check"
+	exit
+fi
 dpkg -i kapacitor_1.3.1_amd64.deb
 
 # install Chronograf
 wget https://dl.influxdata.com/chronograf/releases/chronograf_1.3.3.0_amd64.deb
+sha1sum chronograf_1.3.3.0_amd64.deb | grep '9b567598c078ba1ad73dd587d7f32163e0886471' &> /dev/null
+if [ $? == 1 ]; then
+	echo "Chronograf download failed sha1sum check"
+	exit
+fi
 dpkg -i chronograf_1.3.3.0_amd64.deb
 
 systemctl start influxdb

test/services/apache/install.sh

@@ -28,18 +28,6 @@
 sudo apt-get update
 sudo apt-get -y install apache2
 
-TEST_VIDEO="20180212104221flame-project-full.mp4"
-TEST_VIDEO_ARCHIVE=$TEST_VIDEO".gz"
-DEST_DIR="/var/www/html/" 
-DEST_FILE=$DEST_DIR"/"$TEST_VIDEO_ARCHIVE
-
-# Copy files for MPEG-DASH testing
-curl "ftp://ftp.it-innovation.soton.ac.uk/testdata/video/"$TEST_VIDEO_ARCHIVE --user flame-rw:DR8ngj3ogSjd8gl -o $DEST_FILE
-tar -xvf $DEST_FILE -C $DEST_DIR
-
-rm -rf $DEST_FILE
-mv $DEST_DIR"/"$TEST_VIDEO $DEST_DIR"/"test_video
-
 # start apache
 apachectl -k start
 apachectl -k restart

test/services/apache/telegraf_apache.conf

+[[inputs.apache]]
+  ## An array of URLs to gather from, must be directed at the machine
+  ## readable version of the mod_status page including the auto query string.
+  ## Default is "http://localhost/server-status?auto".
+  urls = ["http://localhost:8890/server-status?auto"]
+
+  ## Credentials for basic HTTP authentication.
+  # username = "myuser"
+  # password = "mypassword"
+
+  ## Maximum time to receive response.
+  # response_timeout = "5s"
+
+  ## Optional SSL Config
+  # ssl_ca = "/etc/telegraf/ca.pem"
+  # ssl_cert = "/etc/telegraf/cert.pem"
+  # ssl_key = "/etc/telegraf/key.pem"
+  ## Use SSL but skip chain & host verification
+  # insecure_skip_verify = false
\ No newline at end of file

test/services/ffmpeg/telegraf_template.conf

-# Telegraf configuration
-
-# Telegraf is entirely plugin driven. All metrics are gathered from the
-# declared inputs, and sent to the declared outputs.
-
-# Plugins must be declared in here to be active.
-# To deactivate a plugin, comment out the name and any variables.
-
-# Use 'telegraf -config telegraf.conf -test' to see what metrics a config
-# file would generate.
-
-# Global tags can be specified here in key="value" format.
-[global_tags]
-  # location of the data centre
-  location="{{LOCATION}}"
-  # media service template id
-  sfc="{{SFC_ID}}"
-  # media service instance
-  sfc_i="{{SFC_ID_INSTANCE}}"
-  # service function type
-  sf="{{SF_ID}}"
-  # service function instance id
-  sf_i="{{SF_ID_INSTANCE}}"
-  # ipendpoint id aka surrogate instance
-  ipendpoint="{{IP_ENDPOINT_ID}}"
-
-# Configuration for telegraf agent
-[agent]
-  ## Default data collection interval for all inputs
-  interval = "10s"
-  ## Rounds collection interval to 'interval'
-  ## ie, if interval="10s" then always collect on :00, :10, :20, etc.
-  round_interval = true
-
-  ## Telegraf will cache metric_buffer_limit metrics for each output, and will
-  ## flush this buffer on a successful write.
-  metric_buffer_limit = 1000
-  ## Flush the buffer whenever full, regardless of flush_interval.
-  flush_buffer_when_full = true
-
-  ## Collection jitter is used to jitter the collection by a random amount.
-  ## Each plugin will sleep for a random time within jitter before collecting.
-  ## This can be used to avoid many plugins querying things like sysfs at the
-  ## same time, which can have a measurable effect on the system.
-  collection_jitter = "0s"
-
-  ## Default flushing interval for all outputs. You shouldn't set this below
-  ## interval. Maximum flush_interval will be flush_interval + flush_jitter
-  flush_interval = "10s"
-  ## Jitter the flush interval by a random amount. This is primarily to avoid
-  ## large write spikes for users running a large number of telegraf instances.
-  ## ie, a jitter of 5s and interval 10s means flushes will happen every 10-15s
-  flush_jitter = "0s"
-
-  ## Logging configuration:
-  ## Run telegraf in debug mode
-  debug = false
-  ## Run telegraf in quiet mode
-  quiet = false
-  ## Specify the log file name. The empty string means to log to stdout.
-  logfile = "/var/log/telegraf/telegraf.log"
-
-  ## Override default hostname, if empty use os.Hostname()
-  hostname = ""
-
-
-###############################################################################
-#                                  OUTPUTS                                    #
-###############################################################################
-
-# Configuration for influxdb server to send metrics to
-[[outputs.influxdb]]
-  # The full HTTP or UDP endpoint URL for your InfluxDB instance.
-  # Multiple urls can be specified but it is assumed that they are part of the same
-  # cluster, this means that only ONE of the urls will be written to each interval.
-  # urls = ["udp://127.0.0.1:8089"] # UDP endpoint example
-  urls = ["{{INFLUXDB_URL}}"] # required
-  # The target database for metrics (telegraf will create it if not exists)
-  database = "{{DATABASE_NAME}}" # required
-  # Precision of writes, valid values are "ns", "us" (or "µs"), "ms", "s", "m", "h".
-  # note: using second precision greatly helps InfluxDB compression
-  precision = "s"
-
-  ## Write timeout (for the InfluxDB client), formatted as a string.
-  ## If not provided, will default to 5s. 0s means no timeout (not recommended).
-  timeout = "5s"
-  # username = "telegraf"
-  # password = "metricsmetricsmetricsmetrics"
-  # Set the user agent for HTTP POSTs (can be useful for log differentiation)
-  # user_agent = "telegraf"
-  # Set UDP payload size, defaults to InfluxDB UDP Client default (512 bytes)
-  # udp_payload = 512
-
-
-###############################################################################
-#                                  INPUTS                                     #
-###############################################################################
-# # Influx HTTP write listener
-[[inputs.http_listener]]
-  ## Address and port to host HTTP listener on
-  service_address = ":8186"
-
-  ## timeouts
-  read_timeout = "10s"
-  write_timeout = "10s"
-
-  ## HTTPS
-  #tls_cert= "/etc/telegraf/cert.pem"
-  #tls_key = "/etc/telegraf/key.pem"
-
-  ## MTLS
-  #tls_allowed_cacerts = ["/etc/telegraf/clientca.pem"]
\ No newline at end of file

test/services/ipendpoint/telegraf_template.conf

-# Telegraf configuration
-
-# Telegraf is entirely plugin driven. All metrics are gathered from the
-# declared inputs, and sent to the declared outputs.
-
-# Plugins must be declared in here to be active.
-# To deactivate a plugin, comment out the name and any variables.
-
-# Use 'telegraf -config telegraf.conf -test' to see what metrics a config
-# file would generate.
-
-# Global tags can be specified here in key="value" format.
-[global_tags]
-  # location of the data centre
-  location="{{LOCATION}}"
-  # media service template id
-  sfc="{{SFC_ID}}"
-  # media service instance
-  sfc_i="{{SFC_ID_INSTANCE}}"
-  # service function type
-  sf="{{SF_ID}}"
-  # service function instance id
-  sf_i="{{SF_ID_INSTANCE}}"
-  # ipendpoint id aka surrogate instance
-  ipendpoint="{{IP_ENDPOINT_ID}}"
-
-# Configuration for telegraf agent
-[agent]
-  ## Default data collection interval for all inputs
-  interval = "10s"
-  ## Rounds collection interval to 'interval'
-  ## ie, if interval="10s" then always collect on :00, :10, :20, etc.
-  round_interval = true
-
-  ## Telegraf will cache metric_buffer_limit metrics for each output, and will
-  ## flush this buffer on a successful write.
-  metric_buffer_limit = 1000
-  ## Flush the buffer whenever full, regardless of flush_interval.
-  flush_buffer_when_full = true
-
-  ## Collection jitter is used to jitter the collection by a random amount.
-  ## Each plugin will sleep for a random time within jitter before collecting.
-  ## This can be used to avoid many plugins querying things like sysfs at the
-  ## same time, which can have a measurable effect on the system.
-  collection_jitter = "0s"
-
-  ## Default flushing interval for all outputs. You shouldn't set this below
-  ## interval. Maximum flush_interval will be flush_interval + flush_jitter
-  flush_interval = "10s"
-  ## Jitter the flush interval by a random amount. This is primarily to avoid
-  ## large write spikes for users running a large number of telegraf instances.
-  ## ie, a jitter of 5s and interval 10s means flushes will happen every 10-15s
-  flush_jitter = "0s"
-
-  ## Logging configuration:
-  ## Run telegraf in debug mode
-  debug = false
-  ## Run telegraf in quiet mode
-  quiet = false
-  ## Specify the log file name. The empty string means to log to stdout.
-  logfile = "/var/log/telegraf/telegraf.log"
-
-  ## Override default hostname, if empty use os.Hostname()
-  hostname = ""
-
-
-###############################################################################
-#                                  OUTPUTS                                    #
-###############################################################################
-
-# Configuration for influxdb server to send metrics to
-[[outputs.influxdb]]
-  # The full HTTP or UDP endpoint URL for your InfluxDB instance.
-  # Multiple urls can be specified but it is assumed that they are part of the same
-  # cluster, this means that only ONE of the urls will be written to each interval.
-  # urls = ["udp://127.0.0.1:8089"] # UDP endpoint example
-  urls = ["{{INFLUXDB_URL}}"] # required
-  # The target database for metrics (telegraf will create it if not exists)
-  database = "{{DATABASE_NAME}}" # required
-  # Precision of writes, valid values are "ns", "us" (or "µs"), "ms", "s", "m", "h".
-  # note: using second precision greatly helps InfluxDB compression
-  precision = "s"
-
-  ## Write timeout (for the InfluxDB client), formatted as a string.
-  ## If not provided, will default to 5s. 0s means no timeout (not recommended).
-  timeout = "5s"
-  # username = "telegraf"
-  # password = "metricsmetricsmetricsmetrics"
-  # Set the user agent for HTTP POSTs (can be useful for log differentiation)
-  # user_agent = "telegraf"
-  # Set UDP payload size, defaults to InfluxDB UDP Client default (512 bytes)
-  # udp_payload = 512
-
-
-###############################################################################
-#                                  INPUTS                                     #
-###############################################################################
-# # Influx HTTP write listener
-[[inputs.http_listener]]
-  ## Address and port to host HTTP listener on
-  service_address = ":8186"
-
-  ## timeouts
-  read_timeout = "10s"
-  write_timeout = "10s"
-
-  ## HTTPS
-  #tls_cert= "/etc/telegraf/cert.pem"
-  #tls_key = "/etc/telegraf/key.pem"
-
-  ## MTLS
-  #tls_allowed_cacerts = ["/etc/telegraf/clientca.pem"]
\ No newline at end of file

test/services/jmeter/install.sh

-#!/bin/bash
-#/////////////////////////////////////////////////////////////////////////
-#//
-#// (c) University of Southampton IT Innovation Centre, 2017
-#//
-#// Copyright in this software belongs to University of Southampton
-#// IT Innovation Centre of Gamma House, Enterprise Road,
-#// Chilworth Science Park, Southampton, SO16 7NS, UK.
-#//
-#// This software may not be used, sold, licensed, transferred, copied
-#// or reproduced in whole or in part in any manner or form or in or
-#// on any media by any person other than in accordance with the terms
-#// of the Licence Agreement supplied with the software, or otherwise
-#// without the prior written consent of the copyright owners.
-#//
-#// This software is distributed WITHOUT ANY WARRANTY, without even the
-#// implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
-#// PURPOSE, except where stated in the Licence Agreement supplied with
-#// the software.
-#//
-#//      Created By :            Michael Boniface
-#//      Created Date :          12/12/2017
-#//      Created for Project :   FLAME
-#//
-#/////////////////////////////////////////////////////////////////////////
-
-# install java 8
-add-apt-repository -y ppa:webupd8team/java
-apt-get -y update
-
-# tell the oracle java installer the license has been read
-echo debconf shared/accepted-oracle-license-v1-1 select true | debconf-set-selections
-echo debconf shared/accepted-oracle-license-v1-1 seen true | debconf-set-selections
-DEBIAN_FRONTEND=noninteractive apt-get install -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" oracle-java8-installer
-
-# install JMeter
-JMETER_VERSION="apache-jmeter-4.0"
-wget http://mirror.vorboss.net/apache//jmeter/binaries/$JMETER_VERSION.tgz -P /tmp
-tar -C /opt -xf /tmp/$JMETER_VERSION.tgz
-rm -rf /tmp/$JMETER_VERSION.tgz
-
-# Add jmeter to the path system wide
-#echo 'PATH="'$PATH':/opt/'$JMETER_VERSION'/bin"' > /etc/environment
-
-echo 'PATH="'$PATH':/opt/apache-jmeter-4.0/bin"' > /etc/environment
-source /etc/environment

test/services/mongo/telegraf_mongo.conf

+[[inputs.mongodb]]
+  ## An array of URLs of the form:
+  ##   "mongodb://" [user ":" pass "@"] host [ ":" port]
+  ## For example:
+  ##   mongodb://user:auth_key@10.10.3.30:27017,
+  ##   mongodb://10.10.3.33:18832,
+  servers = ["mongodb://127.0.0.1:27017"]
+  gather_perdb_stats = false
+
+  ## Optional SSL Config
+  # ssl_ca = "/etc/telegraf/ca.pem"
+  # ssl_cert = "/etc/telegraf/cert.pem"
+  # ssl_key = "/etc/telegraf/key.pem"
+  ## Use SSL but skip chain & host verification
+  # insecure_skip_verify = false
\ No newline at end of file

test/services/mongo/telegraf_template.conf

-# Telegraf configuration
-
-# Telegraf is entirely plugin driven. All metrics are gathered from the
-# declared inputs, and sent to the declared outputs.
-
-# Plugins must be declared in here to be active.
-# To deactivate a plugin, comment out the name and any variables.
-
-# Use 'telegraf -config telegraf.conf -test' to see what metrics a config
-# file would generate.
-
-# Global tags can be specified here in key="value" format.
-[global_tags]
-  # location of the data centre
-  location="{{LOCATION}}"
-  # media service template id
-  sfc="{{SFC_ID}}"
-  # media service instance
-  sfc_i="{{SFC_ID_INSTANCE}}"
-  # service function type
-  sf="{{SF_ID}}"
-  # service function instance id
-  sf_i="{{SF_ID_INSTANCE}}"
-  # ipendpoint id aka surrogate instance
-  ipendpoint="{{IP_ENDPOINT_ID}}"
-
-# Configuration for telegraf agent
-[agent]
-  ## Default data collection interval for all inputs
-  interval = "10s"
-  ## Rounds collection interval to 'interval'
-  ## ie, if interval="10s" then always collect on :00, :10, :20, etc.
-  round_interval = true
-
-  ## Telegraf will cache metric_buffer_limit metrics for each output, and will
-  ## flush this buffer on a successful write.
-  metric_buffer_limit = 1000
-  ## Flush the buffer whenever full, regardless of flush_interval.
-  flush_buffer_when_full = true
-
-  ## Collection jitter is used to jitter the collection by a random amount.
-  ## Each plugin will sleep for a random time within jitter before collecting.
-  ## This can be used to avoid many plugins querying things like sysfs at the
-  ## same time, which can have a measurable effect on the system.
-  collection_jitter = "0s"
-
-  ## Default flushing interval for all outputs. You shouldn't set this below
-  ## interval. Maximum flush_interval will be flush_interval + flush_jitter
-  flush_interval = "10s"
-  ## Jitter the flush interval by a random amount. This is primarily to avoid
-  ## large write spikes for users running a large number of telegraf instances.
-  ## ie, a jitter of 5s and interval 10s means flushes will happen every 10-15s
-  flush_jitter = "0s"
-
-  ## Logging configuration:
-  ## Run telegraf in debug mode
-  debug = false
-  ## Run telegraf in quiet mode
-  quiet = false
-  ## Specify the log file name. The empty string means to log to stdout.
-  logfile = "/var/log/telegraf/telegraf.log"
-
-  ## Override default hostname, if empty use os.Hostname()
-  hostname = ""
-
-
-###############################################################################
-#                                  OUTPUTS                                    #
-###############################################################################
-
-# Configuration for influxdb server to send metrics to
-[[outputs.influxdb]]
-  # The full HTTP or UDP endpoint URL for your InfluxDB instance.
-  # Multiple urls can be specified but it is assumed that they are part of the same
-  # cluster, this means that only ONE of the urls will be written to each interval.
-  # urls = ["udp://127.0.0.1:8089"] # UDP endpoint example
-  urls = ["{{INFLUXDB_URL}}"] # required
-  # The target database for metrics (telegraf will create it if not exists)
-  database = "{{DATABASE_NAME}}" # required
-  # Precision of writes, valid values are "ns", "us" (or "µs"), "ms", "s", "m", "h".
-  # note: using second precision greatly helps InfluxDB compression
-  precision = "s"
-
-  ## Write timeout (for the InfluxDB client), formatted as a string.
-  ## If not provided, will default to 5s. 0s means no timeout (not recommended).
-  timeout = "5s"
-  # username = "telegraf"
-  # password = "metricsmetricsmetricsmetrics"
-  # Set the user agent for HTTP POSTs (can be useful for log differentiation)
-  # user_agent = "telegraf"
-  # Set UDP payload size, defaults to InfluxDB UDP Client default (512 bytes)
-  # udp_payload = 512
-
-
-###############################################################################
-#                                  INPUTS                                     #
-###############################################################################
-# # Influx HTTP write listener
-[[inputs.http_listener]]
-  ## Address and port to host HTTP listener on
-  service_address = ":8186"
-
-  ## timeouts
-  read_timeout = "10s"
-  write_timeout = "10s"
-
-  ## HTTPS
-  #tls_cert= "/etc/telegraf/cert.pem"
-  #tls_key = "/etc/telegraf/key.pem"
-
-  ## MTLS
-  #tls_allowed_cacerts = ["/etc/telegraf/clientca.pem"]
-
-[[inputs.mongodb]]
-  ## An array of URLs of the form:
-  ##   "mongodb://" [user ":" pass "@"] host [ ":" port]
-  ## For example:
-  ##   mongodb://user:auth_key@10.10.3.30:27017,
-  ##   mongodb://10.10.3.33:18832,
-  servers = ["mongodb://127.0.0.1:27017"]
-  gather_perdb_stats = false
-
-  ## Optional SSL Config
-  # ssl_ca = "/etc/telegraf/ca.pem"
-  # ssl_cert = "/etc/telegraf/cert.pem"
-  # ssl_key = "/etc/telegraf/key.pem"
-  ## Use SSL but skip chain & host verification
-  # insecure_skip_verify = false
\ No newline at end of file

test/services/nginx/install.sh

@@ -32,18 +32,4 @@ yes Y | apt-get install nginx
 # http://nginx.org/en/docs/http/ngx_http_stub_status_module.html
 cp -rf $REPO_ROOT/test/services/nginx/nginx.conf /etc/nginx/nginx.conf
 nginx -s reload
-systemctl start nginx
-
-# Copy in the test video, this needs to be part of a test set up and not here.
-
-TEST_VIDEO="20180212104221flame-project-full.mp4"
-TEST_VIDEO_ARCHIVE=$TEST_VIDEO".gz"
-DEST_DIR="/usr/share/nginx/html" 
-DEST_FILE=$DEST_DIR"/"$TEST_VIDEO_ARCHIVE
-
-# Copy files for MPEG-DASH testing
-curl "ftp://ftp.it-innovation.soton.ac.uk/testdata/video/"$TEST_VIDEO_ARCHIVE --user flame-rw:DR8ngj3ogSjd8gl -o $DEST_FILE
-tar -xvf $DEST_FILE -C $DEST_DIR
-
-rm -rf $DEST_FILE
-mv $DEST_DIR"/"$TEST_VIDEO $DEST_DIR"/"test_video
\ No newline at end of file
+systemctl start nginx
\ No newline at end of file

test/services/nginx/telegraf_nginx.conf

+# Read Nginx's basic status information (ngx_http_stub_status_module)
+[[inputs.nginx]]
+  ## An array of Nginx stub_status URI to gather stats.
+  urls = ["http://localhost:80/nginx_status"]
+
+  ## HTTP response timeout (default: 5s)
+#  response_timeout = "5s"
\ No newline at end of file

test/services/nginx/telegraf_template.conf

-# Telegraf configuration
-
-# Telegraf is entirely plugin driven. All metrics are gathered from the
-# declared inputs, and sent to the declared outputs.
-
-# Plugins must be declared in here to be active.
-# To deactivate a plugin, comment out the name and any variables.
-
-# Use 'telegraf -config telegraf.conf -test' to see what metrics a config
-# file would generate.
-
-# Global tags can be specified here in key="value" format.
-[global_tags]
-  # location of the data centre
-  location="{{LOCATION}}"
-  # media service template id
-  sfc="{{SFC_ID}}"
-  # media service instance
-  sfc_i="{{SFC_ID_INSTANCE}}"
-  # service function type
-  sf="{{SF_ID}}"
-  # service function instance id
-  sf_i="{{SF_ID_INSTANCE}}"
-  # ipendpoint id aka surrogate instance
-  ipendpoint="{{IP_ENDPOINT_ID}}"
-
-# Configuration for telegraf agent
-[agent]
-  ## Default data collection interval for all inputs
-  interval = "10s"
-  ## Rounds collection interval to 'interval'
-  ## ie, if interval="10s" then always collect on :00, :10, :20, etc.
-  round_interval = true
-
-  ## Telegraf will cache metric_buffer_limit metrics for each output, and will
-  ## flush this buffer on a successful write.
-  metric_buffer_limit = 1000
-  ## Flush the buffer whenever full, regardless of flush_interval.
-  flush_buffer_when_full = true
-
-  ## Collection jitter is used to jitter the collection by a random amount.
-  ## Each plugin will sleep for a random time within jitter before collecting.
-  ## This can be used to avoid many plugins querying things like sysfs at the
-  ## same time, which can have a measurable effect on the system.
-  collection_jitter = "0s"
-
-  ## Default flushing interval for all outputs. You shouldn't set this below
-  ## interval. Maximum flush_interval will be flush_interval + flush_jitter
-  flush_interval = "10s"
-  ## Jitter the flush interval by a random amount. This is primarily to avoid
-  ## large write spikes for users running a large number of telegraf instances.
-  ## ie, a jitter of 5s and interval 10s means flushes will happen every 10-15s
-  flush_jitter = "0s"
-
-  ## Logging configuration:
-  ## Run telegraf in debug mode
-  debug = false
-  ## Run telegraf in quiet mode
-  quiet = false
-  ## Specify the log file name. The empty string means to log to stdout.
-  logfile = "/etc/telegraf/telegraf.log"
-
-  ## Override default hostname, if empty use os.Hostname()
-  hostname = ""
-
-
-###############################################################################
-#                                  OUTPUTS                                    #
-###############################################################################
-
-# Configuration for influxdb server to send metrics to
-[[outputs.influxdb]]
-  # The full HTTP or UDP endpoint URL for your InfluxDB instance.
-  # Multiple urls can be specified but it is assumed that they are part of the same
-  # cluster, this means that only ONE of the urls will be written to each interval.
-  # urls = ["udp://127.0.0.1:8089"] # UDP endpoint example
-  urls = ["{{INFLUXDB_URL}}"] # required
-  # The target database for metrics (telegraf will create it if not exists)
-  database = "{{DATABASE_NAME}}" # required
-  # Precision of writes, valid values are "ns", "us" (or "µs"), "ms", "s", "m", "h".
-  # note: using second precision greatly helps InfluxDB compression
-  precision = "s"
-
-  ## Write timeout (for the InfluxDB client), formatted as a string.
-  ## If not provided, will default to 5s. 0s means no timeout (not recommended).
-  timeout = "5s"
-  # username = "telegraf"
-  # password = "metricsmetricsmetricsmetrics"
-  # Set the user agent for HTTP POSTs (can be useful for log differentiation)
-  # user_agent = "telegraf"
-  # Set UDP payload size, defaults to InfluxDB UDP Client default (512 bytes)
-  # udp_payload = 512
-
-###############################################################################
-#                                  INPUTS                                     #
-###############################################################################
-# # Read metrics about network interface usage
- [[inputs.net]]
-#   ## By default, telegraf gathers stats from any up interface (excluding loopback)
-#   ## Setting interfaces will tell it to gather these explicit interfaces,
-#   ## regardless of status.
-#   ##
-#   # interfaces = ["eth0"]
-
-# Read metrics about cpu usage
-[[inputs.cpu]]
-  ## Whether to report per-cpu stats or not
-  percpu = true
-  ## Whether to report total system cpu stats or not
-  totalcpu = true
-  ## If true, collect raw CPU time metrics.
-  collect_cpu_time = false
-  ## If true, compute and report the sum of all non-idle CPU states.
- #report_active = false
-
-
-# Read metrics about disk usage by mount point
-[[inputs.disk]]
-  ## By default, telegraf gather stats for all mountpoints.
-  ## Setting mountpoints will restrict the stats to the specified mountpoints.
-  # mount_points = ["/"]
-
-  ## Ignore some mountpoints by filesystem type. For example (dev)tmpfs (usually
-  ## present on /run, /var/run, /dev/shm or /dev).
-  ignore_fs = ["tmpfs", "devtmpfs", "devfs"]
-
-
-# Read metrics about disk IO by device
-[[inputs.diskio]]
-  ## By default, telegraf will gather stats for all devices including
-  ## disk partitions.
-  ## Setting devices will restrict the stats to the specified devices.
-  # devices = ["sda", "sdb"]
-  ## Uncomment the following line if you need disk serial numbers.
-  # skip_serial_number = false
-  #
-  ## On systems which support it, device metadata can be added in the form of
-  ## tags.
-  ## Currently only Linux is supported via udev properties. You can view
-  ## available properties for a device by running:
-  ## 'udevadm info -q property -n /dev/sda'
-  # device_tags = ["ID_FS_TYPE", "ID_FS_USAGE"]
-  #
-  ## Using the same metadata source as device_tags, you can also customize the
-  ## name of the device via templates.
-  ## The 'name_templates' parameter is a list of templates to try and apply to
-  ## the device. The template may contain variables in the form of '$PROPERTY' or
-  ## '${PROPERTY}'. The first template which does not contain any variables not
-  ## present for the device is used as the device name tag.
-  ## The typical use case is for LVM volumes, to get the VG/LV name instead of
-  ## the near-meaningless DM-0 name.
-  # name_templates = ["$ID_FS_LABEL","$DM_VG_NAME/$DM_LV_NAME"]
-
-# Read metrics about memory usage
-[[inputs.mem]]
-  # no configuration
-
-# # Influx HTTP write listener
-[[inputs.http_listener]]
-  ## Address and port to host HTTP listener on
-  service_address = ":8186"
-
-  ## timeouts
-  read_timeout = "10s"
-  write_timeout = "10s"
-
-# Read Nginx's basic status information (ngx_http_stub_status_module)
-[[inputs.nginx]]
-  ## An array of Nginx stub_status URI to gather stats.
-  urls = ["http://localhost:80/nginx_status"]
-
-  ## HTTP response timeout (default: 5s)
-#  response_timeout = "5s"
\ No newline at end of file

test/services/pytest/install.sh

@@ -24,5 +24,6 @@
 #//
 #/////////////////////////////////////////////////////////////////////////
 apt-get update
-apt-get -y install python-pip python-dev build-essential
+apt-get install python3 python3-pip
+update-alternatives --install /usr/bin/python python /usr/bin/python3 10
 pip install pytest pyyaml
\ No newline at end of file

test/services/vlc/telegraf_template.conf

-# Telegraf configuration
-
-# Telegraf is entirely plugin driven. All metrics are gathered from the
-# declared inputs, and sent to the declared outputs.
-
-# Plugins must be declared in here to be active.
-# To deactivate a plugin, comment out the name and any variables.
-
-# Use 'telegraf -config telegraf.conf -test' to see what metrics a config
-# file would generate.
-
-# Global tags can be specified here in key="value" format.
-[global_tags]
-  # location of the data centre
-  location="{{LOCATION}}"
-  # media service template id
-  sfc="{{SFC_ID}}"
-  # media service instance
-  sfc_i="{{SFC_ID_INSTANCE}}"
-  # service function type
-  sf="{{SF_ID}}"
-  # service function instance id
-  sf_i="{{SF_ID_INSTANCE}}"
-  # ipendpoint id aka surrogate instance
-  ipendpoint="{{IP_ENDPOINT_ID}}"
-
-# Configuration for telegraf agent
-[agent]
-  ## Default data collection interval for all inputs
-  interval = "10s"
-  ## Rounds collection interval to 'interval'
-  ## ie, if interval="10s" then always collect on :00, :10, :20, etc.
-  round_interval = true
-
-  ## Telegraf will cache metric_buffer_limit metrics for each output, and will
-  ## flush this buffer on a successful write.
-  metric_buffer_limit = 1000
-  ## Flush the buffer whenever full, regardless of flush_interval.
-  flush_buffer_when_full = true
-
-  ## Collection jitter is used to jitter the collection by a random amount.
-  ## Each plugin will sleep for a random time within jitter before collecting.
-  ## This can be used to avoid many plugins querying things like sysfs at the
-  ## same time, which can have a measurable effect on the system.
-  collection_jitter = "0s"
-
-  ## Default flushing interval for all outputs. You shouldn't set this below
-  ## interval. Maximum flush_interval will be flush_interval + flush_jitter
-  flush_interval = "10s"
-  ## Jitter the flush interval by a random amount. This is primarily to avoid
-  ## large write spikes for users running a large number of telegraf instances.
-  ## ie, a jitter of 5s and interval 10s means flushes will happen every 10-15s
-  flush_jitter = "0s"
-
-  ## Logging configuration:
-  ## Run telegraf in debug mode
-  debug = false
-  ## Run telegraf in quiet mode
-  quiet = false
-  ## Specify the log file name. The empty string means to log to stdout.
-  logfile = "/var/log/telegraf/telegraf.log"
-
-  ## Override default hostname, if empty use os.Hostname()
-  hostname = ""
-
-
-###############################################################################
-#                                  OUTPUTS                                    #
-###############################################################################
-
-# Configuration for influxdb server to send metrics to
-[[outputs.influxdb]]
-  # The full HTTP or UDP endpoint URL for your InfluxDB instance.
-  # Multiple urls can be specified but it is assumed that they are part of the same
-  # cluster, this means that only ONE of the urls will be written to each interval.
-  # urls = ["udp://127.0.0.1:8089"] # UDP endpoint example
-  urls = ["{{INFLUXDB_URL}}"] # required
-  # The target database for metrics (telegraf will create it if not exists)
-  database = "{{DATABASE_NAME}}" # required
-  # Precision of writes, valid values are "ns", "us" (or "µs"), "ms", "s", "m", "h".
-  # note: using second precision greatly helps InfluxDB compression
-  precision = "s"
-
-  ## Write timeout (for the InfluxDB client), formatted as a string.
-  ## If not provided, will default to 5s. 0s means no timeout (not recommended).
-  timeout = "5s"
-  # username = "telegraf"
-  # password = "metricsmetricsmetricsmetrics"
-  # Set the user agent for HTTP POSTs (can be useful for log differentiation)
-  # user_agent = "telegraf"
-  # Set UDP payload size, defaults to InfluxDB UDP Client default (512 bytes)
-  # udp_payload = 512
-
-
-###############################################################################
-#                                  INPUTS                                     #
-###############################################################################
-# # Influx HTTP write listener
-[[inputs.http_listener]]
-  ## Address and port to host HTTP listener on
-  service_address = ":8186"
-
-  ## timeouts
-  read_timeout = "10s"
-  write_timeout = "10s"
-
-  ## HTTPS
-  #tls_cert= "/etc/telegraf/cert.pem"
-  #tls_key = "/etc/telegraf/key.pem"
-
-  ## MTLS
-  #tls_allowed_cacerts = ["/etc/telegraf/clientca.pem"]
\ No newline at end of file

test/streaming/setup.sh

@@ -47,4 +47,22 @@ curl -i -X POST -H "Content-Type: application/json" http://localhost:8888/chrono
 curl -i -X POST -H "Content-Type: application/json" http://localhost:9092/kapacitor/v1/tasks -d @$TEST_DIR/rules.json
 
 # Set up dashboard
-curl -i -X POST -H "Content-Type: application/json" http://localhost:8888/chronograf/v1/dashboards -d @$TEST_DIR/dashboard.json
\ No newline at end of file
+curl -i -X POST -H "Content-Type: application/json" http://localhost:8888/chronograf/v1/dashboards -d @$TEST_DIR/dashboard.json
+
+# Copy test data
+# Apache
+# DEST_DIR="/var/www/html/" 
+
+# NGINX
+DEST_DIR="/usr/share/nginx/html" 
+
+TEST_VIDEO="20180212104221flame-project-full.mp4"
+TEST_VIDEO_ARCHIVE=$TEST_VIDEO".gz"
+DEST_FILE=$DEST_DIR"/"$TEST_VIDEO_ARCHIVE
+
+# Copy files for MPEG-DASH testing
+curl "ftp://ftp.it-innovation.soton.ac.uk/testdata/video/"$TEST_VIDEO_ARCHIVE --user flame-rw:DR8ngj3ogSjd8gl -o $DEST_FILE
+tar -xvf $DEST_FILE -C $DEST_DIR
+
+rm -rf $DEST_FILE
+mv $DEST_DIR"/"$TEST_VIDEO $DEST_DIR"/"test_video
\ No newline at end of file

test/telegraf-agents/__init__.py

+#!/usr/bin/python3
\ No newline at end of file

test/telegraf-agents/conftest.py

+#!/usr/bin/python3
+
+import pytest
+import yaml
+
+@pytest.fixture(scope="module")
+def telegraf_agent_config():
+    """Returns the service configuration deployed for the telegraf conf tests."""
+    with open("test/telegraf-agents/rspec.yml", 'r') as stream:
+        data_loaded = yaml.load(stream)
+    return data_loaded
\ No newline at end of file