Added test for configuring telegraf

Vagrantfile

@@ -76,9 +76,11 @@ Vagrant.configure("2") do |config|
 
       # Port forwarding
       puts "Forwarding the following specified ports for #{host["name"]}:"
-      host['forward_ports'].each do |port|
-        puts "Forwarding guest:#{port["guest"]} => host:#{port["host"]}"
-        instance_config.vm.network "forwarded_port", guest: port["guest"], host: port["host"]
+      if host.has_key? 'forward_ports'
+        host['forward_ports'].each do |port|
+          puts "Forwarding guest:#{port["guest"]} => host:#{port["host"]}"
+          instance_config.vm.network "forwarded_port", guest: port["guest"], host: port["host"]
+        end
       end
       
       # Switch case added here to make clmc-service provisioning simple without having to have a complex rspec.yml file
@@ -86,6 +88,8 @@ Vagrant.configure("2") do |config|
       
       puts "Instance name #{instance_name}:"
       case instance_name
+        when 'test-runner'
+          instance_config.vm.provision :shell, :path => "test/services/pytest/install.sh"
         when 'clmc-service'
           instance_config.vm.provision :shell, :path => "scripts/clmc-service/install.sh"
           instance_config.vm.provision :shell, :path => "test/services/pytest/install.sh"         
@@ -100,10 +104,12 @@ Vagrant.configure("2") do |config|
           instance_config.vm.provision :shell, inline: "cp /vagrant/test/services/#{host["service_name"]}/telegraf_#{host["service_name"]}.conf /etc/telegraf/telegraf.d/"     
 
           # CLMC agent general and output configuration
-          instance_config.vm.provision :shell, :path => "scripts/clmc-agent/configure.sh", :args => "/vagrant/scripts/clmc-agent/telegraf.conf /vagrant/scripts/clmc-agent/telegraf_output.conf #{host["location"]} #{host["sfc_id"]} #{host["sfc_id_instance"]} #{host["sf_id"]} #{host["sf_id_instance"]} #{host["ipendpoint_id"]} #{host["influxdb_url"]} #{host["database_name"]}"  
+          instance_config.vm.provision :shell, :path => "scripts/clmc-agent/write_config.sh"
+
+          instance_config.vm.provision :shell, :path => "scripts/clmc-agent/configure.sh", :args => "#{host["location"]} #{host["sfc_id"]} #{host["sfc_id_instance"]} #{host["sf_id"]} #{host["sf_id_instance"]} #{host["ipendpoint_id"]} #{host["influxdb_url"]} #{host["database_name"]}"  
 
           # CLMC start agent
-          instance_config.vm.provision :shell, inline: "systemctl start telegraf"            
+          instance_config.vm.provision :shell, inline: "service telegraf restart"            
       end
 
       

scripts/clmc-agent/configure.sh

@@ -30,64 +30,25 @@ set -euo pipefail
 echo "Configuring Telegraf agent general and output configuration"
 
 # Get command line parameters
-if [ "$#" -ne 10 ]; then
+if [ "$#" -ne 8 ]; then
     echo "Error: illegal number of arguments: "$#
-      echo "Usage: config.sh TELEGRAF_GENERAL_CONF_FILE TELEGRAF_OUTPUT_CONF_FILE LOCATION SFC_ID SFC_ID_INSTANCE SF_ID SF_ID_INSTANCE IP_ENDPOINT_ID INFLUXDB_URL DATABASE_NAME"
-      exit 
+      echo "Usage: configure.sh LOCATION SFC_ID SFC_ID_INSTANCE SF_ID SF_ID_INSTANCE IP_ENDPOINT_ID INFLUXDB_URL DATABASE_NAME"
+      exit 1 
 fi
 
-TELEGRAF_GENERAL_CONF_FILE=$1
-TELEGRAF_OUTPUT_CONF_FILE=$2
-LOCATION=$3
-SFC_ID=$4
-SFC_ID_INSTANCE=$5
-SF_ID=$6
-SF_ID_INSTANCE=$7
-IP_ENDPOINT_ID=$8
-INFLUXDB_URL=$9
-DATABASE_NAME=${10}
+LOCATION=$1
+SFC_ID=$2
+SFC_ID_INSTANCE=$3
+SF_ID=$4
+SF_ID_INSTANCE=$5
+IP_ENDPOINT_ID=$6
+INFLUXDB_URL=$7
+DATABASE_NAME=$8
 
 TELEGRAF_CONF_DIR="/etc/telegraf"
-TELEGRAF_CONF_FILE="$TELEGRAF_CONF_DIR/telegraf.conf"
-TELEGRAF_INCLUDE_CONF_DIR="/etc/telegraf/telegraf.d"
-TELEGRAF_OUTPUT_FILE="$TELEGRAF_INCLUDE_CONF_DIR/telegraf_output.conf"
-
-echo "Checking Telegraf installation"
-
-# Check the configuration files provided exist
-
-echo "TELEGRAF_GENERAL_CONF_FILE: "$TELEGRAF_GENERAL_CONF_FILE
-if [ ! -f $TELEGRAF_GENERAL_CONF_FILE ]; then
-    echo "Error: Telegraf conf template file not found: "$TELEGRAF_GENERAL_CONF_FILE
-    exit
-fi
-
-echo "TELEGRAF_OUTPUT_CONF_FILE: "$TELEGRAF_OUTPUT_CONF_FILE
-if [ ! -f $TELEGRAF_OUTPUT_CONF_FILE ]; then
-    echo "Error: Telegraf output conf specific file not found: "$TELEGRAF_OUTPUT_CONF_FILE
-    exit
-fi
-
-# Check the target telegraf directory exists
-if [ ! -d "$TELEGRAF_CONF_DIR" ]; then
-    echo "Error: Telegraf conf directory does not exist on target machine. Check that telegraf is installed "$TELEGRAF_CONF_DIR
-    exit
-fi
-
-# Check the target telegraf directory exists
-if [ ! -d $TELEGRAF_INCLUDE_CONF_DIR ]; then
-    echo "Error: Telegraf conf include directory does not exist on target machine. Check that telegraf is installed "$TELEGRAF_INCLUDE_CONF_DIR
-    exit
-fi
-
-echo "Copying configuration"
-
-# Copy configuration
-echo "Telegraf general config file: " $TELEGRAF_GENERAL_CONF_FILE
-cp $TELEGRAF_GENERAL_CONF_FILE $TELEGRAF_CONF_FILE
-
-echo "Telegraf output config file: " $TELEGRAF_OUTPUT_CONF_FILE
-cp $TELEGRAF_OUTPUT_CONF_FILE $TELEGRAF_OUTPUT_FILE
+TELEGRAF_CONF_FILE=$TELEGRAF_CONF_DIR"/telegraf.conf"
+TELEGRAF_INCLUDE_CONF_DIR=$TELEGRAF_CONF_DIR"/telegraf.d"
+TELEGRAF_OUTPUT_CONF_FILE=$TELEGRAF_INCLUDE_CONF_DIR"/telegraf_output.conf"
 
 # Replace template parameters on general configuration
 sed -i 's/$LOCATION/'$LOCATION'/g' $TELEGRAF_CONF_FILE
@@ -97,12 +58,6 @@ sed -i 's/$SF_ID/'$SF_ID'/g' /etc/telegraf/telegraf.conf
 sed -i 's/$SF_ID_INSTANCE}}/'$SF_ID_INSTANCE'/g' $TELEGRAF_CONF_FILE
 sed -i 's/$IP_ENDPOINT_ID/'$IP_ENDPOINT_ID'/g' $TELEGRAF_CONF_FILE
 
-# Replace parameters on output configuration
-echo "INFLUXDB_URL: " $INFLUXDB_URL
-echo "DATABASE_NAME: " $DATABASE_NAME
-
+# Replace parameters in output configuration file
+sed -i 's|$INFLUXDB_URL|'$INFLUXDB_URL'|g' $TELEGRAF_OUTPUT_CONF_FILE
+sed -i 's/$DATABASE_NAME/'$DATABASE_NAME'/g' $TELEGRAF_OUTPUT_CONF_FILE
\ No newline at end of file
-sed -i 's|$INFLUXDB_URL|'$INFLUXDB_URL'|g' $TELEGRAF_OUTPUT_FILE
-sed -i 's/$DATABASE_NAME/'$DATABASE_NAME'/g' $TELEGRAF_OUTPUT_FILE
-
-# Start telegraf
-systemctl restart telegraf

scripts/clmc-agent/write_config.sh

+#!/bin/bash
+#/////////////////////////////////////////////////////////////////////////
+#//
+#// (c) University of Southampton IT Innovation Centre, 2017
+#//
+#// Copyright in this software belongs to University of Southampton
+#// IT Innovation Centre of Gamma House, Enterprise Road,
+#// Chilworth Science Park, Southampton, SO16 7NS, UK.
+#//
+#// This software may not be used, sold, licensed, transferred, copied
+#// or reproduced in whole or in part in any manner or form or in or
+#// on any media by any person other than in accordance with the terms
+#// of the Licence Agreement supplied with the software, or otherwise
+#// without the prior written consent of the copyright owners.
+#//
+#// This software is distributed WITHOUT ANY WARRANTY, without even the
+#// implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+#// PURPOSE, except where stated in the Licence Agreement supplied with
+#// the software.
+#//
+#//      Created By :            Michael Boniface
+#//      Created Date :          19/03/2018
+#//      Created for Project :   FLAME
+#//
+#/////////////////////////////////////////////////////////////////////////
+
+TELEGRAF_CONF_DIR="/etc/telegraf"
+TELEGRAF_CONF_FILE=$TELEGRAF_CONF_DIR"/telegraf.conf"
+TELEGRAF_INCLUDE_CONF_DIR=$TELEGRAF_CONF_DIR"/telegraf.d"
+TELEGRAF_OUTPUT_CONF_FILE=$TELEGRAF_INCLUDE_CONF_DIR"/telegraf_output.conf"
+
+echo "Checking Telegraf installation"
+
+# Check the target telegraf directory exists
+if [ ! -d "$TELEGRAF_CONF_DIR" ]; then
+    echo "Error: Telegraf conf directory does not exist on target machine. Check that telegraf is installed "$TELEGRAF_CONF_DIR
+    exit 1
+fi
+
+# Check the target telegraf directory exists
+if [ ! -d $TELEGRAF_INCLUDE_CONF_DIR ]; then
+    echo "Error: Telegraf conf include directory does not exist on target machine. Check that telegraf is installed "$TELEGRAF_INCLUDE_CONF_DIR
+    exit 1
+fi
+
+# Copy configuration
+echo "Telegraf general config file: " $TELEGRAF_CONF_FILE
+(cat <<'EOF'
+[global_tags]
+  location="$LOCATION"
+  sfc="$SFC_ID"
+  sfc_i="$SFC_ID_INSTANCE"
+  sf="$SF_ID"
+  sf_i="$SF_ID_INSTANCE"
+  ipendpoint="$IP_ENDPOINT_ID"
+[agent]
+  interval = "10s"
+  round_interval = true
+  metric_buffer_limit = 1000
+  flush_buffer_when_full = true
+  collection_jitter = "0s"
+  flush_interval = "10s"
+  flush_jitter = "0s"
+  debug = false
+  quiet = false
+  logfile = "/var/log/telegraf/telegraf.log"
+  hostname = ""
+EOF
+) > $TELEGRAF_CONF_FILE  
+
+echo "Telegraf output config file: " $TELEGRAF_OUTPUT_CONF_FILE
+(cat <<'EOF'
+[[outputs.influxdb]]
+  urls = ["$INFLUXDB_URL"]
+  database = "$DATABASE_NAME"
+  precision = "s"
+  timeout = "5s"
+EOF
+) > $TELEGRAF_OUTPUT_CONF_FILE

test/scripts/rspec.yml

+hosts:
+  - name: test-runner
+    cpus: 1
+    memory: 2048
+    disk: "10GB"
+    ip_address: "192.168.50.10"
+    
\ No newline at end of file

test/scripts/test_config_telegraf.py

+#!/usr/bin/python3
+
+import pytest
+import subprocess
+
+def test_write_telegraf_conf():
+  TELEGRAF_CONF_DIR="/etc/telegraf"
+  LOCATION="DC1"
+  SFC_ID="media_service_A"
+  SFC_ID_INSTANCE="media_service_A_instance"
+  SF_ID="streaming_service"
+  SF_ID_INSTANCE="streaming_service_instance"
+  IP_ENDPOINT_ID="endpoint"
+  INFLUXDB_URL="http://172.29.236.10"
+  DATABASE_NAME="experimentation_database"  
+
+  try:
+    # run with no telegraf conf directory
+    (out, err, code) = run_command('sudo /vagrant/scripts/clmc-agent/write_config.sh')
+    assert code == 1, "Failed to catch error of no telegraf configuration directory : " + str(code) + ", cmd=" + cmd
+
+    # mk telegraf conf directory
+    run_command("sudo mkdir -p /etc/telegraf")
+
+    # run with no telegraf.d directory
+    (out, err, code) = run_command('sudo /vagrant/scripts/clmc-agent/write_config.sh')
+    assert code == 1, "Failed to catch error of no telegraf include directory : " + str(code) + ", cmd=" + cmd
+
+    # mk telegraf.d directory
+    run_command("sudo mkdir -p /etc/telegraf/telegraf.d")    
+
+    # write configuration to file
+    (out, err, code) = run_command('sudo /vagrant/scripts/clmc-agent/write_config.sh')
+    assert code == 0, "Failed to write configuration files : " + str(code) + ", cmd=" + cmd
+
+    # run with incorrect arguments
+    cmd = 'sudo /vagrant/scripts/clmc-agent/configure.sh' 
+    (out, err, code) = run_command(cmd)
+    assert code == 1, "Failed to return error on incorrect arguments : " + str(code) + ", cmd=" + cmd  
+
+    cmd = 'sudo /vagrant/scripts/clmc-agent/configure.sh ' + LOCATION + ' ' + SFC_ID + ' ' + SFC_ID_INSTANCE + ' ' + SF_ID + ' ' + SF_ID_INSTANCE + ' ' + IP_ENDPOINT_ID + ' ' + INFLUXDB_URL + ' ' + DATABASE_NAME
+    # run everything setup 
+    (out, err, code) = run_command(cmd)
+    assert code == 0, "Configure command returned error, output=" + str(out) + ", cmd=" + cmd
+
+    try:
+        TELEGRAF_GENERAL_CONF_FILE = TELEGRAF_CONF_DIR + "/telegraf.d/telegraf_output.conf"
+        with open(TELEGRAF_GENERAL_CONF_FILE) as general_conf:
+          lines = general_conf.read()
+          assert lines.find(LOCATION), "Cannot find location" 
+          assert lines.find(SFC_ID), "Cannot find sfc_id"
+          assert lines.find(SFC_ID_INSTANCE), "Cannot find sfc_id_instance"  
+          assert lines.find(SF_ID), "Cannot find sfc_id"            
+          assert lines.find(SF_ID_INSTANCE), "Cannot find sf_id_instance"
+          assert lines.find(IP_ENDPOINT_ID), "Cannot find location"                      
+    except FileNotFoundError:
+        assert False, "Telegraf general conf file not found, " + TELEGRAF_GENERAL_CONF_FILE
+
+    try:
+        TELEGRAF_OUTPUT_CONF_FILE = TELEGRAF_CONF_DIR + "/telegraf.conf"
+        with open(TELEGRAF_OUTPUT_CONF_FILE) as output_conf:
+          lines = output_conf.read()
+          assert lines.find(INFLUXDB_URL), "Cannot find influx_db" 
+          assert lines.find(DATABASE_NAME), "Cannot find database"                    
+    except FileNotFoundError:
+        assert False, "Telegraf output conf file not found, " + TELEGRAF_OUTPUT_CONF_FILE
+
+  finally:
+      # clean up telegraf after test
+      #run_command("sudo rm -rf /etc/telegraf")
+      print ("finally")
+
+# wrapper for executing commands on the cli, returning (std_out, std_error, process_return_code)
+def run_command(cmd):
+    """Run a shell command.
+
+    Arguments:
+        cmd {string} -- command to run in the shell
+
+    Returns:
+        stdout, stderr, exit code -- tuple of the process's stdout, stderr and exit code (0 on success)
+    """
+    proc = subprocess.Popen([cmd], stdout=subprocess.PIPE, shell=True)
+    out, err = proc.communicate()
+    return_code = proc.returncode
+    return out, err, return_code
+

test/services/ffmpeg/telegraf_ffmpeg.conf

+# # Influx HTTP write listener
+[[inputs.http_listener]]
+  ## Address and port to host HTTP listener on
+  service_address = ":8186"
+
+  ## timeouts
+  read_timeout = "10s"
+  write_timeout = "10s"
+
+  ## HTTPS
+  #tls_cert= "/etc/telegraf/cert.pem"
+  #tls_key = "/etc/telegraf/key.pem"
+
+  ## MTLS
+  #tls_allowed_cacerts = ["/etc/telegraf/clientca.pem"]
\ No newline at end of file

test/services/host/install.sh

+#!/bin/bash
+#/////////////////////////////////////////////////////////////////////////
+#//
+#// (c) University of Southampton IT Innovation Centre, 2017
+#//
+#// Copyright in this software belongs to University of Southampton
+#// IT Innovation Centre of Gamma House, Enterprise Road,
+#// Chilworth Science Park, Southampton, SO16 7NS, UK.
+#//
+#// This software may not be used, sold, licensed, transferred, copied
+#// or reproduced in whole or in part in any manner or form or in or
+#// on any media by any person other than in accordance with the terms
+#// of the Licence Agreement supplied with the software, or otherwise
+#// without the prior written consent of the copyright owners.
+#//
+#// This software is distributed WITHOUT ANY WARRANTY, without even the
+#// implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+#// PURPOSE, except where stated in the Licence Agreement supplied with
+#// the software.
+#//
+#//      Created By :            Michael Boniface
+#//      Created Date :          23/01/2018
+#//      Created for Project :   FLAME
+#//
+#/////////////////////////////////////////////////////////////////////////
+
+# Install host 
+# This is a dummy script as the endpoint is driven by simulation
\ No newline at end of file

test/services/host/telegraf_host.conf

+
+###############################################################################
+#                                  INPUTS                                     #
+###############################################################################
+# # Read metrics about network interface usage
+ [[inputs.net]]
+#   ## By default, telegraf gathers stats from any up interface (excluding loopback)
+#   ## Setting interfaces will tell it to gather these explicit interfaces,
+#   ## regardless of status.
+#   ##
+#   # interfaces = ["eth0"]
+
+# Read metrics about cpu usage
+[[inputs.cpu]]
+  ## Whether to report per-cpu stats or not
+  percpu = true
+  ## Whether to report total system cpu stats or not
+  totalcpu = true
+  ## If true, collect raw CPU time metrics.
+  collect_cpu_time = false
+  ## If true, compute and report the sum of all non-idle CPU states.
+ #report_active = false
+
+
+# Read metrics about disk usage by mount point
+[[inputs.disk]]
+  ## By default, telegraf gather stats for all mountpoints.
+  ## Setting mountpoints will restrict the stats to the specified mountpoints.
+  # mount_points = ["/"]
+
+  ## Ignore some mountpoints by filesystem type. For example (dev)tmpfs (usually
+  ## present on /run, /var/run, /dev/shm or /dev).
+  ignore_fs = ["tmpfs", "devtmpfs", "devfs"]
+
+
+# Read metrics about disk IO by device
+[[inputs.diskio]]
+  ## By default, telegraf will gather stats for all devices including
+  ## disk partitions.
+  ## Setting devices will restrict the stats to the specified devices.
+  # devices = ["sda", "sdb"]
+  ## Uncomment the following line if you need disk serial numbers.
+  # skip_serial_number = false
+  #
+  ## On systems which support it, device metadata can be added in the form of
+  ## tags.
+  ## Currently only Linux is supported via udev properties. You can view
+  ## available properties for a device by running:
+  ## 'udevadm info -q property -n /dev/sda'
+  # device_tags = ["ID_FS_TYPE", "ID_FS_USAGE"]
+  #
+  ## Using the same metadata source as device_tags, you can also customize the
+  ## name of the device via templates.
+  ## The 'name_templates' parameter is a list of templates to try and apply to
+  ## the device. The template may contain variables in the form of '$PROPERTY' or
+  ## '${PROPERTY}'. The first template which does not contain any variables not
+  ## present for the device is used as the device name tag.
+  ## The typical use case is for LVM volumes, to get the VG/LV name instead of
+  ## the near-meaningless DM-0 name.
+  # name_templates = ["$ID_FS_LABEL","$DM_VG_NAME/$DM_LV_NAME"]
+
+# Read metrics about memory usage
+[[inputs.mem]]
+  # no configuration
+
+# # Influx HTTP write listener
+[[inputs.http_listener]]
+  ## Address and port to host HTTP listener on
+  service_address = ":8186"
+
+  ## timeouts
+  read_timeout = "10s"
+  write_timeout = "10s"
+
+  ## HTTPS
+  #tls_cert= "/etc/telegraf/cert.pem"
+  #tls_key = "/etc/telegraf/key.pem"
+
+  ## MTLS
+  #tls_allowed_cacerts = ["/etc/telegraf/clientca.pem"]
\ No newline at end of file