From b10d13398dbd11c411b50ffe341aaf0ba67824ba Mon Sep 17 00:00:00 2001
From: MJB <mjb@it-innovation.soton.ac.uk>
Date: Fri, 2 Mar 2018 18:04:26 +0000
Subject: [PATCH] added test for telegraf-agents
---
Vagrantfile | 3 +-
.../telegraf_ipendpoint_template.conf | 112 ------------------
test/services/apache/install.sh | 12 --
test/services/jmeter/install.sh | 46 -------
test/services/pytest/install.sh | 3 +-
test/streaming/setup.sh | 8 +-
test/telegraf-agents/__init__.py | 1 +
test/telegraf-agents/conftest.py | 11 ++
test/telegraf-agents/rspec.yml | 98 +++++++++++++++
test/telegraf-agents/test_telegraf_agents.py | 53 +++++++++
10 files changed, 173 insertions(+), 174 deletions(-)
delete mode 100644 scripts/clmc-agent/telegraf_ipendpoint_template.conf
delete mode 100755 test/services/jmeter/install.sh
create mode 100644 test/telegraf-agents/__init__.py
create mode 100644 test/telegraf-agents/conftest.py
create mode 100644 test/telegraf-agents/rspec.yml
create mode 100644 test/telegraf-agents/test_telegraf_agents.py
diff --git a/Vagrantfile b/Vagrantfile
index b0d5152..6f3f4c4 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -87,7 +87,8 @@ Vagrant.configure("2") do |config|
puts "Instance name #{instance_name}:"
case instance_name
when 'clmc-service'
- instance_config.vm.provision :shell, :path => "scripts/clmc-service/install.sh"
+ # instance_config.vm.provision :shell, :path => "scripts/clmc-service/install.sh"
+ instance_config.vm.provision :shell, :path => "test/services/pytest/install.sh"
else
# specific service install
instance_config.vm.provision :shell, :path => "test/services/#{host["service_name"]}/install.sh", env: {"REPO_ROOT" => "/vagrant"}
diff --git a/scripts/clmc-agent/telegraf_ipendpoint_template.conf b/scripts/clmc-agent/telegraf_ipendpoint_template.conf
deleted file mode 100644
index 2358dcc..0000000
--- a/scripts/clmc-agent/telegraf_ipendpoint_template.conf
+++ /dev/null
@@ -1,112 +0,0 @@
-# Telegraf configuration
-
-# Telegraf is entirely plugin driven. All metrics are gathered from the
-# declared inputs, and sent to the declared outputs.
-
-# Plugins must be declared in here to be active.
-# To deactivate a plugin, comment out the name and any variables.
-
-# Use 'telegraf -config telegraf.conf -test' to see what metrics a config
-# file would generate.
-
-# Global tags can be specified here in key="value" format.
-[global_tags]
- # location of the data centre
- location="{{LOCATION}}"
- # media service template id
- sfc="{{SFC_ID}}"
- # media service instance
- sfc_i="{{SFC_ID_INSTANCE}}"
- # service function type
- sf="{{SF_ID}}"
- # service function instance id
- sf_i="{{SF_ID_INSTANCE}}"
- # ipendpoint id aka surrogate instance
- ipendpoint="{{IP_ENDPOINT_ID}}"
-
-# Configuration for telegraf agent
-[agent]
- ## Default data collection interval for all inputs
- interval = "10s"
- ## Rounds collection interval to 'interval'
- ## ie, if interval="10s" then always collect on :00, :10, :20, etc.
- round_interval = true
-
- ## Telegraf will cache metric_buffer_limit metrics for each output, and will
- ## flush this buffer on a successful write.
- metric_buffer_limit = 1000
- ## Flush the buffer whenever full, regardless of flush_interval.
- flush_buffer_when_full = true
-
- ## Collection jitter is used to jitter the collection by a random amount.
- ## Each plugin will sleep for a random time within jitter before collecting.
- ## This can be used to avoid many plugins querying things like sysfs at the
- ## same time, which can have a measurable effect on the system.
- collection_jitter = "0s"
-
- ## Default flushing interval for all outputs. You shouldn't set this below
- ## interval. Maximum flush_interval will be flush_interval + flush_jitter
- flush_interval = "10s"
- ## Jitter the flush interval by a random amount. This is primarily to avoid
- ## large write spikes for users running a large number of telegraf instances.
- ## ie, a jitter of 5s and interval 10s means flushes will happen every 10-15s
- flush_jitter = "0s"
-
- ## Logging configuration:
- ## Run telegraf in debug mode
- debug = false
- ## Run telegraf in quiet mode
- quiet = false
- ## Specify the log file name. The empty string means to log to stdout.
- logfile = "G:/Telegraf/telegraf.log"
-
- ## Override default hostname, if empty use os.Hostname()
- hostname = ""
-
-
-###############################################################################
-# OUTPUTS #
-###############################################################################
-
-# Configuration for influxdb server to send metrics to
-[[outputs.influxdb]]
- # The full HTTP or UDP endpoint URL for your InfluxDB instance.
- # Multiple urls can be specified but it is assumed that they are part of the same
- # cluster, this means that only ONE of the urls will be written to each interval.
- # urls = ["udp://127.0.0.1:8089"] # UDP endpoint example
- urls = ["{{INFLUXDB_URL}}"] # required
- # The target database for metrics (telegraf will create it if not exists)
- database = "{{DATABASE_NAME}}" # required
- # Precision of writes, valid values are "ns", "us" (or "µs"), "ms", "s", "m", "h".
- # note: using second precision greatly helps InfluxDB compression
- precision = "s"
-
- ## Write timeout (for the InfluxDB client), formatted as a string.
- ## If not provided, will default to 5s. 0s means no timeout (not recommended).
- timeout = "5s"
- # username = "telegraf"
- # password = "metricsmetricsmetricsmetrics"
- # Set the user agent for HTTP POSTs (can be useful for log differentiation)
- # user_agent = "telegraf"
- # Set UDP payload size, defaults to InfluxDB UDP Client default (512 bytes)
- # udp_payload = 512
-
-
-###############################################################################
-# INPUTS #
-###############################################################################
-# # Influx HTTP write listener
-[[inputs.http_listener]]
- ## Address and port to host HTTP listener on
- service_address = ":8186"
-
- ## timeouts
- read_timeout = "10s"
- write_timeout = "10s"
-
- ## HTTPS
- #tls_cert= "/etc/telegraf/cert.pem"
- #tls_key = "/etc/telegraf/key.pem"
-
- ## MTLS
- #tls_allowed_cacerts = ["/etc/telegraf/clientca.pem"]
diff --git a/test/services/apache/install.sh b/test/services/apache/install.sh
index 8b57de8..fc93bf5 100755
--- a/test/services/apache/install.sh
+++ b/test/services/apache/install.sh
@@ -28,18 +28,6 @@
sudo apt-get update
sudo apt-get -y install apache2
-TEST_VIDEO="20180212104221flame-project-full.mp4"
-TEST_VIDEO_ARCHIVE=$TEST_VIDEO".gz"
-DEST_DIR="/var/www/html/"
-DEST_FILE=$DEST_DIR"/"$TEST_VIDEO_ARCHIVE
-
-# Copy files for MPEG-DASH testing
-curl "ftp://ftp.it-innovation.soton.ac.uk/testdata/video/"$TEST_VIDEO_ARCHIVE --user flame-rw:DR8ngj3ogSjd8gl -o $DEST_FILE
-tar -xvf $DEST_FILE -C $DEST_DIR
-
-rm -rf $DEST_FILE
-mv $DEST_DIR"/"$TEST_VIDEO $DEST_DIR"/"test_video
-
# start apache
apachectl -k start
apachectl -k restart
diff --git a/test/services/jmeter/install.sh b/test/services/jmeter/install.sh
deleted file mode 100755
index 937af59..0000000
--- a/test/services/jmeter/install.sh
+++ /dev/null
@@ -1,46 +0,0 @@
-#!/bin/bash
-#/////////////////////////////////////////////////////////////////////////
-#//
-#// (c) University of Southampton IT Innovation Centre, 2017
-#//
-#// Copyright in this software belongs to University of Southampton
-#// IT Innovation Centre of Gamma House, Enterprise Road,
-#// Chilworth Science Park, Southampton, SO16 7NS, UK.
-#//
-#// This software may not be used, sold, licensed, transferred, copied
-#// or reproduced in whole or in part in any manner or form or in or
-#// on any media by any person other than in accordance with the terms
-#// of the Licence Agreement supplied with the software, or otherwise
-#// without the prior written consent of the copyright owners.
-#//
-#// This software is distributed WITHOUT ANY WARRANTY, without even the
-#// implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
-#// PURPOSE, except where stated in the Licence Agreement supplied with
-#// the software.
-#//
-#// Created By : Michael Boniface
-#// Created Date : 12/12/2017
-#// Created for Project : FLAME
-#//
-#/////////////////////////////////////////////////////////////////////////
-
-# install java 8
-add-apt-repository -y ppa:webupd8team/java
-apt-get -y update
-
-# tell the oracle java installer the license has been read
-echo debconf shared/accepted-oracle-license-v1-1 select true | debconf-set-selections
-echo debconf shared/accepted-oracle-license-v1-1 seen true | debconf-set-selections
-DEBIAN_FRONTEND=noninteractive apt-get install -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" oracle-java8-installer
-
-# install JMeter
-JMETER_VERSION="apache-jmeter-4.0"
-wget http://mirror.vorboss.net/apache//jmeter/binaries/$JMETER_VERSION.tgz -P /tmp
-tar -C /opt -xf /tmp/$JMETER_VERSION.tgz
-rm -rf /tmp/$JMETER_VERSION.tgz
-
-# Add jmeter to the path system wide
-#echo 'PATH="'$PATH':/opt/'$JMETER_VERSION'/bin"' > /etc/environment
-
-echo 'PATH="'$PATH':/opt/apache-jmeter-4.0/bin"' > /etc/environment
-source /etc/environment
diff --git a/test/services/pytest/install.sh b/test/services/pytest/install.sh
index 0c04381..82fb18f 100644
--- a/test/services/pytest/install.sh
+++ b/test/services/pytest/install.sh
@@ -24,5 +24,6 @@
#//
#/////////////////////////////////////////////////////////////////////////
apt-get update
-apt-get -y install python-pip python-dev build-essential
+apt-get install python3 python3-pip
+update-alternatives --install /usr/bin/python python /usr/bin/python3 10
pip install pytest pyyaml
\ No newline at end of file
diff --git a/test/streaming/setup.sh b/test/streaming/setup.sh
index 71b732b..39b958a 100755
--- a/test/streaming/setup.sh
+++ b/test/streaming/setup.sh
@@ -49,11 +49,15 @@ curl -i -X POST -H "Content-Type: application/json" http://localhost:9092/kapaci
# Set up dashboard
curl -i -X POST -H "Content-Type: application/json" http://localhost:8888/chronograf/v1/dashboards -d @$TEST_DIR/dashboard.json
-# Copy in the test video
+# Copy test data
+# Apache
+# DEST_DIR="/var/www/html/"
+
+# NGINX
+DEST_DIR="/usr/share/nginx/html"
TEST_VIDEO="20180212104221flame-project-full.mp4"
TEST_VIDEO_ARCHIVE=$TEST_VIDEO".gz"
-DEST_DIR="/usr/share/nginx/html"
DEST_FILE=$DEST_DIR"/"$TEST_VIDEO_ARCHIVE
# Copy files for MPEG-DASH testing
diff --git a/test/telegraf-agents/__init__.py b/test/telegraf-agents/__init__.py
new file mode 100644
index 0000000..44f7725
--- /dev/null
+++ b/test/telegraf-agents/__init__.py
@@ -0,0 +1 @@
+#!/usr/bin/python3
\ No newline at end of file
diff --git a/test/telegraf-agents/conftest.py b/test/telegraf-agents/conftest.py
new file mode 100644
index 0000000..08b7d5c
--- /dev/null
+++ b/test/telegraf-agents/conftest.py
@@ -0,0 +1,11 @@
+#!/usr/bin/python3
+
+import pytest
+import yaml
+
+@pytest.fixture(scope="module")
+def telegraf_agent_config():
+ """Returns the service configuration deployed for the telegraf conf tests."""
+ with open("test/telegraf-agents/rspec.yml", 'r') as stream:
+ data_loaded = yaml.load(stream)
+ return data_loaded
\ No newline at end of file
diff --git a/test/telegraf-agents/rspec.yml b/test/telegraf-agents/rspec.yml
new file mode 100644
index 0000000..0aef988
--- /dev/null
+++ b/test/telegraf-agents/rspec.yml
@@ -0,0 +1,98 @@
+hosts:
+ - name: clmc-service
+ cpus: 1
+ memory: 2048
+ disk: "10GB"
+ forward_ports:
+ - guest: 8086
+ host: 8086
+ - guest: 8888
+ host: 8888
+ - guest: 9092
+ host: 9092
+ ip_address: "192.168.50.10"
+ - name: apache
+ cpus: 1
+ memory: 2048
+ disk: "10GB"
+ service_name: "apache"
+ forward_ports:
+ - guest: 80
+ host: 8881
+ ip_address: "192.168.50.11"
+ location: "DC1"
+ sfc_id: "MS_Template_1"
+ sfc_id_instance: "MS_I1"
+ sf_id: "adaptive_streaming"
+ sf_id_instance: "adaptive_streaming_I1"
+ ipendpoint_id: "adaptive_streaming_I1_apache1"
+ influxdb_url: "http://192.168.50.10:8086"
+ database_name: "CLMCMetrics"
+ - name: nginx
+ cpus: 1
+ memory: 2048
+ disk: "10GB"
+ service_name: "nginx"
+ forward_ports:
+ - guest: 80
+ host: 8082
+ ip_address: "192.168.50.13"
+ location: "DC1"
+ sfc_id: "MS_Template_1"
+ sfc_id_instance: "MS_I1"
+ sf_id: "adaptive_streaming"
+ sf_id_instance: "adaptive_streaming_nginx_I1"
+ ipendpoint_id: "adaptive_streaming_nginx_I1_apache1"
+ influxdb_url: "http://192.168.50.10:8086"
+ database_name: "CLMCMetrics"
+ - name: mongo
+ cpus: 1
+ memory: 2048
+ disk: "10GB"
+ service_name: "mongo"
+ forward_ports:
+ - guest: 80
+ host: 8083
+ ip_address: "192.168.50.14"
+ location: "DC1"
+ sfc_id: "MS_Template_1"
+ sfc_id_instance: "MS_I1"
+ sf_id: "metadata_database"
+ sf_id_instance: "metadata_database_I1"
+ ipendpoint_id: "metadata_database_I1_apache1"
+ influxdb_url: "http://192.168.50.10:8086"
+ database_name: "CLMCMetrics"
+ - name: ffmpeg
+ cpus: 1
+ memory: 2048
+ disk: "10GB"
+ service_name: "ffmpeg"
+ forward_ports:
+ - guest: 80
+ host: 8084
+ ip_address: "192.168.50.15"
+ location: "DC1"
+ sfc_id: "MS_Template_1"
+ sfc_id_instance: "MS_I1"
+ sf_id: "metadata_database"
+ sf_id_instance: "metadata_database_I1"
+ ipendpoint_id: "metadata_database_I1_apache1"
+ influxdb_url: "http://192.168.50.10:8086"
+ database_name: "CLMCMetrics"
+ - name: ipendpoint
+ cpus: 1
+ memory: 2048
+ disk: "10GB"
+ service_name: "ipendpoint"
+ forward_ports:
+ - guest: 80
+ host: 8085
+ ip_address: "192.168.50.16"
+ location: "DC1"
+ sfc_id: "MS_Template_1"
+ sfc_id_instance: "MS_I1"
+ sf_id: "adaptive_streaming"
+ sf_id_instance: "adaptive_streaming_I1"
+ ipendpoint_id: "adaptive_streaming_I1_apache1"
+ influxdb_url: "http://192.168.50.10:8086"
+ database_name: "CLMCMetrics"
\ No newline at end of file
diff --git a/test/telegraf-agents/test_telegraf_agents.py b/test/telegraf-agents/test_telegraf_agents.py
new file mode 100644
index 0000000..0321e61
--- /dev/null
+++ b/test/telegraf-agents/test_telegraf_agents.py
@@ -0,0 +1,53 @@
+#!/usr/bin/python3
+
+import sys
+
+if sys.version_info[0] < 3:
+ raise Exception("Python 3 or a more recent version is required.")
+
+import pytest
+import os
+from urllib.parse import urlencode
+from urllib.request import Request, urlopen
+from os.path import join, dirname
+
+def test_service_names(telegraf_agent_config):
+ print(telegraf_agent_config['hosts'][0]['name'])
+ assert telegraf_agent_config['hosts'][0]['name'] == 'clmc-service'
+ assert telegraf_agent_config['hosts'][1]['name'] == 'apache'
+ assert telegraf_agent_config['hosts'][2]['name'] == 'nginx'
+ assert telegraf_agent_config['hosts'][3]['name'] == 'mongo'
+ assert telegraf_agent_config['hosts'][4]['name'] == 'ffmpeg'
+ assert telegraf_agent_config['hosts'][5]['name'] == 'ipendpoint'
+
+def test_ping(telegraf_agent_config):
+ """This test will only run on linux"""
+ for x in telegraf_agent_config['hosts']:
+ print("Testing service" + x['name'] + " " + x['ip_address'])
+ response = os.system("ping -c 1 " + x['ip_address'])
+ assert response == 0, "Could not ping " + x['name'] + " on ip address " + x['ip_address']
+
+@pytest.mark.parametrize("queries",
+ [{'nginx': {'measurement': 'nginx', 'query': 'SELECT mean("requests") AS "mean_active" FROM "CLMCMetrics"."autogen"."nginx"', 'result': '1'}}])
+def test_nginx(telegraf_agent_config, queries):
+
+ measurements = send_query("http://localhost:8086", 'SHOW measurements ON "CLMCMetrics"')
+ assert queries['nginx']['measurement'] in measurements, "Measurement " + measurement + " not in the CLMCMetrics database"
+
+ result = send_query("http://localhost:8086", queries['nginx']['query'])
+ print("result" + result)
+ # assert results is correct
+
+def send_query(url, query):
+ """
+ An auxiliary static method to send a query to a url and retrieve the result
+
+ :param url: the target url to which the query is sent to - a string containing a valid URL address
+ :param query: the query to be executed on the given URL
+ :return: the result of the executed query
+ """
+ query = urlencode({"q": query}).encode("ascii")
+ request = Request("{0}/query".format(url), query)
+ result = urlopen(request)
+
+ return result.read().decode("utf-8").strip()
--
GitLab