diff --git a/Vagrantfile b/Vagrantfile index 405f96701d9b0674a5ab4da41882a6f2cf6ae953..8b5c0a1b8b48a9a0b721de06891f7c9d745c73cf 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -17,6 +17,21 @@ sed -i s/10.0.3/172.40.231/g /etc/default/lxc-net sed -i s/#LXC_DHCP_CONFILE/LXC_DHCP_CONFILE/g /etc/default/lxc-net service lxc-net restart +# enable NTP +# use network time to make sure we are synchronised +echo "Disabling timesyncd..." +timedatectl set-ntp no +until timedatectl | grep -m 1 "Network time on: no"; +do + echo "Waiting for timesyncd to turn off.." + sleep 1 +done +apt-get install ntp +echo "timesync set to ntpd" + +# set timezone to London +timedatectl set-timezone Europe/London + SCRIPT Vagrant.configure("2") do |config| diff --git a/scripts/clmc-agent/configure.sh b/scripts/clmc-agent/configure.sh index 35226d643103e9fbcdf2c5799c371f33ded9791d..9f98a51303b1e363a9dbf125c6c775742cf82849 100755 --- a/scripts/clmc-agent/configure.sh +++ b/scripts/clmc-agent/configure.sh @@ -47,23 +47,23 @@ INFLUXDB_URL=$8 DATABASE_NAME=$9 TELEGRAF_CONF_DIR="/etc/telegraf" -TELEGRAF_CONF_FILE=$TELEGRAF_CONF_DIR"/telegraf.conf" -TELEGRAF_INCLUDE_CONF_DIR=$TELEGRAF_CONF_DIR"/telegraf.d" -TELEGRAF_OUTPUT_CONF_FILE=$TELEGRAF_INCLUDE_CONF_DIR"/telegraf_output.conf" +TELEGRAF_CONF_FILE=${TELEGRAF_CONF_DIR}"/telegraf.conf" +TELEGRAF_INCLUDE_CONF_DIR=${TELEGRAF_CONF_DIR}"/telegraf.d" +TELEGRAF_OUTPUT_CONF_FILE=${TELEGRAF_INCLUDE_CONF_DIR}"/telegraf_output.conf" #cat ${TELEGRAF_OUTPUT_CONF_FILE} # Replace template parameters on general configuration -sed -i 's/$LOCATION/'$LOCATION'/g' $TELEGRAF_CONF_FILE -sed -i 's/$SFC_ID/'$SFC_ID'/g' $TELEGRAF_CONF_FILE -sed -i 's/$SFC_ID_INSTANCE/'$SFC_ID_INSTANCE'/g' $TELEGRAF_CONF_FILE -sed -i 's/$SF_ID/'$SF_ID'/g' $TELEGRAF_CONF_FILE -sed -i 's/$SF_ID_INSTANCE}}/'$SF_ID_INSTANCE'/g' $TELEGRAF_CONF_FILE -sed -i 's/$IP_ENDPOINT_ID/'$IP_ENDPOINT_ID'/g' $TELEGRAF_CONF_FILE -sed -i 's/$SR_ID/'$SR_ID'/g' $TELEGRAF_CONF_FILE +sed -i 's/${LOCATION}/'${LOCATION}'/g' ${TELEGRAF_CONF_FILE} +sed -i 's/${SFC_ID}/'${SFC_ID}'/g' ${TELEGRAF_CONF_FILE} +sed -i 's/${SFC_ID_INSTANCE}/'${SFC_ID_INSTANCE}'/g' ${TELEGRAF_CONF_FILE} +sed -i 's/${SF_ID}/'${SF_ID}'/g' ${TELEGRAF_CONF_FILE} +sed -i 's/${SF_ID_INSTANCE}/'${SF_ID_INSTANCE}'/g' ${TELEGRAF_CONF_FILE} +sed -i 's/${IP_ENDPOINT_ID}/'${IP_ENDPOINT_ID}'/g' ${TELEGRAF_CONF_FILE} +sed -i 's/${SR_ID}/'${SR_ID}'/g' ${TELEGRAF_CONF_FILE} echo "Telegraf Output Configuration File: ${TELEGRAF_OUTPUT_CONF_FILE}" # Replace parameters in output configuration file -sed -i 's|$INFLUXDB_URL|'$INFLUXDB_URL'|g' $TELEGRAF_OUTPUT_CONF_FILE -sed -i 's/$DATABASE_NAME/'$DATABASE_NAME'/g' $TELEGRAF_OUTPUT_CONF_FILE \ No newline at end of file +sed -i 's|${INFLUXDB_URL}|'${INFLUXDB_URL}'|g' ${TELEGRAF_OUTPUT_CONF_FILE} +sed -i 's/${DATABASE_NAME}/'${DATABASE_NAME}'/g' ${TELEGRAF_OUTPUT_CONF_FILE} \ No newline at end of file diff --git a/scripts/clmc-agent/telegraf.conf b/scripts/clmc-agent/telegraf.conf index 6c5aed64e2cd00cfc227434bf25eca387d5fa6ae..07ac73a4dcf6a067a099ce629d9fc4f81bfb1ed2 100644 --- a/scripts/clmc-agent/telegraf.conf +++ b/scripts/clmc-agent/telegraf.conf @@ -33,19 +33,19 @@ # Global tags can be specified here in key="value" format. [global_tags] # location of the data centre - location="$LOCATION" + location="${LOCATION}" # media service template id - sfc="$SFC_ID" + sfc="${SFC_ID}" # media service instance - sfc_i="$SFC_ID_INSTANCE" + sfc_i="${SFC_ID_INSTANCE}" # service function type - sf="$SF_ID" + sf="${SF_ID}" # service function instance id - sf_i="$SF_ID_INSTANCE" + sf_i="${SF_ID_INSTANCE}" # ipendpoint id aka surrogate instance - ipendpoint="$IP_ENDPOINT_ID" + ipendpoint="${IP_ENDPOINT_ID}" # the service router providing access to the network - sr="$SR_ID" + sr="${SR_ID}" # Configuration for telegraf agent [agent] diff --git a/scripts/clmc-agent/telegraf_output.conf b/scripts/clmc-agent/telegraf_output.conf index e77f2476c1d3d8ff330580fb5a8b33350041adf8..7521ac34a05a0c8f7ccdc166a772fbd2d0ee5544 100644 --- a/scripts/clmc-agent/telegraf_output.conf +++ b/scripts/clmc-agent/telegraf_output.conf @@ -29,9 +29,9 @@ # Multiple urls can be specified but it is assumed that they are part of the same # cluster, this means that only ONE of the urls will be written to each interval. # urls = ["udp://127.0.0.1:8089"] # UDP endpoint example - urls = ["$INFLUXDB_URL"] # required + urls = ["${INFLUXDB_URL}"] # required # The target database for metrics (telegraf will create it if not exists) - database = "$DATABASE_NAME" # required + database = "${DATABASE_NAME}" # required # Precision of writes, valid values are "ns", "us" (or "µs"), "ms", "s", "m", "h". # note: using second precision greatly helps InfluxDB compression precision = "s" diff --git a/scripts/test/fixture.sh b/scripts/test/fixture.sh index f6aecda8f18d95173d7a109c0b8dd165b2b84d78..a9ac2a291db960bce4d747aa5847a0a09f165626 100755 --- a/scripts/test/fixture.sh +++ b/scripts/test/fixture.sh @@ -75,9 +75,6 @@ create() { cp ${repo_root}/src/test/clmctest/services/${sf_id}/telegraf*.conf ${container_dir}/etc/telegraf/telegraf.d/ # replace telegraf template with container parameters - # TODO: do we really need both scripts to do this? - cmd=/vagrant/scripts/clmc-agent/configure_template.sh - lxc-attach -n ${service_name} -- ${cmd} cmd="/vagrant/scripts/clmc-agent/configure.sh ${location} ${sfc_id} ${sfc_id_instance} ${sf_id} ${sf_id_instance} ${ipendpoint_id} ${sr_id} ${influxdb_url} ${database_name}" lxc-attach -n ${service_name} -- ${cmd} diff --git a/src/test/clmctest/services/nginx/telegraf_nginx.conf b/src/test/clmctest/services/nginx/telegraf_nginx.conf index 9834cc28d8256271ec8b5299fffbb2a5779131de..e2ab3a4828f4886df4f0e0b811246a3a2ee5b850 100644 --- a/src/test/clmctest/services/nginx/telegraf_nginx.conf +++ b/src/test/clmctest/services/nginx/telegraf_nginx.conf @@ -30,11 +30,11 @@ #pid_file = "/var/run/nginx.pid" exe = "nginx" -# [[inputs.systemctl]] -# ## Service array -# services = [ -# "nginx.service" -# ] +[[inputs.systemctl]] + ## Service array + services = [ + "nginx.service" + ] # # Influx HTTP write listener [[inputs.http_listener]] diff --git a/src/test/clmctest/workshopdemo/README.md b/src/test/clmctest/workshopdemo/README.md index 6f56cc7cf661a49e17e9f2aeca16f33d098837da..f117a025ce47082b8cd3a6ebcd55b550a9e57ac0 100644 --- a/src/test/clmctest/workshopdemo/README.md +++ b/src/test/clmctest/workshopdemo/README.md @@ -1,43 +1,95 @@ -## FLAME media service integration workshop +# FLAME media service integration workshop +## Demonstration set-up notes +### Prerequisites -### Demonstration set-up notes +### Create the Vagrant host and log in -#### Create the Vagrant host and log in - -``` +```shell vagrant up vagrant ssh sudo su ``` -#### Start the containers and check +### NATting (for Windows host) -``` +* Chronograf: port 8888 + +* 2 NGINX installations (/…/services/nginx) + * ports: 9180 and 9181 + +* 2 MINIO installations (/…/services/minio) + * ports: 9182 and 9183 + +### Create containers + +```shell /vagrant/scripts/test/fixture.sh create -f /vagrant/src/test/clmctest/rspec.json +``` -lxc-ls -> clmc-service host1 host2 host3 host4 +### Create all containers, leave nginx1 and clmc started + +```shell +/vagrant/scripts/test/fixture.sh stop -f /vagrant/src/test/clmctest/rspec.json -c host2 +/vagrant/scripts/test/fixture.sh stop -f /vagrant/src/test/clmctest/rspec.json -c host3 +/vagrant/scripts/test/fixture.sh stop -f /vagrant/src/test/clmctest/rspec.json -c host4 ``` -#### NATting (for Windows host) +### Go to demonstration folder and re-configure NGINX container -* Chronograf: port 8888 +```shell +cd /vagrant/src/test/clmctest/workshopdemo/nginxMiniDemo +chmod +x initDemo.sh +./initDemo.sh +``` -* 2 NGINX installations (/…/services/nginx) - - ports: 9180 and 9181 +### Delete MSDemo database on clmc-service -* 2 MINIO installations (/…/services/minio) - - ports: 9182 and 9183 +??? + +## Mini NGINX demonstration + +### Attach to container and install TELEGRAF (manually or otherwise) + +`lxc-attach -n host1` + +> Method for installing Telegraf is described in `/tmp/installTelegraf.sh` + +### Send a simple metric via HTTP listener (whilst in NGINX container) + +```shell +apt-get install curl +curl -i -XPOST 'http://172.40.231.51:8086/write?db=SimpleDemo' --data-binary 'MyServiceComponent requestCount=1' +``` + +This should return a 'HTTP/1.1 204 No Content' response + +### View data point in CLMC + +Go to `localhost:8888`... + +Connect to "SimpleDemo" database. + +## Main scenario demonstration + +### Start the containers and check + +```shell +/vagrant/scripts/test/fixture.sh start -f /vagrant/src/test/clmctest/rspec.json -c host2 +/vagrant/scripts/test/fixture.sh start -f /vagrant/src/test/clmctest/rspec.json -c host3 +/vagrant/scripts/test/fixture.sh start -f /vagrant/src/test/clmctest/rspec.json -c host4 +lxc-ls +> clmc-service host1 host2 host3 host4 +``` -#### Run demo initializer +### Run demo initializer cd /vagrant/src/test/clmctest/workshopdemo chmod +x initDemo.sh ./initDemo.sh -#### Check out the video data you've uploaded +### Check out the video data you've uploaded MINIO 1 http://localhost:9182/minio/democontent/ @@ -45,7 +97,7 @@ http://localhost:9182/minio/democontent/ MINIO 2 http://localhost:9183/minio/democontent/ -#### Stream some video +### Stream some video Use the BitMovin' player: http://bitmovin.com/hls-mpeg-dash-test-player/ @@ -54,7 +106,7 @@ with these sources: http://localhost:9180/car-20120827-manifest.mpd http://localhost:9181/car-20120827-manifest.mpd -#### Tear-down +### Tear-down ``` /vagrant/scripts/test/fixture.sh destroy -f /vagrant/src/test/clmctest/rspec.json diff --git a/src/test/clmctest/workshopdemo/initDemo.sh b/src/test/clmctest/workshopdemo/initDemo.sh index 22d85b61d0f46aeb424d5311cd4df08acb1e715d..e11b5342134b845fb21fc47efd78a9c9bbfc5b28 100644 --- a/src/test/clmctest/workshopdemo/initDemo.sh +++ b/src/test/clmctest/workshopdemo/initDemo.sh @@ -24,10 +24,15 @@ #// #///////////////////////////////////////////////////////////////////////// -# Run from Vagrant host -# Prerequisites +## Run from Vagrant host +### Prerequisites apt-get install -y unzip +# Mini NGINX demonstration + + + +# NGINX/MINIO demonstration mkdir -p /tmp/demoSetup cp * /tmp/demoSetup cd /tmp/demoSetup diff --git a/src/test/clmctest/workshopdemo/nginxMiniDemo/initDemo.sh b/src/test/clmctest/workshopdemo/nginxMiniDemo/initDemo.sh new file mode 100644 index 0000000000000000000000000000000000000000..e487635f692ad20935d2ecc458e3d674f18bf98c --- /dev/null +++ b/src/test/clmctest/workshopdemo/nginxMiniDemo/initDemo.sh @@ -0,0 +1,45 @@ +#!/bin/bash +#///////////////////////////////////////////////////////////////////////// +#// +#// (c) University of Southampton IT Innovation Centre, 2017 +#// +#// Copyright in this software belongs to University of Southampton +#// IT Innovation Centre of Gamma House, Enterprise Road, +#// Chilworth Science Park, Southampton, SO16 7NS, UK. +#// +#// This software may not be used, sold, licensed, transferred, copied +#// or reproduced in whole or in part in any manner or form or in or +#// on any media by any person other than in accordance with the terms +#// of the Licence Agreement supplied with the software, or otherwise +#// without the prior written consent of the copyright owners. +#// +#// This software is distributed WITHOUT ANY WARRANTY, without even the +#// implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR +#// PURPOSE, except where stated in the Licence Agreement supplied with +#// the software. +#// +#// Created By : Simon Crowle +#// Created Date : 07/06/2018 +#// Created for Project : FLAME +#// +#///////////////////////////////////////////////////////////////////////// + +# Instantiate an NGINX container (host1) +/vagrant/scripts/test/fixture.sh create -f /vagrant/src/test/clmctest/rspec.json -c host1 + +# Uninstall TELEGRAF and clean up +lxc-attach -n host1 -- service telegraf stop +lxc-attach -n host1 -- dpkg -r telegraf +lxc-attach -n host1 -- rm -rf /etc/telegraf + +# Copy in demonstration resources and TELEGRAF configuration (ready for editing) +lxc-attach -n host1 -- mkdir -p /tmp/telegraf +cp resources/installTelegraf.sh /var/lib/lxc/host1/rootfs/tmp/ +cp -r telegrafConfigs/* /var/lib/lxc/host1/rootfs/tmp/telegraf + +# Add REPORC for accessing our Telegraf.deb +cp /vagrant/reporc /var/lib/lxc/host1/rootfs/tmp/ + +# Drop MSDemo database on CLMC so that it is empty of other hosts +# N.B. it will be created again automatically as soon as any host pushes data to it +lxc-attach -n clmc-service -- influx -execute 'drop database MSDemo' \ No newline at end of file diff --git a/src/test/clmctest/workshopdemo/nginxMiniDemo/resources/installTelegraf.sh b/src/test/clmctest/workshopdemo/nginxMiniDemo/resources/installTelegraf.sh new file mode 100644 index 0000000000000000000000000000000000000000..6918413e69d8f78e38705bcf8dc1ad37c68996d8 --- /dev/null +++ b/src/test/clmctest/workshopdemo/nginxMiniDemo/resources/installTelegraf.sh @@ -0,0 +1,44 @@ +#!/bin/bash +#///////////////////////////////////////////////////////////////////////// +#// +#// (c) University of Southampton IT Innovation Centre, 2017 +#// +#// Copyright in this software belongs to University of Southampton +#// IT Innovation Centre of Gamma House, Enterprise Road, +#// Chilworth Science Park, Southampton, SO16 7NS, UK. +#// +#// This software may not be used, sold, licensed, transferred, copied +#// or reproduced in whole or in part in any manner or form or in or +#// on any media by any person other than in accordance with the terms +#// of the Licence Agreement supplied with the software, or otherwise +#// without the prior written consent of the copyright owners. +#// +#// This software is distributed WITHOUT ANY WARRANTY, without even the +#// implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR +#// PURPOSE, except where stated in the Licence Agreement supplied with +#// the software. +#// +#// Created By : Simon Crowle +#// Created Date : 07/06/2018 +#// Created for Project : FLAME +#// +#///////////////////////////////////////////////////////////////////////// + +# Download TELEGRAF +# ------------------------------------------------------------------------ +TELEGRAF_VERSION=1.7.0~5618bb0-0 +source /tmp/reporc +wget --user ${REPO_USER} --password ${REPO_PASS} https://flame-nexus.it-innovation.soton.ac.uk/repository/flame-general/it-innovation/telegraf/${TELEGRAF_VERSION}/telegraf-${TELEGRAF_VERSION}.deb -O /tmp/telegraf.deb + +# Install Telegraf +# ------------------------------------------------------------------------ +dpkg -i /tmp/telegraf.deb +service telegraf stop + +# Copy TELEGRAF files into etc +cp -r /tmp/telegraf/* /etc/telegraf/ + +# Edit Telegraf configuration at your leisure... + +# Run telegraf +#service telegraf start \ No newline at end of file diff --git a/src/test/clmctest/workshopdemo/nginxMiniDemo/telegrafConfigs/telegraf.conf b/src/test/clmctest/workshopdemo/nginxMiniDemo/telegrafConfigs/telegraf.conf new file mode 100644 index 0000000000000000000000000000000000000000..b3b188fb299d87ba763abe13f895b93d5f69f4d7 --- /dev/null +++ b/src/test/clmctest/workshopdemo/nginxMiniDemo/telegrafConfigs/telegraf.conf @@ -0,0 +1,20 @@ +#[global_tags] +# location="DC1" +# sfc="MSDemo" +# sfc_i="MSDemo_1" +# sf="nginx" +# sf_i="nginx_1" +# ipendpoint="nginx_1_ep1" +# sr="SR1" +#[agent] +# interval = "10s" +# round_interval = true +# metric_buffer_limit = 1000 +# flush_buffer_when_full = true +# collection_jitter = "0s" +# flush_interval = "10s" +# flush_jitter = "0s" +# debug = false +# quiet = false +# logfile = "/var/log/telegraf/telegraf.log" +# hostname = "" diff --git a/src/test/clmctest/workshopdemo/nginxMiniDemo/telegrafConfigs/telegraf.d/telegraf_host.conf b/src/test/clmctest/workshopdemo/nginxMiniDemo/telegrafConfigs/telegraf.d/telegraf_host.conf new file mode 100644 index 0000000000000000000000000000000000000000..484f9f65211dfe2ffa73749308c6966eee561f96 --- /dev/null +++ b/src/test/clmctest/workshopdemo/nginxMiniDemo/telegrafConfigs/telegraf.d/telegraf_host.conf @@ -0,0 +1,30 @@ +##Plugins to run with a little bit of configuration +#[[inputs.cpu]] +# # Whether to report per-cpu stats or not +# percpu = true +# # Whether to report total system cpu stats or not +# totalcpu = true +# # If true, collect raw CPU time metrics. +# collect_cpu_time = false +# # If true, compute and report the sum of all non-idle CPU states. +# report_active = false + +#[[inputs.disk]] +# ignore_fs = ["tmpfs", "devtmpfs", "devfs"] + +#[[inputs.diskio]] +# devices = ["sda1"] + +## Plugins to run 'out-of-the-box' (no configuration applied) +#[[inputs.kernel]] +#[[inputs.mem]] +#[[inputs.processes]] +#[[inputs.swap]] +#[[inputs.system]] +#[[inputs.net]] +#[[inputs.netstat]] + +## Influx HTTP write listener (for bespoke POSTing of your media service metrics) +#[[inputs.http_listener]] +# # Address and port to host HTTP listener on +# service_address = ":8186" \ No newline at end of file diff --git a/src/test/clmctest/workshopdemo/nginxMiniDemo/telegrafConfigs/telegraf.d/telegraf_nginx.conf b/src/test/clmctest/workshopdemo/nginxMiniDemo/telegrafConfigs/telegraf.d/telegraf_nginx.conf new file mode 100644 index 0000000000000000000000000000000000000000..029e5214f259e1eb20033c9186bcb5f446a147ed --- /dev/null +++ b/src/test/clmctest/workshopdemo/nginxMiniDemo/telegrafConfigs/telegraf.d/telegraf_nginx.conf @@ -0,0 +1,42 @@ +## © University of Southampton IT Innovation Centre, 2018 +## +## Copyright in this software belongs to University of Southampton +## IT Innovation Centre of Gamma House, Enterprise Road, +## Chilworth Science Park, Southampton, SO16 7NS, UK. +## +## This software may not be used, sold, licensed, transferred, copied +## or reproduced in whole or in part in any manner or form or in or +## on any media by any person other than in accordance with the terms +## of the Licence Agreement supplied with the software, or otherwise +## without the prior written consent of the copyright owners. +## +## This software is distributed WITHOUT ANY WARRANTY, without even the +## implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR +## PURPOSE, except where stated in the Licence Agreement supplied with +## the software. +## +## Created By : Rowan Powell +## Created Date : 02-02-2018 +## Created for Project : FLAME + +# Read Nginx's basic status information (ngx_http_stub_status_module) +#[[inputs.nginx]] + ## An array of Nginx stub_status URI to gather stats. + # urls = ["http://localhost:80/nginx_status"] + + ## HTTP response timeout (default: 5s) + # response_timeout = "5s" + +#[[inputs.procstat]] + #pid_file = "/var/run/nginx.pid" + #exe = "nginx" + +# [[inputs.systemctl]] +# ## Service array +# services = [ +# "nginx.service" +# ] + +# ## Sample rate for sampling state of service. +# ## Must be greter that the collection_interval/2 +# #sample_rate = 2 \ No newline at end of file diff --git a/src/test/clmctest/workshopdemo/nginxMiniDemo/telegrafConfigs/telegraf.d/telegraf_output.conf b/src/test/clmctest/workshopdemo/nginxMiniDemo/telegrafConfigs/telegraf.d/telegraf_output.conf new file mode 100644 index 0000000000000000000000000000000000000000..e9a26d972206e94741984312743a59c481211e76 --- /dev/null +++ b/src/test/clmctest/workshopdemo/nginxMiniDemo/telegrafConfigs/telegraf.d/telegraf_output.conf @@ -0,0 +1,6 @@ +# Output plugin to forward on metrics to INFLUX +#[[outputs.influxdb]] +# urls = ["http://172.40.231.51:8086"] +# database = "MSDemo" +# precision = "s" +# timeout = "5s"