Merge branch 'workshop-demo' into service-graph

Vagrantfile

@@ -17,6 +17,21 @@ sed -i s/10.0.3/172.40.231/g /etc/default/lxc-net
 sed -i s/#LXC_DHCP_CONFILE/LXC_DHCP_CONFILE/g /etc/default/lxc-net
 service lxc-net restart
 
+# enable NTP
+# use network time to make sure we are synchronised
+echo "Disabling timesyncd..."
+timedatectl set-ntp no
+until timedatectl | grep -m 1 "Network time on: no";
+do
+  echo "Waiting for timesyncd to turn off.."
+  sleep 1
+done
+apt-get install ntp
+echo "timesync set to ntpd"
+
+# set timezone to London
+timedatectl set-timezone Europe/London
+
 SCRIPT
 
 Vagrant.configure("2") do |config|

scripts/clmc-agent/configure.sh

@@ -47,23 +47,23 @@ INFLUXDB_URL=$8
 DATABASE_NAME=$9
 
 TELEGRAF_CONF_DIR="/etc/telegraf"
-TELEGRAF_CONF_FILE=$TELEGRAF_CONF_DIR"/telegraf.conf"
-TELEGRAF_INCLUDE_CONF_DIR=$TELEGRAF_CONF_DIR"/telegraf.d"
-TELEGRAF_OUTPUT_CONF_FILE=$TELEGRAF_INCLUDE_CONF_DIR"/telegraf_output.conf"
+TELEGRAF_CONF_FILE=${TELEGRAF_CONF_DIR}"/telegraf.conf"
+TELEGRAF_INCLUDE_CONF_DIR=${TELEGRAF_CONF_DIR}"/telegraf.d"
+TELEGRAF_OUTPUT_CONF_FILE=${TELEGRAF_INCLUDE_CONF_DIR}"/telegraf_output.conf"
 
 #cat ${TELEGRAF_OUTPUT_CONF_FILE}
 
 # Replace template parameters on general configuration
-sed -i 's/$LOCATION/'$LOCATION'/g' $TELEGRAF_CONF_FILE
-sed -i 's/$SFC_ID/'$SFC_ID'/g' $TELEGRAF_CONF_FILE
-sed -i 's/$SFC_ID_INSTANCE/'$SFC_ID_INSTANCE'/g' $TELEGRAF_CONF_FILE
-sed -i 's/$SF_ID/'$SF_ID'/g' $TELEGRAF_CONF_FILE
-sed -i 's/$SF_ID_INSTANCE}}/'$SF_ID_INSTANCE'/g' $TELEGRAF_CONF_FILE
-sed -i 's/$IP_ENDPOINT_ID/'$IP_ENDPOINT_ID'/g' $TELEGRAF_CONF_FILE
-sed -i 's/$SR_ID/'$SR_ID'/g' $TELEGRAF_CONF_FILE
+sed -i 's/${LOCATION}/'${LOCATION}'/g' ${TELEGRAF_CONF_FILE}
+sed -i 's/${SFC_ID}/'${SFC_ID}'/g' ${TELEGRAF_CONF_FILE}
+sed -i 's/${SFC_ID_INSTANCE}/'${SFC_ID_INSTANCE}'/g' ${TELEGRAF_CONF_FILE}
+sed -i 's/${SF_ID}/'${SF_ID}'/g' ${TELEGRAF_CONF_FILE}
+sed -i 's/${SF_ID_INSTANCE}/'${SF_ID_INSTANCE}'/g' ${TELEGRAF_CONF_FILE}
+sed -i 's/${IP_ENDPOINT_ID}/'${IP_ENDPOINT_ID}'/g' ${TELEGRAF_CONF_FILE}
+sed -i 's/${SR_ID}/'${SR_ID}'/g' ${TELEGRAF_CONF_FILE}
 
 echo "Telegraf Output Configuration File: ${TELEGRAF_OUTPUT_CONF_FILE}"
 
 # Replace parameters in output configuration file
-sed -i 's|$INFLUXDB_URL|'$INFLUXDB_URL'|g' $TELEGRAF_OUTPUT_CONF_FILE
-sed -i 's/$DATABASE_NAME/'$DATABASE_NAME'/g' $TELEGRAF_OUTPUT_CONF_FILE
+sed -i 's|${INFLUXDB_URL}|'${INFLUXDB_URL}'|g' ${TELEGRAF_OUTPUT_CONF_FILE}
+sed -i 's/${DATABASE_NAME}/'${DATABASE_NAME}'/g' ${TELEGRAF_OUTPUT_CONF_FILE}
\ No newline at end of file

scripts/clmc-agent/telegraf.conf

@@ -33,19 +33,19 @@
 # Global tags can be specified here in key="value" format.
 [global_tags]
   # location of the data centre
-  location="$LOCATION"
+  location="${LOCATION}"
   # media service template id
-  sfc="$SFC_ID"
+  sfc="${SFC_ID}"
   # media service instance
-  sfc_i="$SFC_ID_INSTANCE"
+  sfc_i="${SFC_ID_INSTANCE}"
   # service function type
-  sf="$SF_ID"
+  sf="${SF_ID}"
   # service function instance id
-  sf_i="$SF_ID_INSTANCE"
+  sf_i="${SF_ID_INSTANCE}"
   # ipendpoint id aka surrogate instance
-  ipendpoint="$IP_ENDPOINT_ID"
+  ipendpoint="${IP_ENDPOINT_ID}"
   # the service router providing access to the network
-  sr="$SR_ID"
+  sr="${SR_ID}"
 
 # Configuration for telegraf agent
 [agent]

scripts/clmc-agent/telegraf_output.conf

@@ -29,9 +29,9 @@
   # Multiple urls can be specified but it is assumed that they are part of the same
   # cluster, this means that only ONE of the urls will be written to each interval.
   # urls = ["udp://127.0.0.1:8089"] # UDP endpoint example
-  urls = ["$INFLUXDB_URL"] # required
+  urls = ["${INFLUXDB_URL}"] # required
   # The target database for metrics (telegraf will create it if not exists)
-  database = "$DATABASE_NAME" # required
+  database = "${DATABASE_NAME}" # required
   # Precision of writes, valid values are "ns", "us" (or "µs"), "ms", "s", "m", "h".
   # note: using second precision greatly helps InfluxDB compression
   precision = "s"

scripts/test/fixture.sh

@@ -75,9 +75,6 @@ create() {
             cp ${repo_root}/src/test/clmctest/services/${sf_id}/telegraf*.conf ${container_dir}/etc/telegraf/telegraf.d/
 
             # replace telegraf template with container parameters
-            # TODO: do we really need both scripts to do this?
-            cmd=/vagrant/scripts/clmc-agent/configure_template.sh
-            lxc-attach -n ${service_name} -- ${cmd}
             cmd="/vagrant/scripts/clmc-agent/configure.sh ${location} ${sfc_id} ${sfc_id_instance} ${sf_id} ${sf_id_instance} ${ipendpoint_id} ${sr_id} ${influxdb_url} ${database_name}"
             lxc-attach -n ${service_name} -- ${cmd}
 

src/test/clmctest/services/nginx/telegraf_nginx.conf

@@ -30,11 +30,11 @@
   #pid_file = "/var/run/nginx.pid"
   exe = "nginx"
 
-# [[inputs.systemctl]]
-#   ## Service array
-#   services = [
-#     "nginx.service"
-#   ]
+[[inputs.systemctl]]
+  ## Service array
+  services = [
+    "nginx.service"
+  ]
 
 # # Influx HTTP write listener
 [[inputs.http_listener]]

src/test/clmctest/workshopdemo/README.md

-## FLAME media service integration workshop
+# FLAME media service integration workshop
 
+## Demonstration set-up notes
 
+### Prerequisites
 
-### Demonstration set-up notes
+### Create the Vagrant host and log in
 
-#### Create the Vagrant host and log in
-
-```
+```shell
 vagrant up
 vagrant ssh
 sudo su
 ```
 
-#### Start the containers and check
+### NATting (for Windows host)
 
-```
+* Chronograf: port 8888
+
+* 2 NGINX installations (/…/services/nginx)
+  * ports: 9180 and 9181
+
+* 2 MINIO installations (/…/services/minio)
+  * ports: 9182 and 9183
+
+### Create containers
+
+```shell
 /vagrant/scripts/test/fixture.sh create -f /vagrant/src/test/clmctest/rspec.json
+```
 
-lxc-ls
-> clmc-service host1        host2        host3        host4
+### Create all containers, leave nginx1 and clmc started
+
+```shell
+/vagrant/scripts/test/fixture.sh stop -f /vagrant/src/test/clmctest/rspec.json -c host2
+/vagrant/scripts/test/fixture.sh stop -f /vagrant/src/test/clmctest/rspec.json -c host3
+/vagrant/scripts/test/fixture.sh stop -f /vagrant/src/test/clmctest/rspec.json -c host4
 ```
 
-#### NATting (for Windows host)
+### Go to demonstration folder and re-configure NGINX container
 
-* Chronograf: port 8888
+```shell
+cd /vagrant/src/test/clmctest/workshopdemo/nginxMiniDemo
+chmod +x initDemo.sh
+./initDemo.sh
+```
 
-* 2 NGINX installations (/…/services/nginx)
-    - ports: 9180 and 9181
+### Delete MSDemo database on clmc-service
 
-* 2 MINIO installations (/…/services/minio)
-    - ports: 9182 and 9183
+???
+
+## Mini NGINX demonstration
+
+### Attach to container and install TELEGRAF (manually or otherwise)
+
+`lxc-attach -n host1`
+
+> Method for installing Telegraf is described in `/tmp/installTelegraf.sh`
+
+### Send a simple metric via HTTP listener (whilst in NGINX container)
+
+```shell
+apt-get install curl
+curl -i -XPOST 'http://172.40.231.51:8086/write?db=SimpleDemo' --data-binary 'MyServiceComponent requestCount=1'
+```
+
+This should return a 'HTTP/1.1 204 No Content' response
+
+### View data point in CLMC
+
+Go to `localhost:8888`...
+
+Connect to "SimpleDemo" database.
+
+## Main scenario demonstration
+
+### Start the containers and check
+
+```shell
+/vagrant/scripts/test/fixture.sh start -f /vagrant/src/test/clmctest/rspec.json -c host2
+/vagrant/scripts/test/fixture.sh start -f /vagrant/src/test/clmctest/rspec.json -c host3
+/vagrant/scripts/test/fixture.sh start -f /vagrant/src/test/clmctest/rspec.json -c host4
+lxc-ls
+> clmc-service host1        host2        host3        host4
+```
 
-#### Run demo initializer
+### Run demo initializer
 
 cd /vagrant/src/test/clmctest/workshopdemo
 chmod +x initDemo.sh
 ./initDemo.sh
 
-#### Check out the video data you've uploaded
+### Check out the video data you've uploaded
 
 MINIO 1
 http://localhost:9182/minio/democontent/
@@ -45,7 +97,7 @@ http://localhost:9182/minio/democontent/
 MINIO 2
 http://localhost:9183/minio/democontent/
 
-#### Stream some video
+### Stream some video
 
 Use the BitMovin' player: http://bitmovin.com/hls-mpeg-dash-test-player/
 
@@ -54,7 +106,7 @@ with these sources:
 http://localhost:9180/car-20120827-manifest.mpd
 http://localhost:9181/car-20120827-manifest.mpd
 
-#### Tear-down
+### Tear-down
 
 ```
 /vagrant/scripts/test/fixture.sh destroy -f /vagrant/src/test/clmctest/rspec.json

src/test/clmctest/workshopdemo/initDemo.sh

@@ -24,10 +24,15 @@
 #//
 #/////////////////////////////////////////////////////////////////////////
 
-# Run from Vagrant host
-# Prerequisites
+## Run from Vagrant host
+### Prerequisites
 apt-get install -y unzip
 
+# Mini NGINX demonstration
+
+
+
+# NGINX/MINIO demonstration
 mkdir -p /tmp/demoSetup
 cp * /tmp/demoSetup
 cd /tmp/demoSetup

src/test/clmctest/workshopdemo/nginxMiniDemo/initDemo.sh

+#!/bin/bash
+#/////////////////////////////////////////////////////////////////////////
+#//
+#// (c) University of Southampton IT Innovation Centre, 2017
+#//
+#// Copyright in this software belongs to University of Southampton
+#// IT Innovation Centre of Gamma House, Enterprise Road,
+#// Chilworth Science Park, Southampton, SO16 7NS, UK.
+#//
+#// This software may not be used, sold, licensed, transferred, copied
+#// or reproduced in whole or in part in any manner or form or in or
+#// on any media by any person other than in accordance with the terms
+#// of the Licence Agreement supplied with the software, or otherwise
+#// without the prior written consent of the copyright owners.
+#//
+#// This software is distributed WITHOUT ANY WARRANTY, without even the
+#// implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+#// PURPOSE, except where stated in the Licence Agreement supplied with
+#// the software.
+#//
+#//      Created By :            Simon Crowle
+#//      Created Date :          07/06/2018
+#//      Created for Project :   FLAME
+#//
+#/////////////////////////////////////////////////////////////////////////
+
+# Instantiate an NGINX container (host1)
+/vagrant/scripts/test/fixture.sh create -f /vagrant/src/test/clmctest/rspec.json -c host1
+
+# Uninstall TELEGRAF and clean up
+lxc-attach -n host1 -- service telegraf stop
+lxc-attach -n host1 -- dpkg -r telegraf
+lxc-attach -n host1 -- rm -rf /etc/telegraf
+
+# Copy in demonstration resources and TELEGRAF configuration (ready for editing)
+lxc-attach -n host1 -- mkdir -p /tmp/telegraf
+cp resources/installTelegraf.sh /var/lib/lxc/host1/rootfs/tmp/
+cp -r telegrafConfigs/* /var/lib/lxc/host1/rootfs/tmp/telegraf
+
+# Add REPORC for accessing our Telegraf.deb
+cp /vagrant/reporc /var/lib/lxc/host1/rootfs/tmp/
+
+# Drop MSDemo database on CLMC so that it is empty of other hosts
+# N.B. it will be created again automatically as soon as any host pushes data to it
+lxc-attach -n clmc-service -- influx -execute 'drop database MSDemo'
\ No newline at end of file

src/test/clmctest/workshopdemo/nginxMiniDemo/resources/installTelegraf.sh

+#!/bin/bash
+#/////////////////////////////////////////////////////////////////////////
+#//
+#// (c) University of Southampton IT Innovation Centre, 2017
+#//
+#// Copyright in this software belongs to University of Southampton
+#// IT Innovation Centre of Gamma House, Enterprise Road,
+#// Chilworth Science Park, Southampton, SO16 7NS, UK.
+#//
+#// This software may not be used, sold, licensed, transferred, copied
+#// or reproduced in whole or in part in any manner or form or in or
+#// on any media by any person other than in accordance with the terms
+#// of the Licence Agreement supplied with the software, or otherwise
+#// without the prior written consent of the copyright owners.
+#//
+#// This software is distributed WITHOUT ANY WARRANTY, without even the
+#// implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+#// PURPOSE, except where stated in the Licence Agreement supplied with
+#// the software.
+#//
+#//      Created By :            Simon Crowle
+#//      Created Date :          07/06/2018
+#//      Created for Project :   FLAME
+#//
+#/////////////////////////////////////////////////////////////////////////
+
+# Download TELEGRAF
+# ------------------------------------------------------------------------
+TELEGRAF_VERSION=1.7.0~5618bb0-0
+source /tmp/reporc
+wget --user ${REPO_USER} --password ${REPO_PASS} https://flame-nexus.it-innovation.soton.ac.uk/repository/flame-general/it-innovation/telegraf/${TELEGRAF_VERSION}/telegraf-${TELEGRAF_VERSION}.deb -O /tmp/telegraf.deb
+
+# Install Telegraf
+# ------------------------------------------------------------------------
+dpkg -i /tmp/telegraf.deb
+service telegraf stop
+
+# Copy TELEGRAF files into etc
+cp -r /tmp/telegraf/* /etc/telegraf/
+
+# Edit Telegraf configuration at your leisure...
+
+# Run telegraf
+#service telegraf start
\ No newline at end of file

src/test/clmctest/workshopdemo/nginxMiniDemo/telegrafConfigs/telegraf.conf

+#[global_tags]
+#  location="DC1"
+#  sfc="MSDemo"
+#  sfc_i="MSDemo_1"
+#  sf="nginx"
+#  sf_i="nginx_1"
+#  ipendpoint="nginx_1_ep1"
+#  sr="SR1"
+#[agent]
+#  interval = "10s"
+#  round_interval = true
+#  metric_buffer_limit = 1000
+#  flush_buffer_when_full = true
+#  collection_jitter = "0s"
+#  flush_interval = "10s"
+#  flush_jitter = "0s"
+#  debug = false
+#  quiet = false
+#  logfile = "/var/log/telegraf/telegraf.log"
+#  hostname = ""

src/test/clmctest/workshopdemo/nginxMiniDemo/telegrafConfigs/telegraf.d/telegraf_host.conf

+##Plugins to run with a little bit of configuration
+#[[inputs.cpu]]
+#  # Whether to report per-cpu stats or not
+#  percpu = true
+#  # Whether to report total system cpu stats or not
+#  totalcpu = true
+#  # If true, collect raw CPU time metrics.
+#  collect_cpu_time = false
+#  # If true, compute and report the sum of all non-idle CPU states.
+#  report_active = false
+
+#[[inputs.disk]]
+#  ignore_fs = ["tmpfs", "devtmpfs", "devfs"]
+
+#[[inputs.diskio]]
+#  devices = ["sda1"]
+
+## Plugins to run 'out-of-the-box' (no configuration applied)
+#[[inputs.kernel]]
+#[[inputs.mem]]
+#[[inputs.processes]]
+#[[inputs.swap]]
+#[[inputs.system]]
+#[[inputs.net]]
+#[[inputs.netstat]]
+
+## Influx HTTP write listener (for bespoke POSTing of your media service metrics)
+#[[inputs.http_listener]]
+#  # Address and port to host HTTP listener on
+#  service_address = ":8186"
\ No newline at end of file

src/test/clmctest/workshopdemo/nginxMiniDemo/telegrafConfigs/telegraf.d/telegraf_nginx.conf

+## © University of Southampton IT Innovation Centre, 2018
+##
+## Copyright in this software belongs to University of Southampton
+## IT Innovation Centre of Gamma House, Enterprise Road, 
+## Chilworth Science Park, Southampton, SO16 7NS, UK.
+##
+## This software may not be used, sold, licensed, transferred, copied
+## or reproduced in whole or in part in any manner or form or in or
+## on any media by any person other than in accordance with the terms
+## of the Licence Agreement supplied with the software, or otherwise
+## without the prior written consent of the copyright owners.
+##
+## This software is distributed WITHOUT ANY WARRANTY, without even the
+## implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+## PURPOSE, except where stated in the Licence Agreement supplied with
+## the software.
+##
+##      Created By :            Rowan Powell
+##      Created Date :          02-02-2018
+##      Created for Project :   FLAME
+
+# Read Nginx's basic status information (ngx_http_stub_status_module)
+#[[inputs.nginx]]
+  ## An array of Nginx stub_status URI to gather stats.
+  # urls = ["http://localhost:80/nginx_status"]
+  
+  ## HTTP response timeout (default: 5s)
+  #  response_timeout = "5s"
+
+#[[inputs.procstat]]
+  #pid_file = "/var/run/nginx.pid"
+  #exe = "nginx"
+
+# [[inputs.systemctl]]
+#   ## Service array
+#   services = [
+#     "nginx.service"
+#   ]
+
+#   ## Sample rate for sampling state of service.
+#   ## Must be greter that the collection_interval/2
+#   #sample_rate = 2
\ No newline at end of file

src/test/clmctest/workshopdemo/nginxMiniDemo/telegrafConfigs/telegraf.d/telegraf_output.conf

+# Output plugin to forward on metrics to INFLUX
+#[[outputs.influxdb]]
+#  urls = ["http://172.40.231.51:8086"]
+#  database = "MSDemo"
+#  precision = "s"
+#  timeout = "5s"