Merge branch 'fms-mon' into 'integration'

Fms mon

See merge request FLAME/flame-clmc!9

Vagrantfile

@@ -55,7 +55,7 @@ hosts = YAML.load_file(host_rspec_file)
 # Start creating VMS using xenial64 as the base box
 Vagrant.configure("2") do |config|
   config.vm.box = "ubuntu/xenial64"
- 
+  #config.vm.box = "hashicorp/precise32"
 
   # Dynamic VMs
   hosts['hosts'].each do |host|
@@ -72,13 +72,13 @@ Vagrant.configure("2") do |config|
       end
 
       # Configure network, not that we only expect 1 test to be running so we have one internal network
-      config.vm.network :private_network, ip: "#{host["ip_address"]}", virtualbox__intnet: "clmc-net"
+      instance_config.vm.network :private_network, ip: "#{host["ip_address"]}", virtualbox__intnet: "clmc-net"
 
       # Port forwarding
       puts "Forwarding the following specified ports for #{host["name"]}:"
       host['forward_ports'].each do |port|
         puts "Forwarding guest:#{port["guest"]} => host:#{port["host"]}"
-        config.vm.network "forwarded_port", guest: port["guest"], host: port["host"]
+        instance_config.vm.network "forwarded_port", guest: port["guest"], host: port["host"]
       end
       
       # Switch case added here to make clmc-service provisioning simple without having to have a complex rspec.yml file
@@ -87,15 +87,15 @@ Vagrant.configure("2") do |config|
       puts "Instance name #{instance_name}:"
       case instance_name
         when 'clmc-service'
-          config.vm.provision :shell, :path => "scripts/clmc-service/#{host["install_script"]}"
-          config.vm.provision :shell, :path => "scripts/clmc-service/#{host["start_script"]}" 
+          instance_config.vm.provision :shell, :path => "scripts/clmc-service/#{host["install_script"]}"
+          instance_config.vm.provision :shell, :path => "scripts/clmc-service/#{host["start_script"]}" 
         else
           # specific service install
           service_install_path = "test/services/#{host["service_name"]}/install-#{host["service_name"]}.sh"
           puts "installing service script: #{service_install_path}"
-          config.vm.provision :shell, :path => service_install_path      
+          instance_config.vm.provision :shell, :path => service_install_path      
           # agent install
-          config.vm.provision :shell, :path => "scripts/clmc-agent/install-clmc-agent.sh", :args => "/vagrant/test/services/#{host["service_name"]}/telegraf_#{host["service_name"]}_template.conf #{host["location"]} #{host["sfc_id"]} #{host["sfc_id_instance"]} #{host["sf_id"]} #{host["sf_id_instance"]} #{host["ipendpoint_id"]} #{host["influxdb_url"]} #{host["database_name"]}"
+          instance_config.vm.provision :shell, :path => "scripts/clmc-agent/install-clmc-agent.sh", :args => "/vagrant/test/services/#{host["service_name"]}/telegraf_#{host["service_name"]}_template.conf #{host["location"]} #{host["sfc_id"]} #{host["sfc_id_instance"]} #{host["sf_id"]} #{host["sf_id_instance"]} #{host["ipendpoint_id"]} #{host["influxdb_url"]} #{host["database_name"]}"
 
       end
 	  end

infra/full/rspec.yml

+hosts:
+  - name: clmc-service
+    cpus: 1
+    memory: 2048
+    disk: "10GB"
+    forward_ports:
+      - guest: 8086
+        host: 8086
+      - guest: 8888
+        host: 8888
+      - guest: 9092
+        host: 9092
+    ip_address: "192.168.50.10"
+    install_script: "install-clmc-service.sh"
+    start_script: "start-clmc-service.sh"
+  - name: apache1
+    cpus: 1
+    memory: 2048
+    disk: "10GB"
+    service_name: "apache"
+    forward_ports:
+      - guest: 80
+        host: 8081
+    ip_address: "192.168.50.11"
+    location: "DC1"
+    sfc_id: "MS_Template_1"
+    sfc_id_instance: "MS_I1"
+    sf_id: "adaptive_streaming"
+    sf_id_instance: "adaptive_streaming_I1"
+    ipendpoint_id: "adaptive_streaming_I1_apache1"
+    influxdb_url: "http://192.168.50.10:8086"
+    database_name: "CLMCMetrics"
+  - name: apache2
+    cpus: 1
+    memory: 2048
+    disk: "10GB"
+    service_name: "apache"
+    forward_ports:
+      - guest: 80
+        host: 8082
+    ip_address: "192.168.50.12"
+    location: "DC2"
+    sfc_id: "MS_Template_1"
+    sfc_id_instance: "MS_I1"
+    sf_id: "adaptive_streaming"
+    sf_id_instance: "adaptive_streaming_I1"
+    ipendpoint_id: "adaptive_streaming_I1_apache2"
+    influxdb_url: "http://192.168.50.10:8086"
+    database_name: "CLMCMetrics"      
+    
+  - name: nginx
+    cpus: 1
+    memory: 2048
+    disk: "10GB"
+    service_name: "nginx"
+    forward_ports:
+      - guest: 80
+        host: 8083
+    ip_address: "192.168.50.13"
+    location: "DC1"
+    sfc_id: "MS_Template_1"
+    sfc_id_instance: "MS_I1"
+    sf_id: "adaptive_streaming"
+    sf_id_instance: "adaptive_streaming_nginx_I1"
+    ipendpoint_id: "adaptive_streaming_nginx_I1_apache1"
+    influxdb_url: "http://192.168.50.10:8086"
+    database_name: "CLMCMetrics"
+  - name: mongo
+    cpus: 1
+    memory: 2048
+    disk: "10GB"
+    service_name: "mongo"
+    forward_ports:
+      - guest: 80
+        host: 8084
+    ip_address: "192.168.50.14"
+    location: "DC1"
+    sfc_id: "MS_Template_1"
+    sfc_id_instance: "MS_I1"
+    sf_id: "metadata_database"
+    sf_id_instance: "metadata_database_I1"
+    ipendpoint_id: "metadata_database_I1_apache1"
+    influxdb_url: "http://192.168.50.10:8086"
+    database_name: "CLMCMetrics" 
+  - name: ffmpeg
+    cpus: 1
+    memory: 2048
+    disk: "10GB"
+    service_name: "ffmpeg"
+    forward_ports:
+      - guest: 80
+        host: 8085
+    ip_address: "192.168.50.14"
+    location: "DC1"
+    sfc_id: "MS_Template_1"
+    sfc_id_instance: "MS_I1"
+    sf_id: "metadata_database"
+    sf_id_instance: "metadata_database_I1"
+    ipendpoint_id: "metadata_database_I1_apache1"
+    influxdb_url: "http://192.168.50.10:8086"
+    database_name: "CLMCMetrics" 

infra/streaming/rspec.yml

@@ -30,55 +30,3 @@ hosts:
     ipendpoint_id: "adaptive_streaming_I1_apache1"
     influxdb_url: "http://192.168.50.10:8086"
     database_name: "CLMCMetrics"
-  - name: apache2
-    cpus: 1
-    memory: 2048
-    disk: "10GB"
-    service_name: "apache"
-    forward_ports:
-      - guest: 80
-        host: 8082
-    ip_address: "192.168.50.12"
-    location: "DC2"
-    sfc_id: "MS_Template_1"
-    sfc_id_instance: "MS_I1"
-    sf_id: "adaptive_streaming"
-    sf_id_instance: "adaptive_streaming_I1"
-    ipendpoint_id: "adaptive_streaming_I1_apache2"
-    influxdb_url: "http://192.168.50.10:8086"
-    database_name: "CLMCMetrics"      
-    
-  - name: nginx
-    cpus: 1
-    memory: 2048
-    disk: "10GB"
-    service_name: "nginx"
-    forward_ports:
-      - guest: 80
-        host: 8083
-    ip_address: "192.168.50.13"
-    location: "DC1"
-    sfc_id: "MS_Template_1"
-    sfc_id_instance: "MS_I1"
-    sf_id: "adaptive_streaming"
-    sf_id_instance: "adaptive_streaming_nginx_I1"
-    ipendpoint_id: "adaptive_streaming_nginx_I1_apache1"
-    influxdb_url: "http://192.168.50.10:8086"
-    database_name: "CLMCMetrics"
-  - name: mongo
-    cpus: 1
-    memory: 2048
-    disk: "10GB"
-    service_name: "mongo"
-    forward_ports:
-      - guest: 80
-        host: 8084
-    ip_address: "192.168.50.14"
-    location: "DC1"
-    sfc_id: "MS_Template_1"
-    sfc_id_instance: "MS_I1"
-    sf_id: "metadata_database"
-    sf_id_instance: "metadata_database_I1"
-    ipendpoint_id: "metadata_database_I1_apache1"
-    influxdb_url: "http://192.168.50.10:8086"
-    database_name: "CLMCMetrics" 

scripts/clmc-agent/install-clmc-agent.sh

@@ -41,7 +41,7 @@ IP_ENDPOINT_ID=$7
 INFLUXDB_URL=$8
 DATABASE_NAME=$9
 
-if [ ! -f $TELEGRAF_CONF_FILE]; then
+if [ ! -f $TELEGRAF_CONF_FILE ]; then
     echo "Error: Telegraf conf template file not found: "$TELEGRAF_CONF_FILE
     exit
 fi

test/services/apache/install-apache.sh

@@ -26,4 +26,12 @@
 
 # Install apache
 sudo apt-get update
-sudo apt-get -y install apache2
\ No newline at end of file
+sudo apt-get -y install apache2 zip
+
+# Copy files for MPEG-DASH testing
+curl "ftp://ftp.it-innovation.soton.ac.uk/testdata/video/20180210-flame-project-video.zip" --user flame-rw:DR8ngj3ogSjd8gl -o "/var/www/html/20180210-flame-project-video.zip"
+unzip /var/www/html/20180210-flame-project-video.zip -d /var/www/html/
+
+# start apache
+apachectl -k start
+apachectl -k restart

test/services/apache/telegraf_apache_template.conf

@@ -58,7 +58,7 @@
   ## Run telegraf in quiet mode
   quiet = false
   ## Specify the log file name. The empty string means to log to stdout.
-  logfile = "G:/Telegraf/telegraf.log"
+  logfile = "/var/log/telegraf/telegraf.log"
 
   ## Override default hostname, if empty use os.Hostname()
   hostname = ""

test/services/ffmpeg/install-ffmpeg.sh

+#!/bin/bash
+
+sudo apt-get update
+sudo apt-get -y install zip python
+
+wget http://zebulon.bok.net/Bento4/binaries/Bento4-SDK-1-5-1-621.x86_64-unknown-linux.zip
+unzip Bento4-SDK-1-5-1-621.x86_64-unknown-linux.zip
+
+mv Bento4-SDK-1-5-1-621.x86_64-unknown-linux /opt/ 
+rm Bento4-SDK-1-5-1-621.x86_64-unknown-linux.zip
+
+sudo add-apt-repository ppa:jonathonf/ffmpeg-3
+sudo apt-get update && sudo apt -y install ffmpeg libav-tools x264 x265
+
+ffmpeg -version

test/services/ffmpeg/telegraf_ffmpeg_template.conf

+# Telegraf configuration
+
+# Telegraf is entirely plugin driven. All metrics are gathered from the
+# declared inputs, and sent to the declared outputs.
+
+# Plugins must be declared in here to be active.
+# To deactivate a plugin, comment out the name and any variables.
+
+# Use 'telegraf -config telegraf.conf -test' to see what metrics a config
+# file would generate.
+
+# Global tags can be specified here in key="value" format.
+[global_tags]
+  # location of the data centre
+  location="{{LOCATION}}"
+  # media service template id
+  sfc="{{SFC_ID}}"
+  # media service instance
+  sfc_i="{{SFC_ID_INSTANCE}}"
+  # service function type
+  sf="{{SF_ID}}"
+  # service function instance id
+  sf_i="{{SF_ID_INSTANCE}}"
+  # ipendpoint id aka surrogate instance
+  ipendpoint="{{IP_ENDPOINT_ID}}"
+
+# Configuration for telegraf agent
+[agent]
+  ## Default data collection interval for all inputs
+  interval = "10s"
+  ## Rounds collection interval to 'interval'
+  ## ie, if interval="10s" then always collect on :00, :10, :20, etc.
+  round_interval = true
+
+  ## Telegraf will cache metric_buffer_limit metrics for each output, and will
+  ## flush this buffer on a successful write.
+  metric_buffer_limit = 1000
+  ## Flush the buffer whenever full, regardless of flush_interval.
+  flush_buffer_when_full = true
+
+  ## Collection jitter is used to jitter the collection by a random amount.
+  ## Each plugin will sleep for a random time within jitter before collecting.
+  ## This can be used to avoid many plugins querying things like sysfs at the
+  ## same time, which can have a measurable effect on the system.
+  collection_jitter = "0s"
+
+  ## Default flushing interval for all outputs. You shouldn't set this below
+  ## interval. Maximum flush_interval will be flush_interval + flush_jitter
+  flush_interval = "10s"
+  ## Jitter the flush interval by a random amount. This is primarily to avoid
+  ## large write spikes for users running a large number of telegraf instances.
+  ## ie, a jitter of 5s and interval 10s means flushes will happen every 10-15s
+  flush_jitter = "0s"
+
+  ## Logging configuration:
+  ## Run telegraf in debug mode
+  debug = false
+  ## Run telegraf in quiet mode
+  quiet = false
+  ## Specify the log file name. The empty string means to log to stdout.
+  logfile = "G:/Telegraf/telegraf.log"
+
+  ## Override default hostname, if empty use os.Hostname()
+  hostname = ""
+
+
+###############################################################################
+#                                  OUTPUTS                                    #
+###############################################################################
+
+# Configuration for influxdb server to send metrics to
+[[outputs.influxdb]]
+  # The full HTTP or UDP endpoint URL for your InfluxDB instance.
+  # Multiple urls can be specified but it is assumed that they are part of the same
+  # cluster, this means that only ONE of the urls will be written to each interval.
+  # urls = ["udp://127.0.0.1:8089"] # UDP endpoint example
+  urls = ["{{INFLUXDB_URL}}"] # required
+  # The target database for metrics (telegraf will create it if not exists)
+  database = "{{DATABASE_NAME}}" # required
+  # Precision of writes, valid values are "ns", "us" (or "µs"), "ms", "s", "m", "h".
+  # note: using second precision greatly helps InfluxDB compression
+  precision = "s"
+
+  ## Write timeout (for the InfluxDB client), formatted as a string.
+  ## If not provided, will default to 5s. 0s means no timeout (not recommended).
+  timeout = "5s"
+  # username = "telegraf"
+  # password = "metricsmetricsmetricsmetrics"
+  # Set the user agent for HTTP POSTs (can be useful for log differentiation)
+  # user_agent = "telegraf"
+  # Set UDP payload size, defaults to InfluxDB UDP Client default (512 bytes)
+  # udp_payload = 512
+
+
+###############################################################################
+#                                  INPUTS                                     #
+###############################################################################
+# # Influx HTTP write listener
+[[inputs.http_listener]]
+  ## Address and port to host HTTP listener on
+  service_address = ":8186"
+
+  ## timeouts
+  read_timeout = "10s"
+  write_timeout = "10s"
+
+  ## HTTPS
+  #tls_cert= "/etc/telegraf/cert.pem"
+  #tls_key = "/etc/telegraf/key.pem"
+
+  ## MTLS
+  #tls_allowed_cacerts = ["/etc/telegraf/clientca.pem"]
\ No newline at end of file

test/services/ffmpeg/transcode.sh

+#!/bin/bash
+#
+# Bento4 must be manually obtained from https://www.bento4.com/downloads/ ffmpeg can be installed as follows 
+# (Debian - http://www.deb-multimedia.org/) ~$ 
+# sudo echo deb http://www.deb-multimedia.org jessie main non-free >> /etc/apt/sources.list
+# ~$ sudo apt update ~$ sudo apt install deb-multimedia-keyring ~$ sudo apt update && sudo apt-get dist-upgrade
+#
+# First encode the video to 24fps!!! and MP4 (h.264)
+#
+# Video MP4 file
+INPUT=$1
+OUTPUT_iFRAMES="$1-iFrames.mp4"
+OUTPUT_FRAGMENTED="$OUTPUT_iFRAMES-Fragmented.mp4"
+
+rm -rf output
+
+# Insert Correct number of I frames
+#ffmpeg -y -i $INPUT -profile:v baseline -level 3.0 -c:a libfdk_aac -ac 2 -ab 128k -c:v libx264 -x264opts 'keyint=24:min-keyint=24:no-scenecut' -b:v 400k -maxrate 400k -bufsize 1000k -vf "scale=-1:360" $OUTPUT_iFRAMES
+
+ffmpeg -y -i $INPUT -profile:v baseline -level 3.0 -c:a aac -ac 2 -ab 128k -c:v libx264 -x264opts 'keyint=24:min-keyint=24:no-scenecut' -b:v 400k -maxrate 400k -bufsize 1000k -vf "scale=-1:360" -strict experimental $OUTPUT_iFRAMES
+
+# fragment MP4
+/opt/Bento4-SDK-1-5-1-621.x86_64-unknown-linux/bin/mp4fragment --timescale 1000 $OUTPUT_iFRAMES $OUTPUT_FRAGMENTED
+
+# Option 1 with Bento4
+/opt/Bento4-SDK-1-5-1-621.x86_64-unknown-linux/bin/mp4dash --mpd-name=stream.mpd --use-segment-list --use-compat-namespace $OUTPUT_FRAGMENTED 
+
+# not sure what this is for
+mv output/audio/en/mp4a/* output/audio/en
+
+rm -rf output/audio/en/mp4

test/services/mongo/install-mongo.sh

@@ -30,3 +30,4 @@ sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 2930ADAE8CAF50
 echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu xenial/mongodb-org/3.6 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-3.6.list
 sudo apt-get update
 sudo apt-get install -y mongodb-org
+sudo service mongod start
\ No newline at end of file

test/services/nginx/install-nginx.sh

@@ -30,8 +30,23 @@ yes Y | sudo apt-get install nginx
 
 # Need to set up basic stats as this not configured by default
 # http://nginx.org/en/docs/http/ngx_http_stub_status_module.html
+echo "Coping new config file into nginx"
+cd /vagrant
+sudo rm ../etc/nginx/nginx.conf
+sudo cp test/services/nginx/nginx.conf ../etc/nginx/nginx.conf
+#cat test/services/nginx/nginx.conf > ../etc/nginx/nginx.conf
 
-nginx -s reload
+
+sudo nginx -s reload
 
 # start NGINX
-systemctl start nginx
\ No newline at end of file
+sudo systemctl start nginx
+# --with-http_stub_status_module
+#nginx -V 2>&1 | grep -o with-http_stub_status_module
+#/vagrant/../etc/nginx/nginx.config
+
+# systemctl status telegraf -l
+
+
+
+# dpkg -l | grep -c nginx 
\ No newline at end of file

test/services/nginx/nginx.conf

+
+
+events {
+  worker_connections  4096;  ## Default: 1024
+}
+http {
+	server {
+		location /nginx_status {
+		  stub_status on;
+		  access_log   off;
+		  allow all;
+	  }
+	}
+}
\ No newline at end of file

test/services/nginx/telegraf_nginx_template.conf

@@ -58,7 +58,7 @@
   ## Run telegraf in quiet mode
   quiet = false
   ## Specify the log file name. The empty string means to log to stdout.
-  logfile = "G:/Telegraf/telegraf.log"
+  logfile = "/etc/telegraf/telegraf.log"
 
   ## Override default hostname, if empty use os.Hostname()
   hostname = ""
@@ -91,7 +91,6 @@
   # Set UDP payload size, defaults to InfluxDB UDP Client default (512 bytes)
   # udp_payload = 512
 
-
 ###############################################################################
 #                                  INPUTS                                     #
 ###############################################################################
@@ -104,24 +103,10 @@
   read_timeout = "10s"
   write_timeout = "10s"
 
-  ## HTTPS
-  #tls_cert= "/etc/telegraf/cert.pem"
-  #tls_key = "/etc/telegraf/key.pem"
-
-  ## MTLS
-  #tls_allowed_cacerts = ["/etc/telegraf/clientca.pem"]
-
 # Read Nginx's basic status information (ngx_http_stub_status_module)
 [[inputs.nginx]]
   ## An array of Nginx stub_status URI to gather stats.
-  urls = ["http://localhost/server_status"]
-
-  ## Optional SSL Config
-  # ssl_ca = "/etc/telegraf/ca.pem"
-  # ssl_cert = "/etc/telegraf/cert.pem"
-  # ssl_key = "/etc/telegraf/key.pem"
-  ## Use SSL but skip chain & host verification
-  # insecure_skip_verify = false
+  urls = ["http://localhost:80/nginx_status"]
 
   ## HTTP response timeout (default: 5s)
-  response_timeout = "5s"
\ No newline at end of file
+#  response_timeout = "5s"
\ No newline at end of file

test/streaming/manual.md

+<!--
+// © University of Southampton IT Innovation Centre, 2017
+//
+// Copyright in this software belongs to University of Southampton
+// IT Innovation Centre of Gamma House, Enterprise Road, 
+// Chilworth Science Park, Southampton, SO16 7NS, UK.
+//
+// This software may not be used, sold, licensed, transferred, copied
+// or reproduced in whole or in part in any manner or form or in or
+// on any media by any person other than in accordance with the terms
+// of the Licence Agreement supplied with the software, or otherwise
+// without the prior written consent of the copyright owners.
+//
+// This software is distributed WITHOUT ANY WARRANTY, without even the
+// implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE, except where stated in the Licence Agreement supplied with
+// the software.
+//
+//      Created By :            Michael Boniface
+//      Created Date :          18-12-2017
+//      Created for Project :   FLAME
+-->
+
+# CLMC Streaming Test
+
+Install VLC video client, used a nightly build
+
+https://nightlies.videolan.org/build/win32/vlc-3.0.0-rc1-20171201-0326/vlc-3.0.0-20171201-0326-rc1-win32.exe
+
+`vagrant --infra=streaming up`
+
+Start the VLC Player
+
+`Media->Open Network Stream`
+
+`Enter the network URL: http://localhost:8890/flame-video/stream.mpd`