diff --git a/.gitattributes b/.gitattributes index 3e6943e574d902da76f56b877fd762e1a2de7d6c..bec2c414fdb06c08082df464c48c057083f8766f 100644 --- a/.gitattributes +++ b/.gitattributes @@ -2,6 +2,7 @@ * text=auto *.txt text *.sh text eol=lf +reporc text eol=lf scripts/* text eol=lf # Denote all files that are truly binary and should not be modified. *.png binary diff --git a/.gitignore b/.gitignore index 45e9d09527913a22e8f320a0993076ea1f16546c..6430516993f3cbd8987e44aaa002104e4d830ff7 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,6 @@ *egg-info* *git-commit-ref* *_version.py* -*reporc ubuntu-xenial-16.04-cloudimg-console.log .idea/ *.egg @@ -14,3 +13,4 @@ ubuntu-xenial-16.04-cloudimg-console.log *$py.class **/.pytest_cache/ build/ + diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 310ff38104fd0d6c8b58a7aa73a66bf7b04763ab..f8311bce80c074c64f6260e68cf9c5a4f572e1bc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -27,11 +27,11 @@ build:tests: stage: build only: - schedules - script: + script: - cd src/test - python setup.py sdist --dist-dir=../../build - cd ../../src/service - - python setup.py sdist --dist-dir=../../build + - python setup.py sdist --dist-dir=../../build artifacts: paths: - build/clmctest-SNAPSHOT.tar.gz @@ -45,14 +45,14 @@ test:all: dependencies: - build:tests script: - - echo "REPO_USER=${REPO_USER}" >> reporc - - echo "REPO_PASS=${REPO_PASS}" >> reporc +# - echo "REPO_USER=${REPO_USER}" > reporc +# - echo "REPO_PASS=${REPO_PASS}" >> reporc - vagrant --fixture=scripts -- up - vagrant --fixture=scripts -- ssh test-runner -- -tt "pip3 install /vagrant/build/clmctest-SNAPSHOT.tar.gz" - vagrant --fixture=scripts -- ssh test-runner -- -tt "pytest -s --pyargs clmctest.scripts" - vagrant --fixture=monitoring -- up + - vagrant --fixture=monitoring -- ssh test-runner -- -tt "pip3 install /vagrant/build/clmcservice-SNAPSHOT.tar.gz" - vagrant --fixture=monitoring -- ssh test-runner -- -tt "pytest -s --pyargs clmctest.monitoring" - - vagrant --fixture=monitoring -- ssh test-runner -- -tt "pip3 install /vagrant/build/clmcservice-SNAPSHOT.tar.gz" - vagrant --fixture=monitoring -- ssh test-runner -- -tt "pytest -s --pyargs clmcservice.tests" - vagrant --fixture=inputs -- up - vagrant --fixture=inputs -- ssh test-runner -- -tt "pytest -s --pyargs clmctest.inputs" @@ -63,5 +63,5 @@ clean: only: - schedules script: - - python scripts/build/deleteallvms.py + - python scripts/test/deleteallvms.py when: always \ No newline at end of file diff --git a/README.md b/README.md index 05f8934137f2f05f9519ff0c6b02eb0cda64c205..405b567128d1d969c767cc7691953bb93e6448cd 100644 --- a/README.md +++ b/README.md @@ -37,71 +37,62 @@ #### Documentation -Implementation documentation and discussion can be found in the [FLAME CLMC Information Model Specification](docs/clmc-information-model.md) - -#### Development Environment - -The development environment is currently Vagrant/Virtualbox providing the infrastructure for dev and testing. - -Vagrant-disksize plugin must be installed. - -`vagrant plugin install vagrant-disksize` +Implementation documentation and discussion can be found in the docs directory. #### Testing Testing is implemented using pytest using the following convention: +* The testing environment is Vagrant/Virtualbox for a base VM with LXC is installed for specific containers. * Tests are written in python using pytest -* Related tests are stored in a python module `clmctest/<testmodule>` to create a suite of tests. All tests are stored in files test_*.py, there can be many tests per file, and many files per module -* Each test module has a rspec.yml that provides the baseline "fixture" for the tests in the module -* Tests are executed against fixtures. Fixtures are modular "setups" created for a test, that are inserted into the python code using dependancy injection. This offers more flexibility than the unit style testing. The baseline deployment is created using `vagrant up` with an appropriate rspec, and the pytest fixture reads the rspec.yml and makes the configuration available to the test. -* Tests are executed from a guest VM (not the host) in the repo root using the command `pytest test/<testmodule>` -* Pytest will scan the directory for all tests including in files test_*.py and run them - -#### Creating a deployment for a test - -Each test has a fixture defining a set of VMs required for the test. The fixture is described in an rspec.yml file that is stored within the clmctest/<testmoduel> directory. For example, the monitoring test fixture is stored in the following file. - -`clmctest\monitoring\rspec.yml` - -The fixture is created by running the vagrant up command with the test module name as a parameter - -`vagrant --fixture=monitoring -- up` - -This will provision the following VMs clmc-service, ipendpoint1, ipendpoint2 - -The **clmc-service** vm includes influx, Kapacitor and Chronograf. The following ports forwarded to the clmc VM from the host machine are as follows: +* Related tests are stored in a python module `src/test/clmctest/<testmodule>` to create a suite of tests. All tests are stored in files test_*.py, there can be many tests per file, and many files per module +* Tests are executed against a set of LXC containers described in `src/test/clmctest/rspec.json`. +* Tests are executed from the test-runner container on the VM using install python modules -* Influx: 8086 -* Chronograf: 8888 -* Kapacitor: 9092 +Here's the instructions -Typically, tests are run from a test-runner VM that has pytest installed. The script used to install pytest is +`vagrant up` -`clmctest/services/pytest/install.sh` +This will create a single VM with LXC installed and configured with lxcbr0 configured for the network 172.40.231.0/24 -To run a test, SSH into the test runner machine +SSH into the VM -`vagrant --fixture=monitoring -- ssh test-runner` +`vagrant ssh` -Then go to the 'vagrant' directory. +Create the services needed for integration tests -`cd /vagrant` +``` +sudo su +/vagrant/scripts/test/conts-create.sh +``` -Run the pytest. +All of the containers created are defined in the file `/vagrant/src/test/clmctest/rspec.json` -`pytest -s clmctest/monitoring/` +Attach to the test-runner -#### Tests run from the host +`lxc-attach -n test-runner` -Tests can be run from the host by sshing commands to the test running. This is described in the .gitlab-ci.yml file. The file shows how a python package is created +Here we need to define a make file but for now the commands are manual -`python setup.py sdist --dist-dir=build` +``` +cd /vagrant/src/test +python setup.py sdist --dist-dir=../../build +cd ../../src/service +python setup.py sdist --dist-dir=../../build +pip3 install /vagrant/build/clmctest-SNAPSHOT.tar.gz +pip3 install /vagrant/build/clmcservice-SNAPSHOT.tar.gz +``` -Then the package is installed +The following modules are unit tests -`vagrant --fixture=scripts -- ssh test-runner -- -tt "pip3 install /vagrant/build/clmctest-SNAPSHOT.tar.gz"` +``` +pytest -s --pyargs clmctest.scripts +pytest -s --pyargs clmcservice.tests +``` -Then the tests are run +The following modules are integration tests -`vagrant --fixture=scripts -- ssh test-runner -- -tt "pytest -s --pyargs clmctest.scripts"` +``` +pytest -s --pyargs clmctest.inputs +pytest -s --pyargs clmctest.monitoring +``` diff --git a/Vagrantfile b/Vagrantfile index 231a03d3990803ed3d887c0b4cc73f45fa4f13b3..3840503cc55a8f85552ffda648c524e4991a3605 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -1,127 +1,25 @@ -#///////////////////////////////////////////////////////////////////////// -#// -#// (c) University of Southampton IT Innovation Centre, 2017 -#// -#// Copyright in this software belongs to University of Southampton -#// IT Innovation Centre of Gamma House, Enterprise Road, -#// Chilworth Science Park, Southampton, SO16 7NS, UK. -#// -#// This software may not be used, sold, licensed, transferred, copied -#// or reproduced in whole or in part in any manner or form or in or -#// on any media by any person other than in accordance with the terms -#// of the Licence Agreement supplied with the software, or otherwise -#// without the prior written consent of the copyright owners. -#// -#// This software is distributed WITHOUT ANY WARRANTY, without even the -#// implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR -#// PURPOSE, except where stated in the Licence Agreement supplied with -#// the software. -#// -#// Created By : Michael Boniface -#// Created Date : 13/12/2017 -#// Created for Project : FLAME -#// -#///////////////////////////////////////////////////////////////////////// -# Requirements -require 'getoptlong' -require 'yaml' +$lxc_script = <<-SCRIPT -# Custom options: -# --fixture <fixturedir> +apt-get update +apt-get install lxc lxc-templates wget bridge-utils jq -y +lxc-checkconfig -# Set defaults -DEFAULT_FIXTURE = "streaming" +touch /etc/lxc/dnsmasq.conf +sed -i s/10.0.3/172.40.231/g /etc/default/lxc-net +sed -i s/#LXC_DHCP_CONFILE/LXC_DHCP_CONFILE/g /etc/default/lxc-net +service lxc-net restart -# Define custom options -opts = GetoptLong.new( - [ '--fixture', GetoptLong::OPTIONAL_ARGUMENT] -) +SCRIPT -# Retrieve custom option values -fixture = DEFAULT_FIXTURE -opts.each do |opt, arg| - case opt - when '--fixture' - fixture = arg - end -end - -# load custom config file -puts "loading custom infrastructure configuration: #{fixture}" -puts "custom config file: /src/test/clmctest/#{fixture}/rspec.yml" -host_rspec_file = "src/test/clmctest/#{fixture}/rspec.yml" -hosts = YAML.load_file(host_rspec_file) - -# Start creating VMS using xenial64 as the base box Vagrant.configure("2") do |config| config.vm.box = "ubuntu/xenial64" - #config.vm.box = "hashicorp/precise32" - - # Dynamic VMs - hosts['hosts'].each do |host| - #p host["name"] - instance_name = host["name"] - config.vm.define instance_name do |instance_config| - - # Specify VM properties - instance_config.vm.hostname = instance_name - instance_config.disksize.size = host["disk"] - instance_config.vm.provider "virtualbox" do |v| - v.customize ["modifyvm", :id, "--memory", host["memory"]] - v.customize ["modifyvm", :id, "--cpus", host["cpus"]] - end - - # Configure network, not that we only expect 1 test to be running so we have one internal network - instance_config.vm.network :private_network, ip: "#{host["ip_address"]}", virtualbox__intnet: "clmc-net" - - # Port forwarding - puts "Forwarding the following specified ports for #{host["name"]}:" - if host.has_key? 'forward_ports' - host['forward_ports'].each do |port| - puts "Forwarding guest:#{port["guest"]} => host:#{port["host"]}" - instance_config.vm.network "forwarded_port", guest: port["guest"], host: port["host"] - end - end - - # Switch case added here to make clmc-service provisioning simple without having to have a complex rspec.yml file - # We only run a service installation script and the agent installation script when creating a specific service VM, not the clmc-service VM - - puts "Instance name #{instance_name}:" - case instance_name - when 'test-runner' - instance_config.vm.provision :shell, :path => "src/test/clmctest/services/pytest/install.sh" - when 'clmc-service' - instance_config.vm.provision :shell, :path => "scripts/clmc-service/install.sh", :args => "#{host["influxdb_url"]} #{host["database_name"]} #{host["report_period"]}", env: {"REPO_ROOT" => "/vagrant"} - else - # specific service install - instance_config.vm.provision :shell, :path => "src/test/clmctest/services/#{host["service_name"]}/install.sh", env: {"REPO_ROOT" => "/vagrant"} - - # CLMC agent install - instance_config.vm.provision "file", source: "reporc", destination: "/vagrant/reporc" - instance_config.vm.provision :shell, :path => "scripts/clmc-agent/install.sh", env: {"REPO_ROOT" => "/vagrant"} - - # CLMC agent service specific input configuration - instance_config.vm.provision :shell, inline: <<-SHELL - - cp /vagrant/scripts/clmc-agent/telegraf.conf /etc/telegraf/ - - cp /vagrant/scripts/clmc-agent/telegraf_output.conf /etc/telegraf/telegraf.d/ - - cp /vagrant/src/test/clmctest/services/#{host["service_name"]}/telegraf_#{host["service_name"]}.conf /etc/telegraf/telegraf.d/ - - SHELL - - # CLMC agent general and output configuration - #instance_config.vm.provision :shell, :path => "scripts/clmc-agent/configure_template.sh" - - instance_config.vm.provision :shell, :path => "scripts/clmc-agent/configure.sh", :args => "#{host["location"]} #{host["sfc_id"]} #{host["sfc_id_instance"]} #{host["sf_id"]} #{host["sf_id_instance"]} #{host["ipendpoint_id"]} #{host["sr_id"]} #{host["influxdb_url"]} #{host["database_name"]}" + config.vm.provider "virtualbox" do |vb| + vb.cpus = 4 + vb.memory = "8192" + end - # CLMC start agent - instance_config.vm.provision :shell, inline: "service telegraf restart" - end + #config.vm.network "forwarded_port", guest: 80, host: 8080 - - end - end - + # Install lxc + config.vm.provision :shell, inline: $lxc_script end diff --git a/reporc b/reporc new file mode 100644 index 0000000000000000000000000000000000000000..55ddc3e14ff38e011c8f9566e068528be60ac8a8 --- /dev/null +++ b/reporc @@ -0,0 +1,2 @@ +REPO_USER=itinnov.flame.integration +REPO_PASS=1tc3ntr3 \ No newline at end of file diff --git a/scripts/clmc-agent/build-telegraf.sh b/scripts/clmc-agent/build-telegraf.sh old mode 100644 new mode 100755 diff --git a/scripts/clmc-agent/configure.sh b/scripts/clmc-agent/configure.sh index 2c53091f5dd118e825ad842cd74356db51b95f31..35226d643103e9fbcdf2c5799c371f33ded9791d 100755 --- a/scripts/clmc-agent/configure.sh +++ b/scripts/clmc-agent/configure.sh @@ -51,7 +51,7 @@ TELEGRAF_CONF_FILE=$TELEGRAF_CONF_DIR"/telegraf.conf" TELEGRAF_INCLUDE_CONF_DIR=$TELEGRAF_CONF_DIR"/telegraf.d" TELEGRAF_OUTPUT_CONF_FILE=$TELEGRAF_INCLUDE_CONF_DIR"/telegraf_output.conf" -echo "Telegraf Configuration File: ${TELEGRAF_CONF_FILE}" +#cat ${TELEGRAF_OUTPUT_CONF_FILE} # Replace template parameters on general configuration sed -i 's/$LOCATION/'$LOCATION'/g' $TELEGRAF_CONF_FILE diff --git a/scripts/clmc-agent/configure_template.sh b/scripts/clmc-agent/configure_template.sh old mode 100644 new mode 100755 diff --git a/scripts/clmc-agent/install.sh b/scripts/clmc-agent/install.sh index d0a0de1ee9dc44bff96dda3696967db13521bfcd..d82b94450ed6227446432ac230142169416866dc 100755 --- a/scripts/clmc-agent/install.sh +++ b/scripts/clmc-agent/install.sh @@ -27,6 +27,9 @@ # Force fail on command fail set -euo pipefail +apt-get update +apt-get install wget -y + echo "Installing Telegraf agent" TELEGRAF_VERSION=1.7.0~5618bb0-0 diff --git a/scripts/clmc-service/install.sh b/scripts/clmc-service/install.sh index 543466fcd27b7abda3fb84eb2ae6c44a1251fa89..32c22d4be9da8789eecda35181713d380455f5a6 100755 --- a/scripts/clmc-service/install.sh +++ b/scripts/clmc-service/install.sh @@ -51,7 +51,16 @@ KAPACITOR_CHECKSUM=eea9b215f241906570eafe3857e1d4c5 CHRONOGRAF_VERSION=1.4.4.2 CHRONOGRAF_CHECKSUM=eea6915aa6db8f134fcd3b095e863b773bfb3a16a26e346dd65904a07df97963 +# install virtualenvwrapper to manage python environments - and check apt-get update +echo "----> Installing Python3 and Pip3" +apt-get install -y python3 python3-pip wget curl +update-alternatives --install /usr/bin/python python /usr/bin/python3 10 + +echo "----> Installing virtualenv and wrapper" +apt-get install -y python3-virtualenv virtualenvwrapper +pip3 install virtualenv +pip3 install virtualenvwrapper # install influx wget https://dl.influxdata.com/influxdb/releases/influxdb_${INFLUX_VERSION}_amd64.deb 2> /dev/null @@ -87,15 +96,7 @@ systemctl start chronograf ## CLMC-SERVICE ## ---------------------------------------------------------------------------------- -# install virtualenvwrapper to manage python environments - and check -echo "----> Installing Python3 and Pip3" -apt-get install -y python3 python3-pip -update-alternatives --install /usr/bin/python python /usr/bin/python3 10 -echo "----> Installing virtualenv and wrapper" -apt-get install -y python3-virtualenv virtualenvwrapper -pip3 install virtualenv -pip3 install virtualenvwrapper echo "----> Configuring virtualenvwrapper" export WORKON_HOME=$HOME/.virtualenvs @@ -172,9 +173,11 @@ done # configure the CLMC service JSON="{\"aggregator_report_period\": ${REPORT_PERIOD}, \"aggregator_database_name\": \"${DATABASE_NAME}\", \"aggregator_database_url\": \"${INFLUX_URL}\"}" +echo "CONFIG JSON=${JSON}" curl -H 'Content-Type: application/json' -X PUT -d "${JSON}" http://localhost:9080/aggregator/config # start the aggregator JSON="{\"action\": \"start\"}" +echo "START ACTION JSON=${JSON}" curl -H 'Content-Type: application/json' -X PUT -d "${JSON}" http://localhost:9080/aggregator/control diff --git a/scripts/test/conts-create.sh b/scripts/test/conts-create.sh new file mode 100755 index 0000000000000000000000000000000000000000..637d8ba519666e16c9681a2d96903fe3914ec1e9 --- /dev/null +++ b/scripts/test/conts-create.sh @@ -0,0 +1,92 @@ +#!/bin/bash +set -eu -o pipefail +cd `dirname $0` + +source conts-env.sh + +# iterate through each service required for integration testing +service_names=$(jq -r '.[].name' ${rspec_file}) +for service_name in $service_names; do + if ! lxc-info -n ${service_name}; then + # create a container with a static ip address + echo "Creating container: ${service_name}" + SERVICE=$(jq --arg NAME ${service_name} '.[] | select(.name==$NAME)' ${rspec_file}) + echo $SERVICE + ip=$(echo $SERVICE | jq -r '.ip_address') + echo "dhcp-host=${service_name},${ip}" >> /etc/lxc/dnsmasq.conf + service lxc-net restart + + lxc-create -t download -n ${service_name} -- --dist ubuntu --release xenial --arch amd64 + + # copy flame clmc files into the root container + echo "Copying files to rootfs" + container_dir="/var/lib/lxc/"${service_name}"/rootfs" + container_vagrant_dir=${container_dir}"${repo_root}" + mkdir -p ${container_vagrant_dir} + cp -f ${repo_root}/reporc "${container_vagrant_dir}" + cp -rf ${repo_root}/scripts ${container_vagrant_dir} + cp -rf ${repo_root}/src ${container_vagrant_dir} + + # start the container + echo "Starting: ${service_name}" + lxc-start -n ${service_name} + echo "Waiting for container to start: ${service_name}" + STARTED="0" + while [ "$STARTED" == "0" ]; do STARTED=$(lxc-info -n ${service_name} -i | wc -l); done; + + # provision software into each container + echo "Provisioning: ${service_name}" + if [ ${service_name} == "clmc-service" ] + then + influxdb_url=$(echo $SERVICE | jq -r '.influxdb_url') + database_name=$(echo $SERVICE | jq -r '.database_name') + report_period=$(echo $SERVICE | jq -r '.report_period') + cmd="${repo_root}/scripts/clmc-service/install.sh ${influxdb_url} ${database_name} ${report_period}" + lxc-attach -n ${service_name} -v REPO_ROOT=${repo_root} -- ${cmd} + elif [ ${service_name} == "test-runner" ] + then + cmd=${repo_root}/src/test/clmctest/services/pytest/install.sh + lxc-attach -n ${service_name} -- ${cmd} + else + # get container parameters + location=$(echo $SERVICE | jq -r '.location') + sf_id=$(echo $SERVICE | jq -r '.sf_id') + sf_id_instance=$(echo $SERVICE | jq -r '.sf_id_instance') + sfc_id=$(echo $SERVICE | jq -r '.sfc_id') + sfc_id_instance=$(echo $SERVICE | jq -r '.sfc_id_instance') + sr_id=$(echo $SERVICE | jq -r '.sr_id') + ipendpoint_id=$(echo $SERVICE | jq -r '.ipendpoint_id') + influxdb_url=$(echo $SERVICE | jq -r '.influxdb_url') + database_name=$(echo $SERVICE | jq -r '.database_name') + + # install service function specific software + cmd=${repo_root}/src/test/clmctest/services/${sf_id}/install.sh + lxc-attach -n ${service_name} -v REPO_ROOT=${repo_root} -- ${cmd} + + # install telegraf + cmd=${repo_root}/scripts/clmc-agent/install.sh + lxc-attach -n ${service_name} -v REPO_ROOT=${repo_root} -- ${cmd} + + # copy telegraf configuration templates + cp -f ${repo_root}/scripts/clmc-agent/telegraf.conf ${container_dir}/etc/telegraf/ + cp -f ${repo_root}/scripts/clmc-agent/telegraf_output.conf ${container_dir}/etc/telegraf/telegraf.d/ + cp ${repo_root}/src/test/clmctest/services/${sf_id}/telegraf_${sf_id}.conf ${container_dir}/etc/telegraf/telegraf.d/ + + # replace telegraf template with container parameters + # @todo do we really need both scripts to do this? + cmd=${repo_root}/scripts/clmc-agent/configure_template.sh + lxc-attach -n ${service_name} -- ${cmd} + cmd="${repo_root}/scripts/clmc-agent/configure.sh ${location} ${sfc_id} ${sfc_id_instance} ${sf_id} ${sf_id_instance} ${ipendpoint_id} ${sr_id} ${influxdb_url} ${database_name}" + lxc-attach -n ${service_name} -- ${cmd} + + # start telegraf + lxc-attach -n ${service_name} -- service telegraf restart + fi + fi +done + +#./conts-stop.sh + +#service lxc-net restart + +#./conts-start.sh \ No newline at end of file diff --git a/scripts/test/conts-destroy.sh b/scripts/test/conts-destroy.sh new file mode 100755 index 0000000000000000000000000000000000000000..456816dc54ba3bc6960f9dc417a01532cd1100ca --- /dev/null +++ b/scripts/test/conts-destroy.sh @@ -0,0 +1,17 @@ +#!/bin/bash +set -eu -o pipefail +cd `dirname $0` + +source conts-env.sh + +service_names=$(jq -r '.[].name' ${rspec_file}) +for service_name in $service_names; do + if lxc-info -n ${service_name}; then + echo "Stopping container: ${service_name}" + lxc-stop -n ${service_name} + echo "Destroying container: ${service_name}" + lxc-destroy -n ${service_name} + ip=$(jq -r --arg NAME ${service_name} '.[] | select(.name==$NAME) | .ip_address' ${rspec_file}) + sed -i "/dhcp-host=${service_name},/d" /etc/lxc/dnsmasq.conf + fi +done diff --git a/scripts/test/conts-env.sh b/scripts/test/conts-env.sh new file mode 100755 index 0000000000000000000000000000000000000000..742d54779cb3a2434cb70b07ef1dff8ceb461f92 --- /dev/null +++ b/scripts/test/conts-env.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +repo_root="/vagrant" +rspec_file=$repo_root"/src/test/clmctest/rspec.json" \ No newline at end of file diff --git a/scripts/test/conts-start.sh b/scripts/test/conts-start.sh new file mode 100755 index 0000000000000000000000000000000000000000..2af56c1a6f3ee7a301bdefd6f9e4fe1ce7487e1e --- /dev/null +++ b/scripts/test/conts-start.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -eu -o pipefail +cd `dirname $0` + +source conts-env.sh + +service_names=$(jq -r '.[].name' ${rspec_file}) +for service_name in $service_names; do + if lxc-info -n ${service_name}; then + echo "Starting container: ${service_name}" + lxc-start -n ${service_name} + fi +done diff --git a/scripts/test/conts-stop.sh b/scripts/test/conts-stop.sh new file mode 100755 index 0000000000000000000000000000000000000000..a64786939ffdbfa2a7916da356d69b3a197c749f --- /dev/null +++ b/scripts/test/conts-stop.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -eu -o pipefail +cd `dirname $0` + +source conts-env.sh + +service_names=$(jq -r '.[].name' ${rspec_file}) +for service_name in $service_names; do + if lxc-info -n ${service_name}; then + echo "Stopping container: ${service_name}" + lxc-stop -n ${service_name} + fi +done \ No newline at end of file diff --git a/scripts/build/deleteallvms.py b/scripts/test/deleteallvms.py similarity index 100% rename from scripts/build/deleteallvms.py rename to scripts/test/deleteallvms.py diff --git a/scripts/build/deleteallvms.sh b/scripts/test/deleteallvms.sh similarity index 100% rename from scripts/build/deleteallvms.sh rename to scripts/test/deleteallvms.sh diff --git a/src/test/clmctest/e2e_response_time/rspec.yml b/src/test/clmctest/e2e_response_time/rspec.yml deleted file mode 100644 index 4fe3767ddb075e033c3b10acd81a885d73cdc18a..0000000000000000000000000000000000000000 --- a/src/test/clmctest/e2e_response_time/rspec.yml +++ /dev/null @@ -1,56 +0,0 @@ -## (c) University of Southampton IT Innovation Centre, 2018 -## -## Copyright in this software belongs to University of Southampton -## IT Innovation Centre of Gamma House, Enterprise Road, -## Chilworth Science Park, Southampton, SO16 7NS, UK. -## -## This software may not be used, sold, licensed, transferred, copied -## or reproduced in whole or in part in any manner or form or in or -## on any media by any person other than in accordance with the terms -## of the Licence Agreement supplied with the software, or otherwise -## without the prior written consent of the copyright owners. -## -## This software is distributed WITHOUT ANY WARRANTY, without even the -## implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR -## PURPOSE, except where stated in the Licence Agreement supplied with -## the software. -## -## Created By : Michael Boniface -## Created Date : 02-02-2018 -## Created for Project : FLAME - -hosts: - - name: clmc-service - cpus: 1 - memory: 2048 - disk: "10GB" - forward_ports: - - guest: 8086 - host: 8086 - - guest: 8888 - host: 8888 - - guest: 9092 - host: 9092 - ip_address: "172.40.231.51" - - name: minio - service_name: "minio" - cpus: 1 - memory: 2048 - disk: "10GB" - forward_ports: - - guest: 9000 - host: 9000 - ip_address: "172.40.231.155" - location: "DC1" - sfc_id: "MS_Template_1" - sfc_id_instance: "MS_I1" - sf_id: "adaptive_streaming" - sf_id_instance: "adaptive_streaming_I1" - ipendpoint_id: "adaptive_streaming_I1_minio" - influxdb_url: "http://172.40.231.51:8086" - database_name: "CLMCMetrics" - - name: test-runner - cpus: 1 - memory: 2048 - disk: "10GB" - ip_address: "172.40.231.200" diff --git a/src/test/clmctest/inputs/conftest.py b/src/test/clmctest/inputs/conftest.py index c3b3b40eb5ee6141e7096d161b591f5e5a43bc19..1f4294969e27e11d9004f68e11580f7d799ebf58 100644 --- a/src/test/clmctest/inputs/conftest.py +++ b/src/test/clmctest/inputs/conftest.py @@ -25,6 +25,7 @@ import pytest import time import yaml +import json import pkg_resources from influxdb import InfluxDBClient @@ -37,10 +38,10 @@ def telegraf_agent_config(request): :param request: access the parameters of the fixture :return: the python object representing the read YAML file """ - rspec = pkg_resources.resource_filename('clmctest.inputs', 'rspec.yml') + rspec = pkg_resources.resource_filename('clmctest', 'rspec.json') print("rspec file: {0}".format(rspec)) with open(rspec, 'r') as stream: - data_loaded = yaml.load(stream) + data_loaded = json.load(stream) return data_loaded @@ -54,7 +55,7 @@ def influxdb(telegraf_agent_config, request): :return: the created Influx DB client """ - db = InfluxDBClient(host=telegraf_agent_config['hosts'][0]['ip_address'], port=8086, database=request.param['database'], timeout=10) + db = InfluxDBClient(host=telegraf_agent_config[0]['ip_address'], port=8086, database=request.param['database'], timeout=10) db.drop_database(request.param['database']) # wait 20 seconds for the 1st measurement to arrive from agents before returning diff --git a/src/test/clmctest/inputs/rspec.yml b/src/test/clmctest/inputs/rspec.yml deleted file mode 100644 index 5084b29ebdb308e7c116f9db5827f0c15ff289d0..0000000000000000000000000000000000000000 --- a/src/test/clmctest/inputs/rspec.yml +++ /dev/null @@ -1,147 +0,0 @@ -## (c) University of Southampton IT Innovation Centre, 2018 -## -## Copyright in this software belongs to University of Southampton -## IT Innovation Centre of Gamma House, Enterprise Road, -## Chilworth Science Park, Southampton, SO16 7NS, UK. -## -## This software may not be used, sold, licensed, transferred, copied -## or reproduced in whole or in part in any manner or form or in or -## on any media by any person other than in accordance with the terms -## of the Licence Agreement supplied with the software, or otherwise -## without the prior written consent of the copyright owners. -## -## This software is distributed WITHOUT ANY WARRANTY, without even the -## implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR -## PURPOSE, except where stated in the Licence Agreement supplied with -## the software. -## -## Created By : Michael Boniface -## Created Date : 02-02-2018 -## Created for Project : FLAME - -hosts: - - name: clmc-service - cpus: 1 - memory: 2048 - disk: "10GB" - forward_ports: - - guest: 8086 - host: 8086 - - guest: 8888 - host: 8888 - - guest: 9092 - host: 9092 - ip_address: "172.40.231.51" - - name: apache - cpus: 1 - memory: 2048 - disk: "10GB" - service_name: "apache" - forward_ports: - - guest: 80 - host: 8881 - ip_address: "172.40.231.150" - location: "DC1" - sfc_id: "MS_Template_1" - sfc_id_instance: "MS_I1" - sf_id: "adaptive_streaming" - sf_id_instance: "adaptive_streaming_I1" - ipendpoint_id: "adaptive_streaming_I1_apache1" - sr_id: "service_router" - influxdb_url: "http://172.40.231.51:8086" - database_name: "CLMCMetrics" - - name: nginx - cpus: 1 - memory: 2048 - disk: "10GB" - service_name: "nginx" - forward_ports: - - guest: 80 - host: 8882 - ip_address: "172.40.231.151" - location: "DC1" - sfc_id: "MS_Template_1" - sfc_id_instance: "MS_I1" - sf_id: "adaptive_streaming" - sf_id_instance: "adaptive_streaming_nginx_I1" - ipendpoint_id: "adaptive_streaming_nginx_I1_apache1" - sr_id: "service_router" - influxdb_url: "http://172.40.231.51:8086" - database_name: "CLMCMetrics" - - name: mongo - cpus: 1 - memory: 2048 - disk: "10GB" - service_name: "mongo" - forward_ports: - - guest: 80 - host: 8883 - ip_address: "172.40.231.152" - location: "DC1" - sfc_id: "MS_Template_1" - sfc_id_instance: "MS_I1" - sf_id: "metadata_database" - sf_id_instance: "metadata_database_I1" - ipendpoint_id: "metadata_database_I1_apache1" - sr_id: "service_router" - influxdb_url: "http://172.40.231.51:8086" - database_name: "CLMCMetrics" - - name: ffmpeg - cpus: 1 - memory: 2048 - disk: "10GB" - service_name: "ffmpeg" - forward_ports: - - guest: 80 - host: 8884 - ip_address: "172.40.231.153" - location: "DC1" - sfc_id: "MS_Template_1" - sfc_id_instance: "MS_I1" - sf_id: "metadata_database" - sf_id_instance: "metadata_database_I1" - ipendpoint_id: "metadata_database_I1_apache1" - sr_id: "service_router" - influxdb_url: "http://172.40.231.51:8086" - database_name: "CLMCMetrics" - - name: host - cpus: 1 - memory: 2048 - disk: "10GB" - service_name: "host" - forward_ports: - - guest: 80 - host: 8885 - ip_address: "172.40.231.154" - location: "DC1" - sfc_id: "MS_Template_1" - sfc_id_instance: "MS_I1" - sf_id: "adaptive_streaming" - sf_id_instance: "adaptive_streaming_I1" - ipendpoint_id: "adaptive_streaming_I1_apache1" - sr_id: "service_router" - influxdb_url: "http://172.40.231.51:8086" - database_name: "CLMCMetrics" - - name: test-runner - cpus: 2 - memory: 4096 - disk: "10GB" - ip_address: "172.40.231.200" - - name: minio - service_name: "minio" - cpus: 1 - memory: 2048 - disk: "10GB" - forward_ports: - - guest: 9000 - host: 9000 - ip_address: "172.40.231.155" - location: "DC1" - sfc_id: "MS_Template_1" - sfc_id_instance: "MS_I1" - sf_id: "adaptive_streaming" - sf_id_instance: "adaptive_streaming_I1" - ipendpoint_id: "adaptive_streaming_I1_minio" - sr_id: "service_router" - influxdb_url: "http://172.40.231.51:8086" - database_name: "CLMCMetrics" \ No newline at end of file diff --git a/src/test/clmctest/inputs/test_config_collector.py b/src/test/clmctest/inputs/test_config_collector.py deleted file mode 100644 index ce320b9a7562d2aad1445973724fb59e01de4abe..0000000000000000000000000000000000000000 --- a/src/test/clmctest/inputs/test_config_collector.py +++ /dev/null @@ -1,163 +0,0 @@ -#!/usr/bin/python3 -""" -## © University of Southampton IT Innovation Centre, 2018 -## -## Copyright in this software belongs to University of Southampton -## IT Innovation Centre of Gamma House, Enterprise Road, -## Chilworth Science Park, Southampton, SO16 7NS, UK. -## -## This software may not be used, sold, licensed, transferred, copied -## or reproduced in whole or in part in any manner or form or in or -## on any media by any person other than in accordance with the terms -## of the Licence Agreement supplied with the software, or otherwise -## without the prior written consent of the copyright owners. -## -## This software is distributed WITHOUT ANY WARRANTY, without even the -## implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR -## PURPOSE, except where stated in the Licence Agreement supplied with -## the software. -## -## Created By : Michael Boniface -## Created Date : 29-04-2018 -## Created for Project : FLAME -""" - -import pytest -import time -import random -import logging -import sys - -from config_collector import ConfigCollector - -STATE_INDEX = 0 -TIME_INDEX = 1 - -samples = [[['active', 0], ['active', 2]], - [['active', 0], ['active', 2], ['active', 4]], - [['active', 0], ['failed', 2]], - [['active', 0], ['active', 2], ['inactive', 4], ['active', 6], ['failed', 8], ['inactive', 10]], - [['active', 0], ['inactive', 2], ['failed', 4], ['active', 6], ['inactive', 8], ['failed', 10]]] - -def get_sample_test(): - global sample_set - global current_index - - sample = (samples[sample_set][current_index][STATE_INDEX], time.time()) - sample_count = len(samples[sample_set]) - if current_index < sample_count-1: - current_index +=1 - else: - current_index = 0 - return sample - -def write_output(measurement): - print("Writing measurement output {0}".format(measurement)) - -sample_set = 0 -current_index = 0 - -def test_agg(): - t = ConfigCollector(get_sample_test, write_output, "resource") - measurement = t.create_measurement(samples[0], 10, 12) - assert measurement[0]['fields']['current_state'] == 'active' - assert measurement[0]['fields']['current_state_time'] == 12 - assert measurement[0]['fields']['active_sum'] == 12 - assert measurement[0]['fields']['active_count'] == 1 - assert measurement[0]['time'] == 12000000000 - - t = ConfigCollector(get_sample_test, write_output, "resource") - measurement = t.create_measurement(samples[1], 10, 14) - assert measurement[0]['fields']['current_state'] == 'active' - assert measurement[0]['fields']['current_state_time'] == 14 - assert measurement[0]['fields']['active_sum'] == 14 - assert measurement[0]['fields']['active_count'] == 1 - assert measurement[0]['time'] == 14000000000 - - t = ConfigCollector(get_sample_test, write_output, "resource") - measurement = t.create_measurement(samples[2], 8, 10) - assert measurement[0]['fields']['current_state'] == 'failed' - assert measurement[0]['fields']['current_state_time'] == 0 - assert measurement[0]['fields']['active_sum'] == 2 - assert measurement[0]['fields']['active_count'] == 1 - assert measurement[0]['fields']['failed_sum'] == 0 - assert measurement[0]['fields']['failed_count'] == 1 - assert measurement[0]['time'] == 10000000000 - - t = ConfigCollector(get_sample_test, write_output, "resource") - measurement = t.create_measurement(samples[3], 2, 12) - assert measurement[0]['fields']['current_state'] == 'inactive' - assert measurement[0]['fields']['current_state_time'] == 0 - assert measurement[0]['fields']['active_sum'] == 6 - assert measurement[0]['fields']['active_count'] == 2 - assert measurement[0]['fields']['inactive_sum'] == 2 - assert measurement[0]['fields']['inactive_count'] == 2 - assert measurement[0]['fields']['failed_sum'] == 2 - assert measurement[0]['fields']['failed_count'] == 1 - assert measurement[0]['time'] == 12000000000 - - t = ConfigCollector(get_sample_test, write_output, "resource") - measurement = t.create_measurement(samples[4], 4, 14) - assert measurement[0]['fields']['current_state'] == 'failed' - assert measurement[0]['fields']['current_state_time'] == 0 - assert measurement[0]['fields']['active_sum'] == 4 - assert measurement[0]['fields']['active_count'] == 2 - assert measurement[0]['fields']['inactive_sum'] == 4 - assert measurement[0]['fields']['inactive_count'] == 2 - assert measurement[0]['fields']['failed_sum'] == 2 - assert measurement[0]['fields']['failed_count'] == 2 - assert measurement[0]['time'] == 14000000000 - -def test_one_period_collection(): - global sample_set - global current_index - - # one measurementing period - sample_set = 1 - current_index = 0 - t = ConfigCollector(get_sample_test, write_output, "resource", 2, 6) - t.start() - time.sleep(8) - t.stop() - print("Current measurement: {0}".format(str(t.current_measurement))) - assert t.current_measurement[0]['fields']['current_state'] == 'active' - assert int(round(t.current_measurement[0]['fields']['current_state_time'])) == 6 - assert int(round(t.current_measurement[0]['fields']['active_sum'])) == 6 - assert int(round(t.current_measurement[0]['fields']['active_count'])) == 1 - -def test_multi_period_single_state_collection(): - global sample_set - global current_index - # two measurementing periods - sample_set = 1 - current_index = 0 - t = ConfigCollector(get_sample_test, write_output, "resource", 1, 3) - t.start() - time.sleep(7) - t.stop() - print("Current measurement: {0}".format(str(t.current_measurement))) - assert t.current_measurement[0]['fields']['current_state'] == 'active' - assert int(round(t.current_measurement[0]['fields']['current_state_time'])) == 6 - assert int(round(t.current_measurement[0]['fields']['active_sum'])) == 6 - assert int(round(t.current_measurement[0]['fields']['active_count'])) == 1 - -# [['active', 0], ['inactive', 2], ['failed', 4], ['active', 6], ['inactive', 8], ['failed', 10]] -def test_multi_period_multi_state_collection(): - global sample_set - global current_index - # 6 samples and 2 measurementing periods - sample_set = 4 - current_index = 0 - t = ConfigCollector(get_sample_test, write_output, "resource", 2, 10) - t.start() - time.sleep(13) - t.stop() - print("Current measurement: {0}".format(str(t.current_measurement))) - assert t.current_measurement[0]['fields']['current_state'] == 'failed' - assert int(round(t.current_measurement[0]['fields']['current_state_time'])) == 0 - assert int(round(t.current_measurement[0]['fields']['active_sum'])) == 4 - assert int(round(t.current_measurement[0]['fields']['active_count'])) == 2 - assert int(round(t.current_measurement[0]['fields']['inactive_sum'])) == 4 - assert int(round(t.current_measurement[0]['fields']['inactive_count'])) == 2 - assert int(round(t.current_measurement[0]['fields']['failed_sum'])) == 2 - assert int(round(t.current_measurement[0]['fields']['failed_count'])) == 2 \ No newline at end of file diff --git a/src/test/clmctest/inputs/test_rspec.py b/src/test/clmctest/inputs/test_rspec.py index 5658e663b786dc2a13687c8dccbc2a1c1beb2cc7..e56bac91e7335a0fcad399dc41f5ac00ae8c1fd5 100644 --- a/src/test/clmctest/inputs/test_rspec.py +++ b/src/test/clmctest/inputs/test_rspec.py @@ -32,8 +32,8 @@ import pytest 'apache', 'nginx', 'mongo', - 'ffmpeg', - 'host' + 'host', + 'minio' ]) def test_service_name(telegraf_agent_config, service_name): """ @@ -43,7 +43,7 @@ def test_service_name(telegraf_agent_config, service_name): :param service_name the service name to test """ - assert any(s['name'] == service_name for s in telegraf_agent_config['hosts']), "{0} not in list of hosts".format(service_name) + assert any(s['name'] == service_name for s in telegraf_agent_config), "{0} not in list of hosts".format(service_name) print("\nSuccessfully passed configuration test for service name {0}\n".format(service_name)) @@ -59,7 +59,7 @@ def test_ping(telegraf_agent_config): ping_count = 1 system_dependent_param = "-n" if system().lower() == "windows" else "-c" - for service in telegraf_agent_config['hosts']: + for service in telegraf_agent_config: command = ["ping", system_dependent_param, str(ping_count), service['ip_address']] assert run(command).returncode == 0, "Service ping test failed for {0} with ip address {1}".format(service['name'], service['ip_address']) print("\nSuccessfully passed ping test for service: {0}\n".format(service['name'])) diff --git a/src/test/clmctest/inputs/test_systemctl_mon.py b/src/test/clmctest/inputs/test_systemctl_mon.py deleted file mode 100644 index d57f69e69e9dd68358b5d6ca370f2b15885cca5b..0000000000000000000000000000000000000000 --- a/src/test/clmctest/inputs/test_systemctl_mon.py +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/python3 -""" -## © University of Southampton IT Innovation Centre, 2018 -## -## Copyright in this software belongs to University of Southampton -## IT Innovation Centre of Gamma House, Enterprise Road, -## Chilworth Science Park, Southampton, SO16 7NS, UK. -## -## This software may not be used, sold, licensed, transferred, copied -## or reproduced in whole or in part in any manner or form or in or -## on any media by any person other than in accordance with the terms -## of the Licence Agreement supplied with the software, or otherwise -## without the prior written consent of the copyright owners. -## -## This software is distributed WITHOUT ANY WARRANTY, without even the -## implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR -## PURPOSE, except where stated in the Licence Agreement supplied with -## the software. -## -## Created By : Michael Boniface -## Created Date : 29-04-2018 -## Created for Project : FLAME -""" - -import pytest -import time -import random -import logging -import sys - -from systemctl_monitor import SystemctlMonitor - -URL = "localhost" -PORT = "8186" -DATABASE = "CLMCMetrics" - -@pytest.mark.parametrize("service_name", [('nginx')]) -def test_create_measurement(telegraf_agent_config, service_name): - - service = 'unknown' - for s in telegraf_agent_config['hosts']: - if s['name'] == service_name: - service = s - continue - assert service != 'unknown', "{0} not in list of hosts".format(service_name) - - mon = SystemctlMonitor(service_name, 2, 10, service['ip_address'], 8186, service['database_name']) - report = {'time': 1526042434.1773288, 'fields': {'loaded.active.running_sum': 231.85903143882751, 'current_state_time': 231.85903143882751, 'current_state': 'loaded.active.running', 'loaded.active.running_count': 1}} - measurement = mon.create_measurement(report) - assert measurement[0]['tags']['resource_name'] == service_name - assert measurement[0]['fields']['current_state'] == report['fields']['current_state'] - -def test_get_systemctl_status(telegraf_agent_config): - mon = SystemctlMonitor('nginx', 2, 10, URL, PORT, DATABASE) - state = mon.get_systemctl_status('nginx') - assert state == 'loaded.active.running' - -def test_monitor(telegraf_agent_config): - mon = SystemctlMonitor('nginx', 2, 10, URL, PORT, DATABASE) - mon.start() - time.sleep(21) - mon.stop() - measurement = mon.get_current_measurement() - print("Current measurement: {0}".format(str(measurement))) \ No newline at end of file diff --git a/src/test/clmctest/inputs/test_telegraf_agents.py b/src/test/clmctest/inputs/test_telegraf_agents.py index c041104c0780b25c78d862825c57604bc74b1bcc..2358d93d6968f83ad7c0712e7dcce25575df638b 100644 --- a/src/test/clmctest/inputs/test_telegraf_agents.py +++ b/src/test/clmctest/inputs/test_telegraf_agents.py @@ -32,12 +32,11 @@ from influxdb import InfluxDBClient ('apache'), ('nginx'), ('mongo'), - ('ffmpeg'), ('host'), ('minio') ]) def test_service_name(telegraf_agent_config, service_name): - assert any(s['name'] == service_name for s in telegraf_agent_config['hosts']), "{0} not in list of hosts".format(service_name) + assert any(s['name'] == service_name for s in telegraf_agent_config), "{0} not in list of hosts".format(service_name) def test_ping(telegraf_agent_config): """ @@ -51,7 +50,7 @@ def test_ping(telegraf_agent_config): ping_count = 1 system_dependent_param = "-n" if system().lower() == "windows" else "-c" - for service in telegraf_agent_config['hosts']: + for service in telegraf_agent_config: command = ["ping", system_dependent_param, str(ping_count), service['ip_address']] assert run(command).returncode == 0, "Service ping test failed for {0} with ip address {1}".format(service['name'], service['ip_address']) print("\nSuccessfully passed ping test for service: {0}\n".format(service['name'])) @@ -63,7 +62,6 @@ def test_ping(telegraf_agent_config): ('mongodb', 'SELECT mean("net_in_bytes") AS "mean" FROM "CLMCMetrics"."autogen"."mongodb"', 0), ('net', 'SELECT mean("bytes_sent") AS "mean" FROM "CLMCMetrics"."autogen"."net"', 0), ('disk', 'SELECT mean("free") AS "mean" FROM "CLMCMetrics"."autogen"."disk"', 0), - ('diskio', 'SELECT mean("write_bytes") AS "mean" FROM "CLMCMetrics"."autogen"."diskio"', 0), ('mem', 'SELECT mean("free") AS "mean" FROM "CLMCMetrics"."autogen"."mem"', 0), # Report MINIO's HTTP request response time (as a rolling difference of the sum total) ('minio_http_requests_duration_seconds', 'SELECT difference(max("sum")) AS "mean" FROM "CLMCMetrics"."autogen"."minio_http_requests_duration_seconds" WHERE time > now() - 1h GROUP BY time(10s)',0), diff --git a/src/test/clmctest/monitoring/conftest.py b/src/test/clmctest/monitoring/conftest.py index 8c85786de9a2e1ccce89ddfd9489aa9c278b6641..819909cc0f220cd825863a9c2d35094da2d925a6 100644 --- a/src/test/clmctest/monitoring/conftest.py +++ b/src/test/clmctest/monitoring/conftest.py @@ -23,7 +23,7 @@ """ import pytest -import yaml +import json import pkg_resources from influxdb import InfluxDBClient from clmctest.monitoring.StreamingSim import Sim @@ -38,11 +38,11 @@ def streaming_sim_config(): :return: the python object representing the read YAML file """ - rspec = pkg_resources.resource_filename('clmctest.monitoring', 'rspec.yml') + rspec = pkg_resources.resource_filename('clmctest', 'rspec.json') print("\nrspec file: {0}".format(rspec)) with open(rspec, 'r') as stream: - data_loaded = yaml.load(stream) + data_loaded = json.load(stream) return data_loaded @@ -56,7 +56,7 @@ def influx_db(streaming_sim_config, request): :return: the created Influx DB client """ - return InfluxDBClient(host=streaming_sim_config['hosts'][0]['ip_address'], port='8086', database=request.param['database'], timeout=10) + return InfluxDBClient(host=streaming_sim_config[0]['ip_address'], port='8086', database=request.param['database'], timeout=10) @pytest.fixture(scope="module") @@ -68,10 +68,14 @@ def simulator(streaming_sim_config): :return: an instance of the simulator """ - influx_url = "http://" + streaming_sim_config['hosts'][0]['ip_address'] + ":8086" - influx_db_name = streaming_sim_config['hosts'][1]['database_name'] - agent1_url = "http://" + streaming_sim_config['hosts'][1]['ip_address'] + ":8186" - agent2_url = "http://" + streaming_sim_config['hosts'][2]['ip_address'] + ":8186" + influx_url = "http://" + streaming_sim_config[0]['ip_address'] + ":8086" + + for service in streaming_sim_config: + if service['name'] == "ipendpoint1": + influx_db_name = service['database_name'] + agent1_url = "http://" + service['ip_address'] + ":8186" + elif service['name'] == "ipendpoint2": + agent2_url = "http://" + service['ip_address'] + ":8186" simulator = Sim(influx_url, influx_db_name, agent1_url, agent2_url) @@ -89,7 +93,7 @@ def e2e_simulator(streaming_sim_config): :return: an instance of the E2E simulator """ - influx_url = "http://" + streaming_sim_config['hosts'][0]['ip_address'] + ":8086" + influx_url = "http://" + streaming_sim_config[0]['ip_address'] + ":8086" return Simulator(database_url=influx_url) @@ -103,6 +107,6 @@ def e2e_aggregator(streaming_sim_config): :return: an instance of the Aggregator class """ - influx_url = "http://" + streaming_sim_config['hosts'][0]['ip_address'] + ":8086" + influx_url = "http://" + streaming_sim_config[0]['ip_address'] + ":8086" return TestAggregator(database_url=influx_url) diff --git a/src/test/clmctest/monitoring/rspec.yml b/src/test/clmctest/monitoring/rspec.yml deleted file mode 100644 index fa5297ab2aca0d58b3c9ecd9b5373c5a3ee3f49e..0000000000000000000000000000000000000000 --- a/src/test/clmctest/monitoring/rspec.yml +++ /dev/null @@ -1,80 +0,0 @@ -## (c) University of Southampton IT Innovation Centre, 2018 -## -## Copyright in this software belongs to University of Southampton -## IT Innovation Centre of Gamma House, Enterprise Road, -## Chilworth Science Park, Southampton, SO16 7NS, UK. -## -## This software may not be used, sold, licensed, transferred, copied -## or reproduced in whole or in part in any manner or form or in or -## on any media by any person other than in accordance with the terms -## of the Licence Agreement supplied with the software, or otherwise -## without the prior written consent of the copyright owners. -## -## This software is distributed WITHOUT ANY WARRANTY, without even the -## implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR -## PURPOSE, except where stated in the Licence Agreement supplied with -## the software. -## -## Created By : Michael Boniface -## Created Date : 02-02-2018 -## Created for Project : FLAME - -hosts: - - name: clmc-service - cpus: 1 - memory: 2048 - disk: "10GB" - forward_ports: - - guest: 8086 - host: 8086 - - guest: 8888 - host: 8888 - - guest: 9092 - host: 9092 - - guest: 9080 - host: 9080 - ip_address: "172.40.231.51" - influxdb_url: "http://172.40.231.51:8086" - database_name: "CLMCMetrics" - report_period: 25 - - name: ipendpoint1 - cpus: 1 - memory: 2048 - disk: "10GB" - service_name: "ipendpoint" - forward_ports: - - guest: 80 - host: 8081 - ip_address: "172.40.231.170" - location: "nova" - sfc_id: "media_service_A" - sfc_id_instance: "StackID" - sf_id: "test-sf-clmc-agent-build" - sf_id_instance: "ms-A.ict-flame.eu" - ipendpoint_id: "endpoint1.ms-A.ict-flame.eu" - sr_id: "service_router" - influxdb_url: "http://172.40.231.51:8086" - database_name: "CLMCMetrics" - - name: ipendpoint2 - cpus: 1 - memory: 2048 - disk: "10GB" - service_name: "ipendpoint" - forward_ports: - - guest: 80 - host: 8082 - ip_address: "172.40.231.171" - location: "nova" - sfc_id: "media_service_A" - sfc_id_instance: "StackID" - sf_id: "test-sf-clmc-agent-build" - sf_id_instance: "ms-A.ict-flame.eu" - ipendpoint_id: "endpoint2.ms-A.ict-flame.eu" - sr_id: "service_router" - influxdb_url: "http://172.40.231.51:8086" - database_name: "CLMCMetrics" - - name: test-runner - cpus: 1 - memory: 2048 - disk: "10GB" - ip_address: "172.40.231.200" \ No newline at end of file diff --git a/src/test/clmctest/monitoring/test_rspec.py b/src/test/clmctest/monitoring/test_rspec.py index a9bd72884b235315a288ed1a971add4f445c04c8..999b98c4e8ee6c57505e1f2059f194e33b3a19a7 100644 --- a/src/test/clmctest/monitoring/test_rspec.py +++ b/src/test/clmctest/monitoring/test_rspec.py @@ -40,7 +40,7 @@ def test_service_names(streaming_sim_config, service_name): :param service_name the service name to test """ - assert any(s['name'] == service_name for s in streaming_sim_config['hosts']), "{0} not in list of hosts".format(service_name) + assert any(s['name'] == service_name for s in streaming_sim_config), "{0} not in list of hosts".format(service_name) print("\nSuccessfully passed configuration test for service name {0}\n".format(service_name)) @@ -56,7 +56,7 @@ def test_ping(streaming_sim_config): ping_count = 1 system_dependent_param = "-n" if system().lower() == "windows" else "-c" - for service in streaming_sim_config['hosts']: + for service in streaming_sim_config: command = ["ping", system_dependent_param, str(ping_count), service['ip_address']] assert run(command).returncode == 0, "Service ping test failed for {0} with ip address {1}".format(service['name'], service['ip_address']) print("\nSuccessfully passed ping test for service: {0}\n".format(service['name'])) diff --git a/src/test/clmctest/rspec.json b/src/test/clmctest/rspec.json new file mode 100644 index 0000000000000000000000000000000000000000..376194cc5f292ad1c919f6728451bf6e0145d5ea --- /dev/null +++ b/src/test/clmctest/rspec.json @@ -0,0 +1,105 @@ +[{ + "name": "clmc-service", + "ip_address": "172.40.231.51", + "influxdb_url": "http://172.40.231.51:8086", + "database_name": "CLMCMetrics", + "report_period": "25" +}, +{ + "name": "apache", + "ip_address": "172.40.231.150", + "location": "DC1", + "sfc_id": "MS_Template_1", + "sfc_id_instance": "MS_I1", + "sf_id": "apache", + "sf_id_instance": "adaptive_streaming_I1", + "ipendpoint_id": "adaptive_streaming_I1_apache1", + "sr_id": "service_router", + "influxdb_url": "http://172.40.231.51:8086", + "database_name": "CLMCMetrics" +}, +{ + "name": "nginx", + "ip_address": "172.40.231.151", + "location": "DC1", + "sfc_id": "MS_Template_1", + "sfc_id_instance": "MS_I1", + "sf_id": "nginx", + "sf_id_instance": "adaptive_streaming_nginx_I1", + "ipendpoint_id": "adaptive_streaming_nginx_I1_apache1", + "sr_id": "service_router", + "influxdb_url": "http://172.40.231.51:8086", + "database_name": "CLMCMetrics" +}, +{ + "name": "mongo", + "service_name": "mongo", + "ip_address": "172.40.231.152", + "location": "DC1", + "sfc_id": "MS_Template_1", + "sfc_id_instance": "MS_I1", + "sf_id": "mongo", + "sf_id_instance": "metadata_database_I1", + "ipendpoint_id": "metadata_database_I1_apache1", + "sr_id": "service_router", + "influxdb_url": "http://172.40.231.51:8086", + "database_name": "CLMCMetrics" +}, +{ + "name": "host", + "service_name": "host", + "ip_address": "172.40.231.154", + "location": "DC1", + "sfc_id": "MS_Template_1", + "sfc_id_instance": "MS_I1", + "sf_id": "host", + "sf_id_instance": "adaptive_streaming_I1", + "ipendpoint_id": "adaptive_streaming_I1_apache1", + "sr_id": "service_router", + "influxdb_url": "http://172.40.231.51:8086", + "database_name": "CLMCMetrics" +}, +{ + "name": "minio", + "ip_address": "172.40.231.155", + "location": "DC1", + "sfc_id": "MS_Template_1", + "sfc_id_instance": "MS_I1", + "sf_id": "minio", + "sf_id_instance": "adaptive_streaming_I1", + "ipendpoint_id": "adaptive_streaming_I1_minio", + "sr_id": "service_router", + "influxdb_url": "http://172.40.231.51:8086", + "database_name": "CLMCMetrics" +}, +{ + "name": "ipendpoint1", + "ip_address": "172.40.231.170", + "location": "nova", + "sfc_id": "media_service_A", + "sfc_id_instance": "StackID", + "sf_id": "ipendpoint", + "sf_id_instance": "ms-A.ict-flame.eu", + "ipendpoint_id": "endpoint1.ms-A.ict-flame.eu", + "sr_id": "service_router", + "influxdb_url": "http://172.40.231.51:8086", + "database_name": "CLMCMetrics" +}, +{ + "name": "ipendpoint2", + "ip_address": "172.40.231.171", + "location": "nova", + "sfc_id": "media_service_A", + "sfc_id_instance": "StackID", + "sf_id": "ipendpoint", + "sf_id_instance": "ms-A.ict-flame.eu", + "ipendpoint_id": "endpoint2.ms-A.ict-flame.eu", + "sr_id": "service_router", + "influxdb_url": "http://172.40.231.51:8086", + "database_name": "CLMCMetrics" +}, +{ + "name": "test-runner", + "ip_address": "172.40.231.200" +} +] \ No newline at end of file diff --git a/src/test/clmctest/rspec.json.ffmpeg b/src/test/clmctest/rspec.json.ffmpeg new file mode 100644 index 0000000000000000000000000000000000000000..2a5bfab67d0e284c87b8aa32a718d7fa45039811 --- /dev/null +++ b/src/test/clmctest/rspec.json.ffmpeg @@ -0,0 +1,119 @@ +[{ + "name": "clmc-service", + "ip_address": "172.40.231.51", + "influxdb_url": "http://172.40.231.51:8086", + "database_name": "CLMCMetrics", + "report_period": "25" +}, +{ + "name": "apache", + "ip_address": "172.40.231.150", + "location": "DC1", + "sfc_id": "MS_Template_1", + "sfc_id_instance": "MS_I1", + "sf_id": "apache", + "sf_id_instance": "adaptive_streaming_I1", + "ipendpoint_id": "adaptive_streaming_I1_apache1", + "sr_id": "service_router", + "influxdb_url": "http://172.40.231.51:8086", + "database_name": "CLMCMetrics" +}, +{ + "name": "nginx", + "ip_address": "172.40.231.151", + "location": "DC1", + "sfc_id": "MS_Template_1", + "sfc_id_instance": "MS_I1", + "sf_id": "nginx", + "sf_id_instance": "adaptive_streaming_nginx_I1", + "ipendpoint_id": "adaptive_streaming_nginx_I1_apache1", + "sr_id": "service_router", + "influxdb_url": "http://172.40.231.51:8086", + "database_name": "CLMCMetrics" +}, +{ + "name": "mongo", + "service_name": "mongo", + "ip_address": "172.40.231.152", + "location": "DC1", + "sfc_id": "MS_Template_1", + "sfc_id_instance": "MS_I1", + "sf_id": "mongo", + "sf_id_instance": "metadata_database_I1", + "ipendpoint_id": "metadata_database_I1_apache1", + "sr_id": "service_router", + "influxdb_url": "http://172.40.231.51:8086", + "database_name": "CLMCMetrics" +}, +{ + "name": "ffmpeg", + "service_name": "ffmpeg", + "ip_address": "172.40.231.153", + "location": "DC1", + "sfc_id": "MS_Template_1", + "sfc_id_instance": "MS_I1", + "sf_id": "ffmpeg", + "sf_id_instance": "metadata_database_I1", + "ipendpoint_id": "metadata_database_I1_apache1", + "sr_id": "service_router", + "influxdb_url": "http://172.40.231.51:8086", + "database_name": "CLMCMetrics" +}, +{ + "name": "host", + "service_name": "host", + "ip_address": "172.40.231.154", + "location": "DC1", + "sfc_id": "MS_Template_1", + "sfc_id_instance": "MS_I1", + "sf_id": "host", + "sf_id_instance": "adaptive_streaming_I1", + "ipendpoint_id": "adaptive_streaming_I1_apache1", + "sr_id": "service_router", + "influxdb_url": "http://172.40.231.51:8086", + "database_name": "CLMCMetrics" +}, +{ + "name": "minio", + "ip_address": "172.40.231.155", + "location": "DC1", + "sfc_id": "MS_Template_1", + "sfc_id_instance": "MS_I1", + "sf_id": "minio", + "sf_id_instance": "adaptive_streaming_I1", + "ipendpoint_id": "adaptive_streaming_I1_minio", + "sr_id": "service_router", + "influxdb_url": "http://172.40.231.51:8086", + "database_name": "CLMCMetrics" +}, +{ + "name": "ipendpoint1", + "ip_address": "172.40.231.170", + "location": "nova", + "sfc_id": "media_service_A", + "sfc_id_instance": "StackID", + "sf_id": "ipendpoint", + "sf_id_instance": "ms-A.ict-flame.eu", + "ipendpoint_id": "endpoint1.ms-A.ict-flame.eu", + "sr_id": "service_router", + "influxdb_url": "http://172.40.231.51:8086", + "database_name": "CLMCMetrics" +}, +{ + "name": "ipendpoint2", + "ip_address": "172.40.231.171", + "location": "nova", + "sfc_id": "media_service_A", + "sfc_id_instance": "StackID", + "sf_id": "ipendpoint", + "sf_id_instance": "ms-A.ict-flame.eu", + "ipendpoint_id": "endpoint2.ms-A.ict-flame.eu", + "sr_id": "service_router", + "influxdb_url": "http://172.40.231.51:8086", + "database_name": "CLMCMetrics" +}, +{ + "name": "test-runner", + "ip_address": "172.40.231.200" +} +] \ No newline at end of file diff --git a/src/test/clmctest/scripts/rspec.yml b/src/test/clmctest/scripts/rspec.yml deleted file mode 100644 index 11d3ca5d505117a27c68c8b4544e630c5eadc4e8..0000000000000000000000000000000000000000 --- a/src/test/clmctest/scripts/rspec.yml +++ /dev/null @@ -1,28 +0,0 @@ -## (c) University of Southampton IT Innovation Centre, 2018 -## -## Copyright in this software belongs to University of Southampton -## IT Innovation Centre of Gamma House, Enterprise Road, -## Chilworth Science Park, Southampton, SO16 7NS, UK. -## -## This software may not be used, sold, licensed, transferred, copied -## or reproduced in whole or in part in any manner or form or in or -## on any media by any person other than in accordance with the terms -## of the Licence Agreement supplied with the software, or otherwise -## without the prior written consent of the copyright owners. -## -## This software is distributed WITHOUT ANY WARRANTY, without even the -## implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR -## PURPOSE, except where stated in the Licence Agreement supplied with -## the software. -## -## Created By : Michael Boniface -## Created Date : 20-03-2018 -## Created for Project : FLAME - -hosts: - - name: test-runner - cpus: 1 - memory: 2048 - disk: "10GB" - ip_address: "172.40.231.200" - \ No newline at end of file diff --git a/src/test/clmctest/scripts/test_config_telegraf.py b/src/test/clmctest/scripts/test_config_telegraf.py index f04471d7a0713ce1b12e120932151c73eebfab07..acd8866fc62f7fac51f8c6bf0f844b032d98c6f3 100644 --- a/src/test/clmctest/scripts/test_config_telegraf.py +++ b/src/test/clmctest/scripts/test_config_telegraf.py @@ -35,7 +35,7 @@ def test_write_telegraf_conf(): SF_ID="streaming_service" SF_ID_INSTANCE="streaming_service_instance" IP_ENDPOINT_ID="endpoint" - SR_ID="sr_id" + SR_ID="service_router" INFLUXDB_URL="http://172.29.236.10" DATABASE_NAME="experimentation_database" @@ -76,8 +76,8 @@ def test_write_telegraf_conf(): assert lines.find(SFC_ID_INSTANCE), "Cannot find sfc_id_instance" assert lines.find(SF_ID), "Cannot find sfc_id" assert lines.find(SF_ID_INSTANCE), "Cannot find sf_id_instance" - assert lines.find(IP_ENDPOINT_ID), "Cannot find endpoint id" - assert lines.find(SR_ID), "Cannot find sr_id" + assert lines.find(IP_ENDPOINT_ID), "Cannot find endpoint" + assert lines.find(SR_ID), "Cannot find sr_id" except FileNotFoundError: assert False, "Telegraf general conf file not found, " + TELEGRAF_GENERAL_CONF_FILE diff --git a/src/test/clmctest/services/apache/install.sh b/src/test/clmctest/services/apache/install.sh old mode 100644 new mode 100755 diff --git a/src/test/clmctest/services/apache/telegraf_apache.conf b/src/test/clmctest/services/apache/telegraf_apache.conf index 3a4650b81037cac5b4d7a99290f62ddb87117807..195e7175aed7e33015813d3ab10ed5d9c753de52 100644 --- a/src/test/clmctest/services/apache/telegraf_apache.conf +++ b/src/test/clmctest/services/apache/telegraf_apache.conf @@ -23,7 +23,7 @@ ## An array of URLs to gather from, must be directed at the machine ## readable version of the mod_status page including the auto query string. ## Default is "http://localhost/server-status?auto". - urls = ["http://localhost:8890/server-status?auto"] + urls = ["http://localhost/server-status?auto"] ## Credentials for basic HTTP authentication. # username = "myuser" diff --git a/src/test/clmctest/services/ffmpeg/install.sh b/src/test/clmctest/services/ffmpeg/install.sh old mode 100644 new mode 100755 diff --git a/src/test/clmctest/services/ffmpeg/transcode.sh b/src/test/clmctest/services/ffmpeg/transcode.sh old mode 100644 new mode 100755 diff --git a/src/test/clmctest/services/host/install.sh b/src/test/clmctest/services/host/install.sh old mode 100644 new mode 100755 diff --git a/src/test/clmctest/services/ipendpoint/install.sh b/src/test/clmctest/services/ipendpoint/install.sh old mode 100644 new mode 100755 diff --git a/src/test/clmctest/services/loadtest-streaming/install.sh b/src/test/clmctest/services/loadtest-streaming/install.sh deleted file mode 100644 index 7d6ef6ddc357283c53da5f0475a517002c685f14..0000000000000000000000000000000000000000 --- a/src/test/clmctest/services/loadtest-streaming/install.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -#///////////////////////////////////////////////////////////////////////// -#// -#// (c) University of Southampton IT Innovation Centre, 2017 -#// -#// Copyright in this software belongs to University of Southampton -#// IT Innovation Centre of Gamma House, Enterprise Road, -#// Chilworth Science Park, Southampton, SO16 7NS, UK. -#// -#// This software may not be used, sold, licensed, transferred, copied -#// or reproduced in whole or in part in any manner or form or in or -#// on any media by any person other than in accordance with the terms -#// of the Licence Agreement supplied with the software, or otherwise -#// without the prior written consent of the copyright owners. -#// -#// This software is distributed WITHOUT ANY WARRANTY, without even the -#// implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR -#// PURPOSE, except where stated in the Licence Agreement supplied with -#// the software. -#// -#// Created By : Michael Boniface -#// Created Date : 14/02/2018 -#// Created for Project : FLAME -#// -#///////////////////////////////////////////////////////////////////////// -set -euo pipefail - -echo "REPO_ROOT:"$REPO_ROOT -eval '$REPO_ROOT/test/services/vlc/install.sh' -eval '$REPO_ROOT/test/services/pytest/install.sh' \ No newline at end of file diff --git a/src/test/clmctest/services/loadtest-streaming/telegraf_loadtest_streaming.conf b/src/test/clmctest/services/loadtest-streaming/telegraf_loadtest_streaming.conf deleted file mode 100644 index 3e30465f39ca7035ec8217909c9bc5d29942fa4e..0000000000000000000000000000000000000000 --- a/src/test/clmctest/services/loadtest-streaming/telegraf_loadtest_streaming.conf +++ /dev/null @@ -1,112 +0,0 @@ -# Telegraf configuration - -# Telegraf is entirely plugin driven. All metrics are gathered from the -# declared inputs, and sent to the declared outputs. - -# Plugins must be declared in here to be active. -# To deactivate a plugin, comment out the name and any variables. - -# Use 'telegraf -config telegraf.conf -test' to see what metrics a config -# file would generate. - -# Global tags can be specified here in key="value" format. -[global_tags] - # location of the data centre - location="{{LOCATION}}" - # media service template id - sfc="{{SFC_ID}}" - # media service instance - sfc_i="{{SFC_ID_INSTANCE}}" - # service function type - sf="{{SF_ID}}" - # service function instance id - sf_i="{{SF_ID_INSTANCE}}" - # ipendpoint id aka surrogate instance - ipendpoint="{{IP_ENDPOINT_ID}}" - -# Configuration for telegraf agent -[agent] - ## Default data collection interval for all inputs - interval = "10s" - ## Rounds collection interval to 'interval' - ## ie, if interval="10s" then always collect on :00, :10, :20, etc. - round_interval = true - - ## Telegraf will cache metric_buffer_limit metrics for each output, and will - ## flush this buffer on a successful write. - metric_buffer_limit = 1000 - ## Flush the buffer whenever full, regardless of flush_interval. - flush_buffer_when_full = true - - ## Collection jitter is used to jitter the collection by a random amount. - ## Each plugin will sleep for a random time within jitter before collecting. - ## This can be used to avoid many plugins querying things like sysfs at the - ## same time, which can have a measurable effect on the system. - collection_jitter = "0s" - - ## Default flushing interval for all outputs. You shouldn't set this below - ## interval. Maximum flush_interval will be flush_interval + flush_jitter - flush_interval = "10s" - ## Jitter the flush interval by a random amount. This is primarily to avoid - ## large write spikes for users running a large number of telegraf instances. - ## ie, a jitter of 5s and interval 10s means flushes will happen every 10-15s - flush_jitter = "0s" - - ## Logging configuration: - ## Run telegraf in debug mode - debug = false - ## Run telegraf in quiet mode - quiet = false - ## Specify the log file name. The empty string means to log to stdout. - logfile = "/var/log/telegraf/telegraf.log" - - ## Override default hostname, if empty use os.Hostname() - hostname = "" - - -############################################################################### -# OUTPUTS # -############################################################################### - -# Configuration for influxdb server to send metrics to -[[outputs.influxdb]] - # The full HTTP or UDP endpoint URL for your InfluxDB instance. - # Multiple urls can be specified but it is assumed that they are part of the same - # cluster, this means that only ONE of the urls will be written to each interval. - # urls = ["udp://127.0.0.1:8089"] # UDP endpoint example - urls = ["{{INFLUXDB_URL}}"] # required - # The target database for metrics (telegraf will create it if not exists) - database = "{{DATABASE_NAME}}" # required - # Precision of writes, valid values are "ns", "us" (or "µs"), "ms", "s", "m", "h". - # note: using second precision greatly helps InfluxDB compression - precision = "s" - - ## Write timeout (for the InfluxDB client), formatted as a string. - ## If not provided, will default to 5s. 0s means no timeout (not recommended). - timeout = "5s" - # username = "telegraf" - # password = "metricsmetricsmetricsmetrics" - # Set the user agent for HTTP POSTs (can be useful for log differentiation) - # user_agent = "telegraf" - # Set UDP payload size, defaults to InfluxDB UDP Client default (512 bytes) - # udp_payload = 512 - - -############################################################################### -# INPUTS # -############################################################################### -# # Influx HTTP write listener -[[inputs.http_listener]] - ## Address and port to host HTTP listener on - service_address = ":8186" - - ## timeouts - read_timeout = "10s" - write_timeout = "10s" - - ## HTTPS - #tls_cert= "/etc/telegraf/cert.pem" - #tls_key = "/etc/telegraf/key.pem" - - ## MTLS - #tls_allowed_cacerts = ["/etc/telegraf/clientca.pem"] \ No newline at end of file diff --git a/src/test/clmctest/services/minio/install.sh b/src/test/clmctest/services/minio/install.sh old mode 100644 new mode 100755 index 5c7ff69d34e0d0f97dc93e3bf2d532a2f08d0c90..24c5e2556fa8fb6cd63aaa09ecbcba723893dbc9 --- a/src/test/clmctest/services/minio/install.sh +++ b/src/test/clmctest/services/minio/install.sh @@ -32,13 +32,7 @@ sudo apt-get install -y wget git GOROOT=/usr/local/go PATH=$PATH:$GOROOT/bin CGO_ENABLED=0 -ENV MINIO_UPDATE=off - -# And also for separate MINIO process -echo "export GOROOT=/usr/local/go" >> ~/.profile -echo "export PATH=$PATH:$GOROOT/bin" >> ~/.profile -echo "export CGO_ENABLED=0" >> ~/.profile -echo "export ENV MINIO_UPDATE=off" >> ~/.profile +env MINIO_UPDATE=off # Install GO # ----------------------------------------------------------------------- @@ -56,7 +50,7 @@ cd minio go install -v -ldflags "$(go run buildscripts/gen-ldflags.go)" # check minio configuration available -MINIO_CONF_SOURCE=$REPO_ROOT"/clmctest/services/minio/minio.conf" +MINIO_CONF_SOURCE=$REPO_ROOT"/src/test/clmctest/services/minio/minio.conf" if [ ! -f "$MINIO_CONF_SOURCE" ]; then echo "Error: MINIO conf file does not exist on in the repo. "$MINIO_CONF_SOURCE exit 1 @@ -72,12 +66,37 @@ if [ ! -f "$MINIO_CONF_TARGET" ]; then exit 1 fi -mkdir -p /vagrant/minio_data +# create local data directory +data_dir=/var/lib/minio/minio_data +mkdir -p ${data_dir} -# Start MINIO +# Install minio as systemctl service # ----------------------------------------------------------------------- -nohup minio server --config-dir /etc/minio /vagrant/minio_data &>/dev/null & -echo Started MINIO +start_script_file="/usr/local/go/bin/minio.service.sh" +echo "#!/bin/bash" > $start_script_file +echo "${GOROOT}/bin/minio server --config-dir /etc/minio ${data_dir} &" >> $start_script_file + +chmod 755 $start_script_file + +file="/lib/systemd/system/minio.service" +echo "[Unit]" > $file +echo "Description=minio" >> $file +echo "After=network.target" >> $file +echo "" >> $file +echo "[Service]" >> $file +echo "Environment=GOROOT=${GOROOT}" >> $file +echo "Environment=PATH=${PATH}" >> $file +echo "Environment=CGO_ENABLED=${CGO_ENABLED}" >> $file +echo "Environment=MINIO_UPDATE=${MINIO_UPDATE}" >> $file +echo "Type=forking" >> $file +echo "ExecStart=${start_script_file}" >> $file +echo "" >> $file +echo "[Install]" >> $file +echo "WantedBy=multi-user.target" >> $file + +systemctl daemon-reload +systemctl enable minio.service +systemctl start minio.service diff --git a/src/test/clmctest/services/mongo/install.sh b/src/test/clmctest/services/mongo/install.sh old mode 100644 new mode 100755 index e00502fe6746d41101372afe1f88a7ffcec11a1a..4c68f77afddce548e5f2676f99d8c7f9168c04c1 --- a/src/test/clmctest/services/mongo/install.sh +++ b/src/test/clmctest/services/mongo/install.sh @@ -25,9 +25,11 @@ #///////////////////////////////////////////////////////////////////////// # Install apache -sudo apt-get update -sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 2930ADAE8CAF5059EE73BB4B58712A2291FA4AD5 +apt-get update +apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 2930ADAE8CAF5059EE73BB4B58712A2291FA4AD5 echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu xenial/mongodb-org/3.6 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-3.6.list -sudo apt-get update -sudo apt-get install -y mongodb-org -sudo service mongod start \ No newline at end of file +apt-get update +apt-get install -y mongodb-org +systemctl enable mongod +systemctl start mongod + \ No newline at end of file diff --git a/src/test/clmctest/services/nginx/install.sh b/src/test/clmctest/services/nginx/install.sh old mode 100644 new mode 100755 index 7080fe826162a59914e787d5a779579b57f5ffea..cc9a9b1e4a543d7eb846303a473952ce9420b939 --- a/src/test/clmctest/services/nginx/install.sh +++ b/src/test/clmctest/services/nginx/install.sh @@ -26,12 +26,12 @@ # Install nginx apt-get update -yes Y | apt-get install nginx +apt-get install nginx -y # Need to set up basic stats as this not configured by default # http://nginx.org/en/docs/http/ngx_http_stub_status_module.html -NGINX_CONF_SOURCE=$REPO_ROOT"/clmctest/services/nginx/nginx.conf" +NGINX_CONF_SOURCE=$REPO_ROOT"/src/test/clmctest/services/nginx/nginx.conf" NGINX_CONF_TARGET="/etc/nginx/nginx.conf" # Check the target telegraf directory exists @@ -49,33 +49,4 @@ if [ ! -f "$NGINX_CONF_TARGET" ]; then fi nginx -s reload -systemctl start nginx - -## install a configuration monitoring service, this needs to be in a venv with the rest of the CLMC -sudo apt-get install python3 python3-pip -y -sudo pip3 install pyaml influxdb - -svc="nginxmon" - -echo "install systemctl monitoring service" -svc_file="${svc}.service" -echo "[Unit]" > $svc_file -echo "Description=nginxmon" >> $svc_file -echo "After=network-online.target" >> $svc_file -echo "" >> $svc_file -echo "[Service]" >> $svc_file -echo "WorkingDirectory=${inst}/${dir}" >> $svc_file -echo "ExecStart=/usr/bin/python3 ${REPO_ROOT}/src/monitoring/systemctl_monitor.py -service nginx -rate 2 -agg 10 -host localhost -port 8186 -db CLMCMetrics" >> $svc_file -echo "ExecStop=/usr/bin/bash ${REPO_ROOT}/src/monitoring/stop_systemctl_monitor.sh" >> $svc_file -echo "" >> $svc_file -echo "[Install]" >> $svc_file -echo "WantedBy=network-online.target" >> $svc_file -sudo cp $svc_file /lib/systemd/system -rm $svc_file - -echo "enable" -sudo systemctl daemon-reload -sudo systemctl enable ${svc} - -echo "start" -sudo systemctl start ${svc} \ No newline at end of file +systemctl start nginx \ No newline at end of file diff --git a/src/test/clmctest/services/pytest/install.sh b/src/test/clmctest/services/pytest/install.sh old mode 100644 new mode 100755 diff --git a/src/test/clmctest/services/vlc/install.sh b/src/test/clmctest/services/vlc/install.sh deleted file mode 100644 index 02e3b8fe01696913d8e5b65967a152bfb485eee3..0000000000000000000000000000000000000000 --- a/src/test/clmctest/services/vlc/install.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -#///////////////////////////////////////////////////////////////////////// -#// -#// (c) University of Southampton IT Innovation Centre, 2017 -#// -#// Copyright in this software belongs to University of Southampton -#// IT Innovation Centre of Gamma House, Enterprise Road, -#// Chilworth Science Park, Southampton, SO16 7NS, UK. -#// -#// This software may not be used, sold, licensed, transferred, copied -#// or reproduced in whole or in part in any manner or form or in or -#// on any media by any person other than in accordance with the terms -#// of the Licence Agreement supplied with the software, or otherwise -#// without the prior written consent of the copyright owners. -#// -#// This software is distributed WITHOUT ANY WARRANTY, without even the -#// implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR -#// PURPOSE, except where stated in the Licence Agreement supplied with -#// the software. -#// -#// Created By : Michael Boniface -#// Created Date : 12/02/2018 -#// Created for Project : FLAME -#// -#///////////////////////////////////////////////////////////////////////// - -add-apt-repository -y ppa:videolan/master-daily -apt-get update -apt-get -y install vlc \ No newline at end of file