Skip to content
Snippets Groups Projects
Commit 5eeabdc8 authored by MJB's avatar MJB
Browse files

#44 missed commits

parent 0c107602
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 49 additions and 363 deletions
.gitlab-ci.yml
+ 1
− 1
View file @ 5eeabdc8
......@@ -63,5 +63,5 @@ clean:
only:
- schedules
script:
- python scripts/build/deleteallvms.py
- python scripts/test/deleteallvms.py
when: always
\ No newline at end of file
Vagrantfile
+ 4
− 1
View file @ 5eeabdc8
......@@ -4,7 +4,10 @@ apt-get update
apt-get install lxc lxc-templates wget bridge-utils jq -y
lxc-checkconfig
sudo cp -f /vagrant/clmctest/lxc-net /etc/default/
touch /etc/lxc/dnsmasq.conf
sed -i s/10.0.3/172.40.231/g /etc/default/lxc-net
sed -i s/#LXC_DHCP_CONFILE/LXC_DHCP_CONFILE/g /etc/default/lxc-net
service lxc-net restart
SCRIPT
......
clmctest/conts-create.sh deleted 100755 → 0
+ 0
− 76
View file @ 0c107602
#!/bin/bash
set -eu -o pipefail
rspec_file=rspec.json
service_names=$(jq -r '.[].name' ${rspec_file})
for service_name in $service_names; do
if ! lxc-info -n ${service_name}; then
echo "Creating container: ${service_name}"
lxc-create -t download -n ${service_name} -- --dist ubuntu --release xenial --arch amd64
SERVICE=$(jq --arg NAME ${service_name} '.[] | select(.name==$NAME)' ${rspec_file})
echo $SERVICE
ip=$(echo $SERVICE | jq -r '.ip_address')
echo "dhcp-host=${service_name},${ip}" >> /etc/lxc/dnsmasq.conf
echo "Copying files"
container_dir="/var/lib/lxc/"${service_name}"/rootfs"
container_vagrant_dir=${container_dir}"/vagrant"
mkdir -p ${container_vagrant_dir}
cp -f /vagrant/reporc "${container_vagrant_dir}"
cp -rf /vagrant/scripts ${container_vagrant_dir}
cp -rf /vagrant/clmctest ${container_vagrant_dir}
# start the container
lxc-start -n ${service_name}
sleep 10
echo "Provisioning: ${service_name}"
if [ ${service_name} == "clmc-service" ]
then
cmd=/vagrant/scripts/clmc-service/install.sh
lxc-attach -n ${service_name} -- ${cmd}
elif [ ${service_name} == "test-runner" ]
then
cmd=/vagrant/clmctest/services/pytest/install.sh
lxc-attach -n ${service_name} -- ${cmd}
else
location=$(echo $SERVICE | jq -r '.location')
sf_id=$(echo $SERVICE | jq -r '.sf_id')
sf_id_instance=$(echo $SERVICE | jq -r '.sf_id_instance')
sfc_id=$(echo $SERVICE | jq -r '.sfc_id')
sfc_id_instance=$(echo $SERVICE | jq -r '.sfc_id_instance')
sr_id=$(echo $SERVICE | jq -r '.sr_id')
ipendpoint_id=$(echo $SERVICE | jq -r '.ipendpoint_id')
influxdb_url=$(echo $SERVICE | jq -r '.influxdb_url')
database_name=$(echo $SERVICE | jq -r '.database_name')
cmd=/vagrant/clmctest/services/${sf_id}/install.sh
lxc-attach -n ${service_name} -v REPO_ROOT=/vagrant -- ${cmd}
cmd=/vagrant/scripts/clmc-agent/install.sh
lxc-attach -n ${service_name} -v REPO_ROOT=/vagrant -- ${cmd}
cp -f /vagrant/scripts/clmc-agent/telegraf.conf ${container_dir}/etc/telegraf/
cp -f /vagrant/scripts/clmc-agent/telegraf_output.conf ${container_dir}/etc/telegraf/telegraf.d/
cp /vagrant/clmctest/services/${sf_id}/telegraf_${sf_id}.conf ${container_dir}/etc/telegraf/telegraf.d/
cmd=/vagrant/scripts/clmc-agent/configure_template.sh
lxc-attach -n ${service_name} -- ${cmd}
cmd="/vagrant/scripts/clmc-agent/configure.sh ${location} ${sfc_id} ${sfc_id_instance} ${sf_id} ${sf_id_instance} ${ipendpoint_id} ${sr_id} ${influxdb_url} ${database_name}"
lxc-attach -n ${service_name} -- ${cmd}
lxc-attach -n ${service_name} -- service telegraf restart
fi
fi
done
./conts-stop.sh
service lxc-net restart
./conts-start.sh
\ No newline at end of file
clmctest/conts-destroy.sh deleted 100755 → 0
+ 0
− 16
View file @ 0c107602
#!/bin/bash
rspec_file=rspec.json
service_names=$(jq -r '.[].name' ${rspec_file})
for service_name in $service_names; do
if lxc-info -n ${service_name}; then
echo "Stopping container: ${service_name}"
lxc-stop -n ${service_name}
echo "Destroying container: ${service_name}"
lxc-destroy -n ${service_name}
ip=$(jq -r --arg NAME ${service_name} '.[] | select(.name==$NAME) | .ip_address' ${rspec_file})
#TODO: this line doesn't work for some reason and it's also dangerous because an IP could match more than 1 line in the file
#sed -i '/${ip}/d' /etc/lxc/dnsmasq.conf
fi
done
clmctest/conts-start.sh deleted 100755 → 0
+ 0
− 11
View file @ 0c107602
#!/bin/bash
rspec_file=rspec.json
service_names=$(jq -r '.[].name' ${rspec_file})
for service_name in $service_names; do
if lxc-info -n ${service_name}; then
echo "Starting container: ${service_name}"
lxc-start -n ${service_name}
fi
done
clmctest/conts-stop.sh deleted 100755 → 0
+ 0
− 11
View file @ 0c107602
#!/bin/bash
rspec_file=rspec.json
service_names=$(jq -r '.[].name' ${rspec_file})
for service_name in $service_names; do
if lxc-info -n ${service_name}; then
echo "Stopping container: ${service_name}"
lxc-stop -n ${service_name}
fi
done
\ No newline at end of file
clmctest/lxc-net deleted 100644 → 0
+ 0
− 32
View file @ 0c107602
# This file is auto-generated by lxc.postinst if it does not
# exist. Customizations will not be overridden.
# Leave USE_LXC_BRIDGE as "true" if you want to use lxcbr0 for your
# containers. Set to "false" if you'll use virbr0 or another existing
# bridge, or mavlan to your host's NIC.
USE_LXC_BRIDGE="true"
# If you change the LXC_BRIDGE to something other than lxcbr0, then
# you will also need to update your /etc/lxc/default.conf as well as the
# configuration (/var/lib/lxc/<container>/config) for any containers
# already created using the default config to reflect the new bridge
# name.
# If you have the dnsmasq daemon installed, you'll also have to update
# /etc/dnsmasq.d/lxc and restart the system wide dnsmasq daemon.
LXC_BRIDGE="lxcbr0"
LXC_ADDR="172.40.231.1"
LXC_NETMASK="255.255.255.0"
LXC_NETWORK="172.40.231.0/24"
LXC_DHCP_RANGE="172.40.231.2,172.40.231.254"
LXC_DHCP_MAX="253"
# Uncomment the next line if you'd like to use a conf-file for the lxcbr0
# dnsmasq. For instance, you can use 'dhcp-host=mail1,10.0.3.100' to have
# container 'mail1' always get ip address 10.0.3.100.
LXC_DHCP_CONFILE=/etc/lxc/dnsmasq.conf
# Uncomment the next line if you want lxcbr0's dnsmasq to resolve the .lxc
# domain. You can then add "server=/lxc/10.0.3.1' (or your actual $LXC_ADDR)
# to your system dnsmasq configuration file (normally /etc/dnsmasq.conf,
# or /etc/NetworkManager/dnsmasq.d/lxc.conf on systems that use NetworkManager).
# Once these changes are made, restart the lxc-net and network-manager services.
# 'container1.lxc' will then resolve on your host.
#LXC_DOMAIN="lxc"
clmctest/rspec.json deleted 100644 → 0
+ 0
− 116
View file @ 0c107602
[{
"name": "clmc-service",
"ip_address": "172.40.231.51"
},
{
"name": "apache",
"ip_address": "172.40.231.150",
"location": "DC1",
"sfc_id": "MS_Template_1",
"sfc_id_instance": "MS_I1",
"sf_id": "apache",
"sf_id_instance": "adaptive_streaming_I1",
"ipendpoint_id": "adaptive_streaming_I1_apache1",
"sr_id": "service_router",
"influxdb_url": "http://172.40.231.51:8086",
"database_name": "CLMCMetrics"
},
{
"name": "nginx",
"ip_address": "172.40.231.151",
"location": "DC1",
"sfc_id": "MS_Template_1",
"sfc_id_instance": "MS_I1",
"sf_id": "nginx",
"sf_id_instance": "adaptive_streaming_nginx_I1",
"ipendpoint_id": "adaptive_streaming_nginx_I1_apache1",
"sr_id": "service_router",
"influxdb_url": "http://172.40.231.51:8086",
"database_name": "CLMCMetrics"
},
{
"name": "mongo",
"service_name": "mongo",
"ip_address": "172.40.231.152",
"location": "DC1",
"sfc_id": "MS_Template_1",
"sfc_id_instance": "MS_I1",
"sf_id": "mongo",
"sf_id_instance": "metadata_database_I1",
"ipendpoint_id": "metadata_database_I1_apache1",
"sr_id": "service_router",
"influxdb_url": "http://172.40.231.51:8086",
"database_name": "CLMCMetrics"
},
{
"name": "ffmpeg",
"service_name": "ffmpeg",
"ip_address": "172.40.231.153",
"location": "DC1",
"sfc_id": "MS_Template_1",
"sfc_id_instance": "MS_I1",
"sf_id": "ffmpeg",
"sf_id_instance": "metadata_database_I1",
"ipendpoint_id": "metadata_database_I1_apache1",
"sr_id": "service_router",
"influxdb_url": "http://172.40.231.51:8086",
"database_name": "CLMCMetrics"
},
{
"name": "host",
"service_name": "host",
"ip_address": "172.40.231.154",
"location": "DC1",
"sfc_id": "MS_Template_1",
"sfc_id_instance": "MS_I1",
"sf_id": "host",
"sf_id_instance": "adaptive_streaming_I1",
"ipendpoint_id": "adaptive_streaming_I1_apache1",
"sr_id": "service_router",
"influxdb_url": "http://172.40.231.51:8086",
"database_name": "CLMCMetrics"
},
{
"name": "minio",
"ip_address": "172.40.231.155",
"location": "DC1",
"sfc_id": "MS_Template_1",
"sfc_id_instance": "MS_I1",
"sf_id": "minio",
"sf_id_instance": "adaptive_streaming_I1",
"ipendpoint_id": "adaptive_streaming_I1_minio",
"sr_id": "service_router",
"influxdb_url": "http://172.40.231.51:8086",
"database_name": "CLMCMetrics"
},
{
"name": "ipendpoint1",
"ip_address": "172.40.231.170",
"location": "nova",
"sfc_id": "media_service_A",
"sfc_id_instance": "StackID",
"sf_id": "ipendpoint",
"sf_id_instance": "ms-A.ict-flame.eu",
"ipendpoint_id": "endpoint1.ms-A.ict-flame.eu",
"sr_id": "service_router",
"influxdb_url": "http://172.40.231.51:8086",
"database_name": "CLMCMetrics"
},
{
"name": "ipendpoint2",
"ip_address": "172.40.231.171",
"location": "nova",
"sfc_id": "media_service_A",
"sfc_id_instance": "StackID",
"sf_id": "ipendpoint",
"sf_id_instance": "ms-A.ict-flame.eu",
"ipendpoint_id": "endpoint2.ms-A.ict-flame.eu",
"sr_id": "service_router",
"influxdb_url": "http://172.40.231.51:8086",
"database_name": "CLMCMetrics"
},
{
"name": "test-runner",
"ip_address": "172.40.231.200"
}
]
\ No newline at end of file
scripts/build/deleteallvms.py deleted 100644 → 0
+ 0
− 46
View file @ 0c107602
#!/bin/python
## © University of Southampton IT Innovation Centre, 2017
##
## Copyright in this software belongs to University of Southampton
## IT Innovation Centre of Gamma House, Enterprise Road,
## Chilworth Science Park, Southampton, SO16 7NS, UK.
##
## This software may not be used, sold, licensed, transferred, copied
## or reproduced in whole or in part in any manner or form or in or
## on any media by any person other than in accordance with the terms
## of the Licence Agreement supplied with the software, or otherwise
## without the prior written consent of the copyright owners.
##
## This software is distributed WITHOUT ANY WARRANTY, without even the
## implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
## PURPOSE, except where stated in the Licence Agreement supplied with
## the software.
##
## Created By : Michael Boniface
## Created Date : 28-03-2018
## Created for Project : FLAME
import subprocess
import sys
proc = subprocess.Popen(['vboxmanage', 'list', 'vms'], stdout=subprocess.PIPE, shell=True)
out, err = proc.communicate()
if proc.returncode != 0:
print("Could list virtualbox vms")
sys.exit()
vms = out.strip().decode('ascii')
lines = vms.split("\n")
for vm in lines:
vm_id = vm.split()[0].replace('"', '')
print("Removing VM: {0}".format(vm_id))
proc = subprocess.Popen(['vboxmanage', 'controlvm', vm_id, 'poweroff'], stdout=subprocess.PIPE, shell=True)
out, err = proc.communicate()
if proc.returncode != 0:
print("Error power off VM: {0}, {1}, {2}".format(vm_id, out, err))
proc = subprocess.Popen(['vboxmanage', 'unregistervm', vm_id, '--delete'], stdout=subprocess.PIPE, shell=True)
out, err = proc.communicate()
if proc.returncode != 0:
print("Error unregister VM: {0}, {1}, {2}".format(vm_id, out, err))
scripts/build/deleteallvms.sh deleted 100755 → 0
+ 0
− 32
View file @ 0c107602
#!/bin/bash
## © University of Southampton IT Innovation Centre, 2017
##
## Copyright in this software belongs to University of Southampton
## IT Innovation Centre of Gamma House, Enterprise Road,
## Chilworth Science Park, Southampton, SO16 7NS, UK.
##
## This software may not be used, sold, licensed, transferred, copied
## or reproduced in whole or in part in any manner or form or in or
## on any media by any person other than in accordance with the terms
## of the Licence Agreement supplied with the software, or otherwise
## without the prior written consent of the copyright owners.
##
## This software is distributed WITHOUT ANY WARRANTY, without even the
## implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
## PURPOSE, except where stated in the Licence Agreement supplied with
## the software.
##
## Created By : Michael Boniface
## Created Date : 28-03-2018
## Created for Project : FLAME
vboxmanage list vms | awk ' {print $1} ' | awk -F',' '{gsub(/"/, "", $1); print $1}' > runningvms
while read vm_id; do
echo $vm_id
vboxmanage controlvm $vm_id poweroff
vboxmanage unregistervm $vm_id --delete
done <runningvms
rm runningvms
scripts/clmc-service/install.sh
+ 3
− 1
View file @ 5eeabdc8
......@@ -54,7 +54,7 @@ CHRONOGRAF_CHECKSUM=eea6915aa6db8f134fcd3b095e863b773bfb3a16a26e346dd65904a07df9
# install virtualenvwrapper to manage python environments - and check
apt-get update
echo "----> Installing Python3 and Pip3"
apt-get install -y python3 python3-pip wget
apt-get install -y python3 python3-pip wget curl
update-alternatives --install /usr/bin/python python /usr/bin/python3 10
echo "----> Installing virtualenv and wrapper"
......@@ -169,9 +169,11 @@ done
# configure the CLMC service
JSON="{\"aggregator_report_period\": ${REPORT_PERIOD}, \"aggregator_database_name\": \"${DATABASE_NAME}\", \"aggregator_database_url\": \"${INFLUX_URL}\"}"
echo "CONFIG JSON=${JSON}"
curl -H 'Content-Type: application/json' -X PUT -d "${JSON}" http://localhost:9080/aggregator/config
# start the aggregator
JSON="{\"action\": \"start\"}"
echo "START ACTION JSON=${JSON}"
curl -H 'Content-Type: application/json' -X PUT -d "${JSON}" http://localhost:9080/aggregator/control
src/test/clmctest/monitoring/conftest.py
+ 2
− 2
View file @ 5eeabdc8
......@@ -93,7 +93,7 @@ def e2e_simulator(streaming_sim_config):
:return: an instance of the E2E simulator
"""
influx_url = "http://" + streaming_sim_config['hosts'][0]['ip_address'] + ":8086"
influx_url = "http://" + streaming_sim_config[0]['ip_address'] + ":8086"
return Simulator(database_url=influx_url)
......@@ -107,6 +107,6 @@ def e2e_aggregator(streaming_sim_config):
:return: an instance of the Aggregator class
"""
influx_url = "http://" + streaming_sim_config['hosts'][0]['ip_address'] + ":8086"
influx_url = "http://" + streaming_sim_config[0]['ip_address'] + ":8086"
return TestAggregator(database_url=influx_url)
src/test/clmctest/services/apache/telegraf_apache.conf
+ 1
− 1
View file @ 5eeabdc8
......@@ -23,7 +23,7 @@
## An array of URLs to gather from, must be directed at the machine
## readable version of the mod_status page including the auto query string.
## Default is "http://localhost/server-status?auto".
urls = ["http://localhost:8890/server-status?auto"]
urls = ["http://localhost/server-status?auto"]
## Credentials for basic HTTP authentication.
# username = "myuser"
......
src/test/clmctest/services/minio/install.sh
+ 29
− 10
View file @ 5eeabdc8
......@@ -34,12 +34,6 @@ PATH=$PATH:$GOROOT/bin
CGO_ENABLED=0
env MINIO_UPDATE=off
# And also for separate MINIO process
echo "export GOROOT=/usr/local/go" >> ~/.profile
echo "export PATH=$PATH:$GOROOT/bin" >> ~/.profile
echo "export CGO_ENABLED=0" >> ~/.profile
echo "export ENV MINIO_UPDATE=off" >> ~/.profile
# Install GO
# -----------------------------------------------------------------------
cd /tmp
......@@ -72,12 +66,37 @@ if [ ! -f "$MINIO_CONF_TARGET" ]; then
exit 1
fi
mkdir -p /vagrant/minio_data
# create local data directory
data_dir=/var/lib/minio/minio_data
mkdir -p ${data_dir}
# Start MINIO
# Install minio as systemctl service
# -----------------------------------------------------------------------
nohup minio server --config-dir /etc/minio /vagrant/minio_data &>/dev/null &
echo Started MINIO
start_script_file="/usr/local/go/bin/minio.service.sh"
echo "#!/bin/bash" > $start_script_file
echo "${GOROOT}/bin/minio server --config-dir /etc/minio ${data_dir} &" >> $start_script_file
chmod 755 $start_script_file
file="/lib/systemd/system/minio.service"
echo "[Unit]" > $file
echo "Description=minio" >> $file
echo "After=network.target" >> $file
echo "" >> $file
echo "[Service]" >> $file
echo "Environment=GOROOT=${GOROOT}" >> $file
echo "Environment=PATH=${PATH}" >> $file
echo "Environment=CGO_ENABLED=${CGO_ENABLED}" >> $file
echo "Environment=MINIO_UPDATE=${MINIO_UPDATE}" >> $file
echo "Type=forking" >> $file
echo "ExecStart=${start_script_file}" >> $file
echo "" >> $file
echo "[Install]" >> $file
echo "WantedBy=multi-user.target" >> $file
systemctl daemon-reload
systemctl enable minio.service
systemctl start minio.service
......
src/test/clmctest/services/mongo/install.sh
+ 7
− 5
View file @ 5eeabdc8
......@@ -25,9 +25,11 @@
#/////////////////////////////////////////////////////////////////////////
# Install apache
sudo apt-get update
sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 2930ADAE8CAF5059EE73BB4B58712A2291FA4AD5
apt-get update
apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 2930ADAE8CAF5059EE73BB4B58712A2291FA4AD5
echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu xenial/mongodb-org/3.6 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-3.6.list
sudo apt-get update
sudo apt-get install -y mongodb-org
sudo service mongod start
\ No newline at end of file
apt-get update
apt-get install -y mongodb-org
systemctl enable mongod
systemctl start mongod
\ No newline at end of file
src/test/clmctest/services/nginx/install.sh
+ 2
− 2
View file @ 5eeabdc8
......@@ -48,5 +48,5 @@ if [ ! -f "$NGINX_CONF_TARGET" ]; then
exit 1
fi
#nginx -s reload
#systemctl start nginx
\ No newline at end of file
nginx -s reload
systemctl start nginx
\ No newline at end of file
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment