#88 initial implementation of LXD for CLMC testing

04fa335c · MJB · 77a5cae1 · 04fa335c · 04fa335c · 04fa335c
Commit 04fa335c authored 6 years ago by MJB
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -49,15 +49,15 @@ test:all:
    - echo "REPO_USER=${REPO_USER}" > $CI_PROJECT_DIR/reporc 
    - echo "REPO_PASS=${REPO_PASS}" >> $CI_PROJECT_DIR/reporc
    - sudo scripts/test/fixture.sh create -f src/test/clmctest/rspec.json -r $CI_PROJECT_DIR
-    - sudo mkdir /var/lib/lxc/test-runner/rootfs/vagrant/build
+    - sudo mkdir /var/lib/lxd/containers/test-runner/rootfs/vagrant/build
-    - sudo cp build/clmctest-SNAPSHOT.tar.gz /var/lib/lxc/test-runner/rootfs/vagrant/build
+    - sudo cp build/clmctest-SNAPSHOT.tar.gz /var/lib/lxd/containers/test-runner/rootfs/vagrant/build
-    - sudo cp build/clmcservice-SNAPSHOT.tar.gz /var/lib/lxc/test-runner/rootfs/vagrant/build
+    - sudo cp build/clmcservice-SNAPSHOT.tar.gz /var/lib/lxd/containers/test-runner/rootfs/vagrant/build
-    - sudo lxc-attach -n test-runner -- pip3 install /vagrant/build/clmctest-SNAPSHOT.tar.gz
+    - sudo lxc exec test-runner -- pip3 install /vagrant/build/clmctest-SNAPSHOT.tar.gz
-    - sudo lxc-attach -n test-runner -- pip3 install /vagrant/build/clmcservice-SNAPSHOT.tar.gz
+    - sudo lxc exec test-runner -- pip3 install /vagrant/build/clmcservice-SNAPSHOT.tar.gz
-    - sudo lxc-attach -n test-runner -- pytest -s --tb=short -rfp --pyargs clmctest.scripts
+    - sudo lxc exec test-runner -- pytest -s --tb=short -rfp --pyargs clmctest.scripts
-    - sudo lxc-attach -n test-runner -- pytest -s --tb=short -rfp --pyargs clmcservice.tests  
+    - sudo lxc exec test-runner -- pytest -s --tb=short -rfp --pyargs clmcservice.tests  
-    - sudo lxc-attach -n test-runner -- pytest -s --tb=short -rfp --pyargs clmctest.inputs
+    - sudo lxc exec test-runner -- pytest -s --tb=short -rfp --pyargs clmctest.inputs
-    - sudo lxc-attach -n test-runner -- pytest -s --tb=short -rfp --pyargs clmctest.monitoring
+    - sudo lxc exec test-runner -- pytest -s --tb=short -rfp --pyargs clmctest.monitoring
  when: on_success      
 clean:

--- a/Vagrantfile
+++ b/Vagrantfile
 $lxc_script = <<-SCRIPT
 apt-get update
+apt-get install python3 python3-setuptools jq -y
 # install util for persistent ip tables
 echo iptables-persistent iptables-persistent/autosave_v4 boolean true | sudo debconf-set-selections
 echo iptables-persistent iptables-persistent/autosave_v6 boolean true | sudo debconf-set-selections
 apt-get -y install iptables-persistent
-# install lxc
+lxd init --auto --storage-backend dir 
-apt-get install lxc lxc-templates wget bridge-utils jq -y
+lxc network create lxcbr0 ipv6.address=none ipv4.address=172.40.231.1/24 ipv4.nat=true
-lxc-checkconfig
-# configure lxc for specific CIDR network
-touch /etc/lxc/dnsmasq.conf
-sed -i s/10.0.3/172.40.231/g /etc/default/lxc-net
-sed -i s/#LXC_DHCP_CONFILE/LXC_DHCP_CONFILE/g /etc/default/lxc-net
-service lxc-net restart
 # enable NTP
 # use network time to make sure we are synchronised
 echo "Disabling timesyncd..."
 timedatectl set-ntp no
-until timedatectl | grep -m 1 "Network time on: no";
-do
+# The following hangs with bionic
-  echo "Waiting for timesyncd to turn off.."
+#until timedatectl | grep -m 1 "Network time on: no";
-  sleep 1
+#do
-done
+#  echo "Waiting for timesyncd to turn off.."
-apt-get install ntp
+#  sleep 1
+#done
+apt-get install ntp -y
 echo "timesync set to ntpd"
 # set timezone to London
@@ -35,7 +32,8 @@ timedatectl set-timezone Europe/London
 SCRIPT
 Vagrant.configure("2") do |config|
-  config.vm.box = "ubuntu/xenial64"
+  config.vm.box = "ubuntu/bionic64"
+  config.disksize.size = '50GB'
  config.vm.provider "virtualbox" do |vb|
    vb.cpus = 4
    vb.memory = "8192"

--- a/scripts/clmc-service/install-neo4j.sh
+++ b/scripts/clmc-service/install-neo4j.sh
@@ -55,7 +55,7 @@ while true; do
    sleep 1
 done
-apt-get -y install python3 python3-pip
+apt-get -y install python3 python3-pip python3-setuptools
 update-alternatives --install /usr/bin/python python /usr/bin/python3 10
 apt-get update
 pip3 install influxdb py2neo
--- a/scripts/test/fixture.sh
+++ b/scripts/test/fixture.sh
@@ -9,32 +9,37 @@ create() {
    service_name=$1
    config_file=$2
    repo_root=$3
-    if ! lxc-info -n ${service_name}; then
+    if ! lxc list | grep ${service_name}; then
        # create a container with a static ip address
        echo "Creating container: ${service_name}"
        SERVICE=$(jq --arg NAME ${service_name} '.[] | select(.name==$NAME)' ${config_file})
        echo $SERVICE
        ip=$(echo $SERVICE | jq -r '.ip_address')
-        echo "dhcp-host=${service_name},${ip}" >> /etc/lxc/dnsmasq.conf
-        service lxc-net restart
-        lxc-create -t download -n ${service_name} -- --dist ubuntu --release xenial --arch amd64
+        lxc init ubuntu:16.04 ${service_name}
+        echo "Creating network: ${service_name}"
+        lxc network attach lxcbr0 ${service_name} eth0
+        lxc config device set ${service_name} eth0 ipv4.address ${ip}
+        lxc config set ${service_name} security.privileged true
        # copy flame clmc files into the root container
        echo "Copying files to rootfs"
-        container_dir="/var/lib/lxc/"${service_name}"/rootfs"
+        container_dir="/var/lib/lxd/containers/"${service_name}"/rootfs"
        container_vagrant_dir=${container_dir}"/vagrant"
-        mkdir -p ${container_vagrant_dir}
+        mkdir -p ${container_vagrant_dir}       
-        cp -f ${repo_root}/reporc "${container_vagrant_dir}"
+        cp -f ${repo_root}/reporc ${container_vagrant_dir}   
-        cp -rf ${repo_root}/scripts ${container_vagrant_dir}
+        cp -rf ${repo_root}/scripts ${container_vagrant_dir}     
        cp -rf ${repo_root}/src ${container_vagrant_dir}
+        chown -R 100000:100000 ${container_vagrant_dir} 
        # start the container
        echo "Starting: ${service_name}"
-        lxc-start -n ${service_name}
+        lxc start ${service_name}
-        echo "Waiting for container to start: ${service_name}"
+        while :; do
-        STARTED="0"
+            echo "Waiting for container to start: ${service_name}"
-        while [ "$STARTED" == "0" ]; do STARTED=$(lxc-info -n ${service_name} -i | wc -l); done;
+            lxc file pull ${service_name}/etc/resolv.conf - | grep -q nameserver && break
+            sleep 1
+        done
        # provision software into each container
        echo "Provisioning: ${service_name}"
@@ -44,11 +49,12 @@ create() {
            database_name=$(echo $SERVICE | jq -r '.database_name')
            report_period=$(echo $SERVICE | jq -r '.report_period')
            cmd="/vagrant/scripts/clmc-service/install.sh ${influxdb_url} ${database_name} ${report_period}"
-            lxc-attach -n ${service_name} -v REPO_ROOT="/vagrant" -- ${cmd}
+            echo "Provisioning command ${cmd}"
+            lxc exec ${service_name} --env REPO_ROOT="/vagrant" -- ${cmd}
        elif [ ${service_name} == "test-runner" ]
        then
            cmd=/vagrant/src/test/clmctest/services/pytest/install.sh
-            lxc-attach -n ${service_name} -- ${cmd}
+            lxc exec ${service_name} -- ${cmd}
        else
            # get container parameters
            location=$(echo $SERVICE | jq -r '.location')
@@ -63,29 +69,30 @@ create() {
            # install service function specific software
            cmd=/vagrant/src/test/clmctest/services/${sf_id}/install.sh
-            lxc-attach -n ${service_name} -v REPO_ROOT="/vagrant" -- ${cmd}
+            lxc exec ${service_name} --env REPO_ROOT="/vagrant" -- ${cmd}
            # install telegraf
            cmd=/vagrant/scripts/clmc-agent/install.sh
-            lxc-attach -n ${service_name} -v REPO_ROOT="/vagrant" -- ${cmd}
+            lxc exec ${service_name} --env REPO_ROOT="/vagrant" -- /vagrant/scripts/clmc-agent/install.sh
            # stop telegraf before changing the configs
-            lxc-attach -n ${service_name} -- service telegraf stop
+            lxc exec ${service_name} -- service telegraf stop
            # copy telegraf configuration templates
            cp -f ${repo_root}/scripts/clmc-agent/telegraf.conf ${container_dir}/etc/telegraf/
            cp -f ${repo_root}/scripts/clmc-agent/telegraf_output.conf ${container_dir}/etc/telegraf/telegraf.d/
            # copy the 'host' config into all service containers
-            cp ${repo_root}/src/test/clmctest/services/host/telegraf*.conf ${container_dir}/etc/telegraf/telegraf.d/
+            cp -f ${repo_root}/src/test/clmctest/services/host/telegraf*.conf ${container_dir}/etc/telegraf/telegraf.d/
            # copy the service-specific config
-            cp ${repo_root}/src/test/clmctest/services/${sf_id}/telegraf*.conf ${container_dir}/etc/telegraf/telegraf.d/
+            cp -f ${repo_root}/src/test/clmctest/services/${sf_id}/telegraf*.conf ${container_dir}/etc/telegraf/telegraf.d/
+            chown -R 100000:100000 ${container_dir}/etc/telegraf/
            # replace telegraf template with container parameters
            cmd="/vagrant/scripts/clmc-agent/configure.sh ${location} ${sfc_id} ${sfc_id_instance} ${sf_id} ${sf_id_instance} ${ipendpoint_id} ${sr_id} ${influxdb_url} ${database_name}"
-            lxc-attach -n ${service_name} -- ${cmd}
+            lxc exec ${service_name} -- ${cmd}
            # start telegraf
-            lxc-attach -n ${service_name} -- service telegraf start
+            lxc exec ${service_name} -- service telegraf start
        fi
        # set forward ports
@@ -97,7 +104,7 @@ create() {
                }
                guest_port=$(_jq '.guest')
                host_port=$(_jq '.host')
-                iptables -t nat -A PREROUTING -p tcp -i enp0s3 --dport ${host_port} -j DNAT --to-destination ${ip}:${guest_port}
+                iptables -t nat -I PREROUTING -i enp0s3 -p TCP -d 10.0.2.15 --dport ${host_port} -j DNAT --to-destination ${ip}:${guest_port}
            done
        fi
    fi
@@ -105,33 +112,28 @@ create() {
 start() {
    service_name=$1
-    if lxc-info -n ${service_name}; then
+    if lxc info ${service_name}; then
        echo "Starting container: ${service_name}"
-        lxc-start -n ${service_name}
+        lxc start ${service_name}
    fi
 }
 stop() {
    service_name=$1
-    if lxc-info -n ${service_name}; then
+    if lxc info ${service_name}; then
        echo "Stopping container: ${service_name}"
-        lxc-stop -n ${service_name}
+        lxc stop -n ${service_name}
    fi
 }
 destroy() {
    service_name=$1
    config_file=$2
-    if lxc-info -n ${service_name}; then
+    if lxc list | grep ${service_name}; then
        echo "Stopping container: ${service_name}"
-        lxc-stop -n ${service_name}
+        lxc stop ${service_name}
        echo "Destroying container: ${service_name}"
-        lxc-destroy -n ${service_name}
+        lxc delete ${service_name}
-        # remove static ip
-        SERVICE=$(jq --arg NAME ${service_name} '.[] | select(.name==$NAME)' ${config_file})
-        ip=$(echo $SERVICE | jq -r '.ip_address')
-        sed -i "/dhcp-host=${service_name},/d" /etc/lxc/dnsmasq.conf
        # remove forward ports
        ports=$(echo $SERVICE | jq -r '.forward_ports')
@@ -143,7 +145,7 @@ destroy() {
                }
                guest_port=$(_jq '.guest')
                host_port=$(_jq '.host')
-                iptables -t nat -D PREROUTING -p tcp -i enp0s3 --dport ${host_port} -j DNAT --to-destination ${ip}:${guest_port}
+                iptables -t nat -D PREROUTING -i enp0s3 -p TCP -d 10.0.2.15 --dport ${host_port} -j DNAT --to-destination ${ip}:${guest_port}
            done
        fi
    fi
@@ -208,8 +210,6 @@ for service_name in $service_names; do
    fi
 done
 echo "------>Create iptables summary"
 iptables -t nat -L -n -v
 iptables-save > /etc/iptables/rules.v4
--- a/scripts/test/install-git-runner.sh
+++ b/scripts/test/install-git-runner.sh
@@ -12,5 +12,7 @@ gitlab-runner start
 gitlab-runner register -n --url https://gitlab.it-innovation.soton.ac.uk/ --r sN3wMQp8EiAv7znNwG5s --executor shell --locked true --name ${HOSTNAME} --description ${HOSTNAME} --run-untagged true
-apt-get -y install python3 python3-pip python-influxdb
+# install test prerequisites
+apt-get -y install python3 python3-pip python-influxdb jq
 update-alternatives --install /usr/bin/python python /usr/bin/python3 10