Merge remote-tracking branch 'origin' into clmcservice

README.md

@@ -62,14 +62,14 @@ SSH into the VM
 The containers are controlled using a script called /vagrant/scripts/test/fixtures.sh
 
 ```
-Usage: fixturs.sh create|start|stop|destroy [-f config_file] [-r repo_root] [-c service_name]"
+Usage: fixture.sh create|start|stop|destroy [-f config_file] [-r repo_root] [-c service_name]"
 ```
 
 To create all the services needed for integration tests
 
 ```
 sudo su
-/vagrant/scripts/test/fixtures.sh create -f /vagrant/src/test/clmctest/rspec.json
+/vagrant/scripts/test/fixture.sh create -f /vagrant/src/test/clmctest/rspec.json
 ```
 
 The containers created are defined an rspec.json file, there's an example here `/vagrant/src/test/clmctest/rspec.json`
@@ -79,7 +79,7 @@ The `fixtures.sh` script defaults to look for a rspec.json in the current direct
 To create|start|stop|destroy specific services use the -c option e.g.
 
 ```
-/vagrant/scripts/test/fixtures.sh create -f /vagrant/src/test/clmctest/rspec.json -c clmc-service
+/vagrant/scripts/test/fixture.sh create -f /vagrant/src/test/clmctest/rspec.json -c clmc-service
 ```
 
 Attach to the test-runner to run the tests

Vagrantfile

 $lxc_script = <<-SCRIPT
 
 apt-get update
+
+# install util for persistent ip tables
+echo iptables-persistent iptables-persistent/autosave_v4 boolean true | sudo debconf-set-selections
+echo iptables-persistent iptables-persistent/autosave_v6 boolean true | sudo debconf-set-selections
+apt-get -y install iptables-persistent
+
+# install lxc
 apt-get install lxc lxc-templates wget bridge-utils jq -y
 lxc-checkconfig
 
+# configure lxc for specific CIDR network
 touch /etc/lxc/dnsmasq.conf
 sed -i s/10.0.3/172.40.231/g /etc/default/lxc-net
 sed -i s/#LXC_DHCP_CONFILE/LXC_DHCP_CONFILE/g /etc/default/lxc-net
@@ -18,7 +26,7 @@ Vagrant.configure("2") do |config|
     vb.memory = "8192"
   end
 
-  #config.vm.network "forwarded_port", guest: 80, host: 8080
+  #config.vm.network "forwarded_port", guest: 8888, host: 8888
 
   # Install lxc
   config.vm.provision :shell, inline: $lxc_script

Vagrantfile.libvirt

 $lxc_script = <<-SCRIPT
 
 apt-get update
+
+# install util for persistent ip tables
+echo iptables-persistent iptables-persistent/autosave_v4 boolean true | sudo debconf-set-selections
+echo iptables-persistent iptables-persistent/autosave_v6 boolean true | sudo debconf-set-selections
+apt-get -y install iptables-persistent
+
+# install lxc
 apt-get install lxc lxc-templates wget bridge-utils jq -y
 lxc-checkconfig
 
+# configure lxc for specific CIDR network
 touch /etc/lxc/dnsmasq.conf
 sed -i s/10.0.3/172.40.231/g /etc/default/lxc-net
 sed -i s/#LXC_DHCP_CONFILE/LXC_DHCP_CONFILE/g /etc/default/lxc-net

scripts/clmc-service/install.sh

@@ -160,7 +160,7 @@ fi
 
 # create directory for CLMC service logs
 echo "----> Creating CLMC web service log directory"
-sudo mkdir /var/log/clmcservice
+mkdir -p /var/log/flame/clmc
 
 # Install minioclmc as systemctl service
 # -----------------------------------------------------------------------
@@ -170,7 +170,7 @@ echo "#!/bin/bash" > $start_script_file
 echo "export WORKON_HOME=${HOME}/.virtualenvs" >> $start_script_file
 echo "source /usr/local/bin/virtualenvwrapper.sh" >> $start_script_file
 echo "workon CLMC" >> $start_script_file
-echo "pserve ${REPO_ROOT}/src/service/production.ini > /dev/null 2>&1 &" >> $start_script_file
+echo "pserve ${REPO_ROOT}/src/service/production.ini &" >> $start_script_file
 
 chmod 755 $start_script_file
 

scripts/test/fixture.sh

@@ -84,6 +84,19 @@ create() {
             # start telegraf
             lxc-attach -n ${service_name} -- service telegraf restart
         fi
+
+        # set forward ports
+        ports=$(echo $SERVICE | jq -r '.forward_ports')
+        if [ "$ports" != "null" ]; then
+            for row in $(echo "${ports}" | jq -r '.[] | @base64'); do
+                _jq() {
+                echo ${row} | base64 --decode | jq -r ${1}
+                }
+                guest_port=$(_jq '.guest')
+                host_port=$(_jq '.host')
+                iptables -t nat -A PREROUTING -p tcp -i enp0s3 --dport ${host_port} -j DNAT --to-destination ${ip}:${guest_port}
+            done
+        fi        
     fi
 }
 
@@ -111,8 +124,25 @@ destroy() {
         lxc-stop -n ${service_name}
         echo "Destroying container: ${service_name}"
         lxc-destroy -n ${service_name}
-        ip=$(jq -r --arg NAME ${service_name} '.[] | select(.name==$NAME) | .ip_address' ${config_file})
+
+        # remove static ip
+        SERVICE=$(jq --arg NAME ${service_name} '.[] | select(.name==$NAME)' ${config_file})
+        ip=$(echo $SERVICE | jq -r '.ip_address')
         sed -i "/dhcp-host=${service_name},/d" /etc/lxc/dnsmasq.conf
+
+        # remove forward ports
+        ports=$(echo $SERVICE | jq -r '.forward_ports')
+        if [ "$ports" != "null" ]; then
+            echo "destroy ports"
+            for row in $(echo "${ports}" | jq -r '.[] | @base64'); do
+                _jq() {
+                echo ${row} | base64 --decode | jq -r ${1}
+                }
+                guest_port=$(_jq '.guest')
+                host_port=$(_jq '.host')
+                iptables -t nat -D PREROUTING -p tcp -i enp0s3 --dport ${host_port} -j DNAT --to-destination ${ip}:${guest_port}
+            done
+        fi        
     fi    
 }
 
@@ -174,3 +204,9 @@ for service_name in $service_names; do
         esac
     fi
 done
+
+
+
+echo "------>Create iptables summary"
+iptables -t nat -L -n -v
+iptables-save > /etc/iptables/rules.v4

src/service/clmcservice/aggregation/aggregator.py

@@ -246,7 +246,7 @@ class AggregatorThread(Thread):
 if __name__ == '__main__':
     # initialise a file logger, only when module's main method is run (NOT when aggregator class is imported somewhere else)
     log = logging.getLogger('aggregator')
-    hdlr = logging.FileHandler('/var/log/clmcservice/aggregator.log', mode='a')
+    hdlr = logging.FileHandler('/var/log/flame/clmc/aggregator.log', mode='a')
     formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
     hdlr.setFormatter(formatter)
     log.addHandler(hdlr)

src/service/development.ini

@@ -67,13 +67,13 @@ formatter = generic
 
 [handler_filelog]
 class = FileHandler
-args = ('/var/log/clmcservice/service.log','a')
+args = ('/var/log/flame/clmc/service.log','a')
 level = NOTSET
 formatter = generic
 
 [handler_exc_handler]
 class = FileHandler
-args = ('/var/log/clmcservice/service-exceptions.log', 'a')
+args = ('/var/log/flame/clmc/service-exceptions.log', 'a')
 level = ERROR
 formatter = exc_formatter
 

src/service/production.ini

@@ -33,7 +33,7 @@ listen = *:9080
 ###
 
 [loggers]
-keys = root, exc_logger, clmcservice
+keys = root, exc_logger, service_logger
 
 [handlers]
 keys = console, filelog, exc_handler
@@ -45,7 +45,7 @@ keys = generic, exc_formatter
 level = INFO
 handlers = console
 
-[logger_clmcservice]
+[logger_service_logger]
 level = INFO
 handlers = filelog
 qualname = service_logger
@@ -63,13 +63,13 @@ formatter = generic
 
 [handler_filelog]
 class = FileHandler
-args = ('/var/log/clmcservice/service.log','a')
+args = ('/var/log/flame/clmc/service.log','a')
 level = NOTSET
 formatter = generic
 
 [handler_exc_handler]
 class = FileHandler
-args = ('/var/log/clmcservice/service-exceptions.log', 'a')
+args = ('/var/log/flame/clmc/service-exceptions.log', 'a')
 level = ERROR
 formatter = exc_formatter
 

src/test/clmctest/rspec.json

@@ -3,7 +3,13 @@
     "ip_address": "172.40.231.51",
     "influxdb_url": "http://172.40.231.51:8086",
     "database_name": "CLMCMetrics",
-    "report_period": "25"    
+    "report_period": "25",  
+    "forward_ports": [
+        { "guest": 8086, "host": 8086 },
+        { "guest": 8888, "host": 8888 },
+        { "guest": 9092, "host": 9092 },
+        { "guest": 9080, "host": 9080 }    
+    ]
 },
 {
     "name": "apache",