Merge branch 'integration' into mediaComponentConfig

.gitattributes

@@ -5,4 +5,6 @@
 scripts/*  text eol=lf
 # Denote all files that are truly binary and should not be modified.
 *.png binary
-*.jpg binary
+*.jpg binary
\ No newline at end of file
+*.build-config/
+*.pytest_cache/
\ No newline at end of file

.githooks/post-commit

+#!/bin/bash
+mkdir .build-config
+git rev-list --count integration > .build-config/git-commit-count
+git describe --all --long > .build-config/git-commit-version
\ No newline at end of file

.gitignore

-/src/clmc-spec/clmc-spec/nbproject/
-/src/clmc-spec/nbproject/
-/src/clmc-spec/target/
 .vagrant/
 .log
+*__pycache__*
+*__init__.pyc
+*egg-info*
 ubuntu-xenial-16.04-cloudimg-console.log
\ No newline at end of file

.gitlab-ci.yml

 stages:  
   - test:scripts
-  - test:streaming_sim
-  - test:telegraf_agents
+  - test:monitoring
+  - test:inputs
   
 test:scripts:
   stage: test:scripts
@@ -9,31 +9,31 @@ test:scripts:
     - vagrant --fixture=scripts -- destroy --force 
   script: 
     - vagrant --fixture=scripts -- up
-    - vagrant --fixture=scripts -- ssh test-runner -- -tt "cd /vagrant && pytest test/scripts/"
+    - vagrant --fixture=scripts -- ssh test-runner -- -tt "cd /vagrant && pytest clmctest/scripts/"
   after_script:
     - vagrant --fixture=scripts -- destroy --force
   when: manual
   
-test:streaming_sim:
+test:monitoring:
   stage: test:streaming_sim
   before_script:
-    - vagrant --fixture=streaming-sim -- destroy --force    
+    - vagrant --fixture=monitoring -- destroy --force    
   script: 
-    - vagrant --fixture=streaming-sim -- up
-    - vagrant --fixture=streaming-sim -- ssh test-runner -- -tt "cd /vagrant && pytest test/streaming-sim/"
+    - vagrant --fixture=monitoring -- up
+    - vagrant --fixture=monitoring -- ssh test-runner -- -tt "cd /vagrant && pytest clmctest/monitoring/"
   after_script:
-    - vagrant --fixture=streaming-sim -- destroy --force  
+    - vagrant --fixture=monitoring -- destroy --force   
   when: manual  
   
-test:telegraf_agents:
-  stage: test:telegraf_agents
+test:inputs:
+  stage: test:inputs
   before_script:
-    - vagrant --fixture=telegraf-agents -- destroy --force      
+    - vagrant --fixture=inputs -- destroy --force      
   script: 
-    - vagrant --fixture=telegraf-agents -- up
-    - vagrant --fixture=telegraf-agents -- ssh test-runner -- -tt "cd /vagrant && pytest test/telegraf-agents/"
+    - vagrant --fixture=inputs -- up
+    - vagrant --fixture=inputs -- ssh test-runner -- -tt "cd /vagrant && pytest clmctest/inputs/"
   after_script:
-    - vagrant --fixture=telegraf-agents -- destroy --force       
+    - vagrant --fixture=inputs -- destroy --force       
   when: manual    
 
   

MANIFEST.in

+include MANIFEST.in
+recursive-include clmctest *.yml *.sh
\ No newline at end of file

Vagrantfile

@@ -48,8 +48,8 @@ end
 
 # load custom config file
 puts "loading custom infrastructure configuration: #{fixture}"
-puts "custom config file: /test/#{fixture}/rspec.yml"
-host_rspec_file = "test/#{fixture}/rspec.yml"
+puts "custom config file: /clmctest/#{fixture}/rspec.yml"
+host_rspec_file = "clmctest/#{fixture}/rspec.yml"
 hosts = YAML.load_file(host_rspec_file)
 
 # Start creating VMS using xenial64 as the base box
@@ -89,21 +89,29 @@ Vagrant.configure("2") do |config|
       puts "Instance name #{instance_name}:"
       case instance_name
         when 'test-runner'
-          instance_config.vm.provision :shell, :path => "test/services/pytest/install.sh"
+          instance_config.vm.provision :shell, :path => "clmctest/services/pytest/install.sh"
         when 'clmc-service'
           instance_config.vm.provision :shell, :path => "scripts/clmc-service/install.sh"       
         else
           # specific service install
-          instance_config.vm.provision :shell, :path => "test/services/#{host["service_name"]}/install.sh", env: {"REPO_ROOT" => "/vagrant"}
+          instance_config.vm.provision :shell, :path => "clmctest/services/#{host["service_name"]}/install.sh", env: {"REPO_ROOT" => "/vagrant"}
     
           # CLMC agent install
           instance_config.vm.provision :shell, :path => "scripts/clmc-agent/install.sh"
 
           # CLMC agent service specific input configuration
-          instance_config.vm.provision :shell, inline: "cp /vagrant/test/services/#{host["service_name"]}/telegraf_#{host["service_name"]}.conf /etc/telegraf/telegraf.d/"     
+          instance_config.vm.provision :shell, inline: <<-SHELL
 
+            cp /vagrant/scripts/clmc-agent/telegraf.conf /etc/telegraf/  
+
+            cp /vagrant/scripts/clmc-agent/telegraf_output.conf /etc/telegraf/telegraf.d/                        
+            
+            cp /vagrant/clmctest/services/#{host["service_name"]}/telegraf_#{host["service_name"]}.conf /etc/telegraf/telegraf.d/ 
+
+          SHELL
+          
           # CLMC agent general and output configuration
-          instance_config.vm.provision :shell, :path => "scripts/clmc-agent/configure_template.sh"
+          #instance_config.vm.provision :shell, :path => "scripts/clmc-agent/configure_template.sh"
 
           instance_config.vm.provision :shell, :path => "scripts/clmc-agent/configure.sh", :args => "#{host["location"]} #{host["sfc_id"]} #{host["sfc_id_instance"]} #{host["sf_id"]} #{host["sf_id_instance"]} #{host["ipendpoint_id"]} #{host["influxdb_url"]} #{host["database_name"]}"  
 

test/telegraf-agents/conftest.py → clmctest/inputs/conftest.py

@@ -2,10 +2,11 @@
 
 import pytest
 import yaml
+import pkg_resources
 from influxdb import InfluxDBClient
 
 
-@pytest.fixture(scope="module", params=[{'config': {'rspec': 'test/telegraf-agents/rspec.yml'}}])
+@pytest.fixture(scope="module")
 def telegraf_agent_config(request):
     """
     Reads the service configuration deployed for the streaming simulation test.
@@ -13,8 +14,9 @@ def telegraf_agent_config(request):
     :param request: access the parameters of the fixture
     :return: the python object representing the read YAML file
     """
-
-    with open(request.param['config']['rspec'], 'r') as stream:
+    rspec = pkg_resources.resource_filename('clmctest.inputs', 'rspec.yml')
+    print("rspec file: {0}".format(rspec))
+    with open(rspec, 'r') as stream:
         data_loaded = yaml.load(stream)
     return data_loaded
 

test/telegraf-agents/rspec.yml → clmctest/inputs/rspec.yml

@@ -10,7 +10,7 @@ hosts:
         host: 8888
       - guest: 9092
         host: 9092
-    ip_address: "192.168.50.10"
+    ip_address: "203.0.113.100"
   - name: apache
     cpus: 1
     memory: 2048
@@ -19,14 +19,14 @@ hosts:
     forward_ports:
       - guest: 80
         host: 8881
-    ip_address: "192.168.50.11"
+    ip_address: "203.0.113.101"
     location: "DC1"
     sfc_id: "MS_Template_1"
     sfc_id_instance: "MS_I1"
     sf_id: "adaptive_streaming"
     sf_id_instance: "adaptive_streaming_I1"
     ipendpoint_id: "adaptive_streaming_I1_apache1"
-    influxdb_url: "http://192.168.50.10:8086"
+    influxdb_url: "http://203.0.113.100:8086"
     database_name: "CLMCMetrics"
   - name: nginx
     cpus: 1
@@ -36,14 +36,14 @@ hosts:
     forward_ports:
       - guest: 80
         host: 8082
-    ip_address: "192.168.50.13"
+    ip_address: "203.0.113.102"
     location: "DC1"
     sfc_id: "MS_Template_1"
     sfc_id_instance: "MS_I1"
     sf_id: "adaptive_streaming"
     sf_id_instance: "adaptive_streaming_nginx_I1"
     ipendpoint_id: "adaptive_streaming_nginx_I1_apache1"
-    influxdb_url: "http://192.168.50.10:8086"
+    influxdb_url: "http://203.0.113.100:8086"
     database_name: "CLMCMetrics"
   - name: mongo
     cpus: 1
@@ -53,14 +53,14 @@ hosts:
     forward_ports:
       - guest: 80
         host: 8083
-    ip_address: "192.168.50.14"
+    ip_address: "203.0.113.103"
     location: "DC1"
     sfc_id: "MS_Template_1"
     sfc_id_instance: "MS_I1"
     sf_id: "metadata_database"
     sf_id_instance: "metadata_database_I1"
     ipendpoint_id: "metadata_database_I1_apache1"
-    influxdb_url: "http://192.168.50.10:8086"
+    influxdb_url: "http://203.0.113.100:8086"
     database_name: "CLMCMetrics" 
   - name: ffmpeg
     cpus: 1
@@ -70,14 +70,14 @@ hosts:
     forward_ports:
       - guest: 80
         host: 8084
-    ip_address: "192.168.50.15"
+    ip_address: "203.0.113.104"
     location: "DC1"
     sfc_id: "MS_Template_1"
     sfc_id_instance: "MS_I1"
     sf_id: "metadata_database"
     sf_id_instance: "metadata_database_I1"
     ipendpoint_id: "metadata_database_I1_apache1"
-    influxdb_url: "http://192.168.50.10:8086"
+    influxdb_url: "http://203.0.113.100:8086"
     database_name: "CLMCMetrics" 
   - name: host
     cpus: 1
@@ -87,17 +87,17 @@ hosts:
     forward_ports:
       - guest: 80
         host: 8085
-    ip_address: "192.168.50.16"
+    ip_address: "203.0.113.105"
     location: "DC1"
     sfc_id: "MS_Template_1"
     sfc_id_instance: "MS_I1"
     sf_id: "adaptive_streaming"
     sf_id_instance: "adaptive_streaming_I1"
     ipendpoint_id: "adaptive_streaming_I1_apache1"
-    influxdb_url: "http://192.168.50.10:8086"
+    influxdb_url: "http://203.0.113.100:8086"
     database_name: "CLMCMetrics"
   - name: test-runner
     cpus: 1
     memory: 2048
     disk: "10GB"
-    ip_address: "192.168.50.17"    
+    ip_address: "203.0.113.150"    
\ No newline at end of file

test/streaming-sim/StreamingSim.py → clmctest/monitoring/StreamingSim.py

 #!/usr/bin/python3
 
-import LineProtocolGenerator as lp
+import clmctest.monitoring.LineProtocolGenerator as lp
 import time
 import urllib.parse
 import pytest

test/streaming-sim/conftest.py → clmctest/monitoring/conftest.py

@@ -2,19 +2,22 @@
 
 import pytest
 import yaml
+import pkg_resources
 from influxdb import InfluxDBClient
 
 
-@pytest.fixture(scope="module", params=[{'config': {'rspec': 'test/streaming-sim/rspec.yml'}}])
-def streaming_sim_config(request):
+@pytest.fixture(scope="module")
+def streaming_sim_config():
     """
     Reads the service configuration deployed for the streaming simulation test.
 
     :param request: access the parameters of the fixture
     :return: the python object representing the read YAML file
     """
+    rspec = pkg_resources.resource_filename('clmctest.monitoring', 'rspec.yml')
+    print("rspec file: {0}".format(rspec))
 
-    with open(request.param['config']['rspec'], 'r') as stream:
+    with open(rspec, 'r') as stream:
         data_loaded = yaml.load(stream)
     return data_loaded
 
@@ -30,3 +33,4 @@ def get_db_client(streaming_sim_config, request):
     """
 
     return InfluxDBClient(host=streaming_sim_config['hosts'][0]['ip_address'], port=8086, database=request.param['database'], timeout=10)
+

test/streaming-sim/rspec.yml → clmctest/monitoring/rspec.yml

@@ -19,7 +19,7 @@ hosts:
     forward_ports:
       - guest: 80
         host: 8081
-    ip_address: "203.0.113.101"
+    ip_address: "203.0.113.140"
     location: "DC1"
     sfc_id: "MS_Template_1"
     sfc_id_instance: "MS_I1"
@@ -36,7 +36,7 @@ hosts:
     forward_ports:
       - guest: 80
         host: 8082
-    ip_address: "203.0.113.102"
+    ip_address: "203.0.113.141"
     location: "DC2"
     sfc_id: "MS_Template_1"
     sfc_id_instance: "MS_I1"
@@ -49,4 +49,4 @@ hosts:
     cpus: 1
     memory: 2048
     disk: "10GB"
-    ip_address: "203.0.113.102"       
+    ip_address: "203.0.113.150"       
\ No newline at end of file

test/streaming-sim/test_simresults.py → clmctest/monitoring/test_simresults.py

 #!/usr/bin/python3
 
 import pytest
-from StreamingSim import run_simulation_fixture
+from clmctest.monitoring.StreamingSim import run_simulation_fixture
 
 
 class TestSimulation(object):

clmctest/scripts/.pytest_cache/v/cache/lastfailed

+{
+  "test_config_telegraf.py::test_write_telegraf_conf": true
+}
\ No newline at end of file