Merge branch 'integration' into 'master'

Integration

See merge request FLAME/consortium/3rdparties/flame-clmc!70

.gitlab-ci.yml

@@ -35,8 +35,8 @@ build:tests:
     - python setup.py sdist --dist-dir=$CI_PROJECT_DIR/build
   artifacts:
     paths:
-    - build/clmctest-2.1.1.tar.gz
-    - build/clmcservice-2.1.1.tar.gz
+    - build/clmctest-2.1.2.tar.gz
+    - build/clmcservice-2.1.2.tar.gz
     expire_in: 1 day
 
 test:all:
@@ -50,8 +50,8 @@ test:all:
     - echo "REPO_PASS=${REPO_PASS}" >> $CI_PROJECT_DIR/reporc
     - sudo scripts/test/fixture.sh create -f src/test/clmctest/rspec.json -r $CI_PROJECT_DIR -c all
     - sudo mkdir /var/lib/lxd/containers/test-runner/rootfs/opt/clmc/build
-    - sudo cp build/clmctest-2.1.1.tar.gz /var/lib/lxd/containers/test-runner/rootfs/opt/clmc/build
-    - sudo lxc exec test-runner -- pip3 install /opt/clmc/build/clmctest-2.1.1.tar.gz
+    - sudo cp build/clmctest-2.1.2.tar.gz /var/lib/lxd/containers/test-runner/rootfs/opt/clmc/build
+    - sudo lxc exec test-runner -- pip3 install /opt/clmc/build/clmctest-2.1.2.tar.gz
     - sudo lxc exec test-runner -- pytest -s --tb=short -rfp --pyargs clmctest
   when: on_success      
   

docs/AlertsSpecification.md

@@ -240,7 +240,8 @@ This includes any of the following: `"flame_sfp", "flame_sf", "flame_server", "f
 Keep in mind that **flame_sfc** and **flame_sfci** are also part of the CLMC Information Model. However, filtering on 
 these tags is automatically generated and added to all InfluxDB queries by using the metadata values from the 
 alerts specification. Therefore, including **flame_sfc** and **flame_sfci** in the **resource_type** is considered INVALID.  
-For more information on the global tags, please check the [documentation](monitoring.md).  
+For more information on the global tags, please check the [documentation](monitoring.md). In addition, this could include
+custom tags which are part of a specific measurement.
 
 * **comparison_operator** - the logical operator to use for comparison - lt (less than), gt (greater than), lte (less than or equal to), etc.
 

scripts/clmc-service/install-clmc-service.sh

@@ -66,43 +66,39 @@ fi
 # switch the CLMC environment - and check
 echo "----> Switching to use CLMC python environment"
 workon CLMC
-if [ $? -ne 0 ] ; then
+if [[ $? -ne 0 ]] ; then
         echo "Failed: switching to CLMC python environment"
 		exit 1
 fi
 
-# install tox - and check
-echo "----> Installing TOX"
-pip3 install tox
-tox --version
-if [ $? -ne 0 ] ; then
-        echo "Failed: installing tox"
-		exit 1
-fi
-
-# navigate to the clmc-webservice - and check
+# navigate to the clmc-service folder and check
 echo "----> Moving to CLMC service"
 cd ${REPO_ROOT}/src/service
-if [ $? -ne 0 ] ; then
+if [[ $? -ne 0 ]] ; then
         echo "Failed: could not find clmc-service"
 		exit 1
 fi
 
-# running tests using tox
-echo "----> Running tox"
-TOX_OUTPUT="$(tox)"
-# check if tox output contains the 'congratulations :)' bit for tests passed
-if [[ $TOX_OUTPUT != *"congratulations :)"* ]]; then
-  echo $TOX_OUTPUT
-  echo "CLMC service unit tests failed."
-  exit 1
+echo "----> Replicating CLMC Python environment"
+pip3 install -r requirements.txt
+if [[ $? -ne 0 ]] ; then
+        echo "Failed: could not replicate CLMC Python environment from requirements.txt"
+		exit 1
+fi
+
+# running tests using pytest
+echo "----> Running tests"
+pytest -v clmcservice
+if [[ $? -ne 0 ]] ; then
+        echo "Failed: CLMC service unit tests have failed."
+		exit 1
 fi
-echo "----> Tox execution of unit tests passed successfully"
+echo "----> Execution of unit tests passed successfully"
 
 # install the service
 echo "----> Installing CLMC web service"
 pip3 install  .
-if [ $? -ne 0 ] ; then
+if [[ $? -ne 0 ]] ; then
         echo "Failed: installing clmc-webservice"
 		exit 1
 fi
@@ -114,7 +110,7 @@ mkdir -p /var/log/flame/clmc
 # initialise the CLMC service database with the model tables
 echo "----> Initialising CLMC database"
 initialize_clmcservice_db production.ini
-if [ $? -ne 0 ] ; then
+if [[ $? -ne 0 ]] ; then
         echo "Failed: initialising CLMC database"
 		exit 1
 fi

src/service/VERSION

-__version__ = "2.1.1"
\ No newline at end of file
+__version__ = "2.1.2"
\ No newline at end of file

src/service/clmcservice/alertsapi/alerts_specification_schema.py

@@ -64,8 +64,10 @@ URL_REGEX = re.compile(
     r'(?:[/?#][^\s]*)?$',  # URL path or query parameters
     re.IGNORECASE)
 
-# Global tags allowed to be used for filtering in the trigger condition
-CLMC_INFORMATION_MODEL_GLOBAL_TAGS = {"flame_sfp", "flame_sf", "flame_server", "flame_location"}
+# Global tags allowed to be used for filtering in the trigger condition - removed the restriction, so that all tags could be used for filtering (not just global)
+# CLMC_INFORMATION_MODEL_GLOBAL_TAGS = {"flame_sfp", "flame_sf", "flame_server", "flame_location"}
+
+
 # NOTICE that "flame_sfc", "flame_sfci" are not allowed, even though they are part of the CLMC Information Model
 # This is because those two tags are automatically added to the InfluxDB queries - the required values are retrieved from the alert spec. metadata
 # "flame_sfe" cannot be used as well, because the value of this tag is only known at runtime.
@@ -95,7 +97,7 @@ ALERTS_SPECIFICATION_SCHEMA = Schema({
                                 "granularity": And(int, lambda p: p > 0),
                                 Optional("aggregation_method"): And(str, lambda s: s in INFLUX_QL_FUNCTIONS),
                                 Optional("resource_type"): {
-                                    And(str, lambda s: s in CLMC_INFORMATION_MODEL_GLOBAL_TAGS): str
+                                    str: str
                                 },
                                 Optional("comparison_operator"): And(str, lambda s: s in COMPARISON_OPERATORS)
                             },

src/service/clmcservice/alertsapi/views.py

@@ -409,7 +409,7 @@ class AlertsConfigurationAPI(object):
                 response = post(kapacitor_api_tasks_url, json=kapacitor_http_request_body)
                 response_content = response.json()
                 # log the response
-                log.info(response_content, response.status_code)
+                log.info("{0} - {1}".format(response_content, response.status_code))
 
                 # track all reported errors
                 if response_content.get("error", "") != "":  # Kapacitor always returns an error key which is set to an empty string if successful
@@ -475,7 +475,7 @@ class AlertsConfigurationAPI(object):
             kapacitor_http_request_body = fill_http_post_handler_vars(handler_id, http_handler_url)
             response = post(kapacitor_api_handlers_url, json=kapacitor_http_request_body)
             response_content = response.json()
-            log.info(response_content, response.status_code)
+            log.info("{0} - {1}".format(response_content, response.status_code))
 
             if response_content.get("error", "") != "":
                 capture_error = True
@@ -535,8 +535,9 @@ class AlertsConfigurationAPI(object):
             raise HTTPBadRequest("Request alert specification file could not be parsed as a valid TOSCA document.")
 
         # validate against CLMC spec.
-        valid_alert_spec = validate_clmc_alerts_specification(tosca_tpl.tpl)
+        valid_alert_spec, error = validate_clmc_alerts_specification(tosca_tpl.tpl, include_error=True)
         if not valid_alert_spec:
+            log.error("Error encountered while validating alerts specification against CLMC alerts specification schema - {0}".format(error))
             raise HTTPBadRequest("Request alert specification file could not be validated as a CLMC TOSCA alerts specification document.")
 
         return tosca_tpl

src/service/requirements.txt

+PasteDeploy==2.0.1
+schema==0.6.8
+psycopg2==2.7.5
+psutil==5.6.1
+colorama==0.4.1
+pbr==5.2.0
+pluggy==0.11.0
+prettytable==0.7.2
+neobolt==1.7.12
+more-itertools==7.0.0
+py==1.8.0
+certifi==2019.3.9
+attrs==19.1.0
+WebOb==1.8.5
+atomicwrites==1.3.0
+waitress==1.1.0
+Click==7.0
+pycparser==2.19
+PyYAML==3.13
+asn1crypto==0.24.0
+SQLAlchemy==1.2.12
+urllib3==1.24.3
+ipaddress==1.0.22
+six==1.12.0
+wcwidth==0.1.7
+pyperclip==1.7.0
+hupper==1.6.1
+chardet==3.0.4
+toml==0.10.0
+pytz==2019.1
+filelock==3.0.10
+MarkupSafe==1.1.1
+translationstring==1.3
+Pygments==2.3.1
+idna==2.8
+pyparsing==2.4.0
+repoze.lru==0.7
+venusian==1.2.0
+zope.deprecation==4.4.0
+stevedore==1.30.1
+Mako==1.0.9
+cffi==1.12.3
+zope.interface==4.6.0
+neotime==1.7.4
+Babel==2.6.0
+python-dateutil==2.8.0
+plaster==1.0
+cmd2==0.9.12
+requests==2.21.0
+pathlib2==2.3.3
+prompt-toolkit==2.0.9
+pytest==3.8.1
+cryptography==2.6.1
+transaction==2.4.0
+cliff==2.14.1
+plaster-pastedeploy==0.6
+influxdb==5.2.0
+py2neo==4.2.0
+zope.sqlalchemy==1.0
+pyOpenSSL==19.0.0
+tosca-parser==1.1.0
+pyramid==1.9.2
+pyramid-mako==1.0.2
+pyramid-exclog==1.0
+pyramid-debugtoolbar==4.5
\ No newline at end of file

src/service/resources/tosca/test-data/clmc-validator/invalid/alerts_test_config-11.yaml

@@ -7,6 +7,7 @@ imports:
 
 metadata:
   servicefunctionchain: companyA-VR
+  sfc: companyA-VR  # correct metadata field is servicefunctionchain
 #  sfci: companyA-VR-premium
 
 topology_template:
@@ -24,7 +25,6 @@ topology_template:
               granularity: 120
               aggregation_method: mean
               resource_type:
-                flame_sfc: companyA-VR  # not allowed - sfc tag is automatically added in the filters
                 flame_location: watershed
               comparison_operator: gt
             action:

src/service/resources/tosca/test-data/clmc-validator/invalid/alerts_test_config-12.yaml

@@ -7,7 +7,7 @@ imports:
 
 metadata:
   servicefunctionchain: companyA-VR
-#  sfci: companyA-VR-premium
+  sfci: companyA-VR-premium   # not allowed - sfc instance tag is automatically added in the filters
 
 topology_template:
 
@@ -24,7 +24,6 @@ topology_template:
               granularity: 120
               aggregation_method: mean
               resource_type:
-                flame_sfci: companyA-VR-premium  # not allowed - sfci tag is automatically added in the filters
                 flame_location: watershed
               comparison_operator: gt
             action:

src/service/resources/tosca/test-data/clmc-validator/invalid/alerts_test_config-3.yaml

@@ -26,7 +26,7 @@ topology_template:
               granularity: 60
               aggregation_method: last
               resource_type:
-                flame_sf_package_id: storage # sf_package_id is not the correct tag name, it is sfp
+                status_code: 200 # integers must be quoted as strings when used as tag values, e.g. '200'
                 flame_sf: storage-users
                 flame_location: watershed
               comparison_operator: lt

src/service/resources/tosca/test-data/clmc-validator/valid/alerts_test_config-2.yaml

@@ -41,6 +41,7 @@ topology_template:
               threshold: -100  # requests have decreased by at least 100
               granularity: 120
               resource_type:
+                status_code: '200'
                 flame_sfp: storage
                 flame_sf: storage-users
                 flame_location: watershed

src/service/setup.py

@@ -52,6 +52,7 @@ def get_version(*relative_path):
 
 
 requires = [
+    'py2neo==4.2.0',
     'plaster_pastedeploy==0.6',
     'pyramid==1.9.2',
     'pyramid_debugtoolbar==4.5',
@@ -61,7 +62,6 @@ requires = [
     'zope.sqlalchemy==1.0',
     'psycopg2==2.7.5',
     'influxdb==5.2.0',
-    'py2neo==4.2.0',
     'pyyaml==3.13',
     'tosca-parser==1.1.0',
     'schema==0.6.8',

src/service/tox.ini

-[tox]
-envlist = py35
-[testenv]
-deps=pytest
-commands=pytest
\ No newline at end of file

src/test/VERSION

-__version__ = "2.1.1"
\ No newline at end of file
+__version__ = "2.1.2"
\ No newline at end of file

src/test/clmctest/alerts/test_alerts.py

@@ -32,6 +32,7 @@ from clmctest.alerts.alert_handler_server import LOG_TEST_FOLDER_PATH
 
 
 NGINX_PORT = 80
+SFEMC = "flame_sfemc"
 
 
 def is_valid_timestamp(str_timestamp):
@@ -147,19 +148,19 @@ class TestAlerts(object):
              "topic_api_endpoint": "/kapacitor/v1/alerts/topics/46fb8800c8a5eeeb04b090d838d475df574a2e6d854b5d678fc981c096eb6c1b",
              "topic_handlers_api_endpoint": "/kapacitor/v1/alerts/topics/46fb8800c8a5eeeb04b090d838d475df574a2e6d854b5d678fc981c096eb6c1b/handlers"},
             {"policy": "scale_nginx_policy", "trigger": "increase_in_active_requests", "task_identifier": "7a9867f9270dba6635ac3760a3b70bc929f5bd0f3bf582e45d27fbd437f528ca",
-             "handlers": ["flame_sfemc", "http://172.40.231.200:9999/"],
+             "handlers": [SFEMC, "http://172.40.231.200:9999/"],
              "topic_identifier": "7a9867f9270dba6635ac3760a3b70bc929f5bd0f3bf582e45d27fbd437f528ca",
              "task_api_endpoint": "/kapacitor/v1/tasks/7a9867f9270dba6635ac3760a3b70bc929f5bd0f3bf582e45d27fbd437f528ca",
              "topic_api_endpoint": "/kapacitor/v1/alerts/topics/7a9867f9270dba6635ac3760a3b70bc929f5bd0f3bf582e45d27fbd437f528ca",
              "topic_handlers_api_endpoint": "/kapacitor/v1/alerts/topics/7a9867f9270dba6635ac3760a3b70bc929f5bd0f3bf582e45d27fbd437f528ca/handlers"},
             {"policy": "scale_nginx_policy", "trigger": "increase_in_running_processes", "task_identifier": "f5edaeb27fb847116be749c3815d240cbf0d7ba79aee1959daf0b3445a70f2c8",
-             "handlers": ["flame_sfemc", "http://172.40.231.200:9999/"],
+             "handlers": [SFEMC, "http://172.40.231.200:9999/"],
              "topic_identifier": "f5edaeb27fb847116be749c3815d240cbf0d7ba79aee1959daf0b3445a70f2c8",
              "task_api_endpoint": "/kapacitor/v1/tasks/f5edaeb27fb847116be749c3815d240cbf0d7ba79aee1959daf0b3445a70f2c8",
              "topic_api_endpoint": "/kapacitor/v1/alerts/topics/f5edaeb27fb847116be749c3815d240cbf0d7ba79aee1959daf0b3445a70f2c8",
              "topic_handlers_api_endpoint": "/kapacitor/v1/alerts/topics/f5edaeb27fb847116be749c3815d240cbf0d7ba79aee1959daf0b3445a70f2c8/handlers"},
             {"policy": "deadman_policy", "trigger": "no_measurements", "task_identifier": "f7dab6fd53001c812d44533d3bbb6ef45f0d1d39b9441bc3c60402ebda85d320",
-             "handlers": ["flame_sfemc", "http://172.40.231.200:9999/"],
+             "handlers": [SFEMC, "http://172.40.231.200:9999/"],
              "topic_identifier": "f7dab6fd53001c812d44533d3bbb6ef45f0d1d39b9441bc3c60402ebda85d320",
              "task_api_endpoint": "/kapacitor/v1/tasks/f7dab6fd53001c812d44533d3bbb6ef45f0d1d39b9441bc3c60402ebda85d320",
              "topic_api_endpoint": "/kapacitor/v1/alerts/topics/f7dab6fd53001c812d44533d3bbb6ef45f0d1d39b9441bc3c60402ebda85d320",
@@ -210,9 +211,9 @@ class TestAlerts(object):
                                                                                        {"policy": "scale_nginx_policy", "trigger": "increase_in_running_processes"}, {"policy": "deadman_policy", "trigger": "no_measurements"}], \
             "Incorrect list of deleted alerts"
         # sort by handler and trigger to ensure comparison order is correct
-        assert sorted(json_response["deleted_handlers"], key=lambda x: (x['handler'], x['trigger'])) == [{"policy": "scale_nginx_policy", "trigger": "increase_in_active_requests", "handler": "flame_sfemc"},
-                                                                                                         {"policy": "scale_nginx_policy", "trigger": "increase_in_running_processes", "handler": "flame_sfemc"},
-                                                                                                         {"policy": "deadman_policy", "trigger": "no_measurements", "handler": "flame_sfemc"},
+        assert sorted(json_response["deleted_handlers"], key=lambda x: (x['handler'], x['trigger'])) == [{"policy": "scale_nginx_policy", "trigger": "increase_in_active_requests", "handler": SFEMC},
+                                                                                                         {"policy": "scale_nginx_policy", "trigger": "increase_in_running_processes", "handler": SFEMC},
+                                                                                                         {"policy": "deadman_policy", "trigger": "no_measurements", "handler": SFEMC},
                                                                                                          {"policy": "scale_nginx_policy", "trigger": "high_requests", "handler": "http://172.40.231.200:9999/"},
                                                                                                          {"policy": "scale_nginx_policy", "trigger": "increase_in_active_requests", "handler": "http://172.40.231.200:9999/"},
                                                                                                          {"policy": "scale_nginx_policy", "trigger": "increase_in_running_processes", "handler": "http://172.40.231.200:9999/"},