diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 13687d939fd18e771cc990d5f4c965e193b41fb7..42cf1e0faead68b693989dd39cbd470d1d414938 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -35,8 +35,8 @@ build:tests:
- python setup.py sdist --dist-dir=$CI_PROJECT_DIR/build
artifacts:
paths:
- - build/clmctest-2.1.1.tar.gz
- - build/clmcservice-2.1.1.tar.gz
+ - build/clmctest-2.1.2.tar.gz
+ - build/clmcservice-2.1.2.tar.gz
expire_in: 1 day
test:all:
@@ -50,8 +50,8 @@ test:all:
- echo "REPO_PASS=${REPO_PASS}" >> $CI_PROJECT_DIR/reporc
- sudo scripts/test/fixture.sh create -f src/test/clmctest/rspec.json -r $CI_PROJECT_DIR -c all
- sudo mkdir /var/lib/lxd/containers/test-runner/rootfs/opt/clmc/build
- - sudo cp build/clmctest-2.1.1.tar.gz /var/lib/lxd/containers/test-runner/rootfs/opt/clmc/build
- - sudo lxc exec test-runner -- pip3 install /opt/clmc/build/clmctest-2.1.1.tar.gz
+ - sudo cp build/clmctest-2.1.2.tar.gz /var/lib/lxd/containers/test-runner/rootfs/opt/clmc/build
+ - sudo lxc exec test-runner -- pip3 install /opt/clmc/build/clmctest-2.1.2.tar.gz
- sudo lxc exec test-runner -- pytest -s --tb=short -rfp --pyargs clmctest
when: on_success
diff --git a/docs/AlertsSpecification.md b/docs/AlertsSpecification.md
index 5201432f1ba42f1322c39dc90f5cfbbcea8f06c6..e9c432fe89a86278b3ffc96087fb8c7a9e26c4ab 100644
--- a/docs/AlertsSpecification.md
+++ b/docs/AlertsSpecification.md
@@ -240,7 +240,8 @@ This includes any of the following: `"flame_sfp", "flame_sf", "flame_server", "f
Keep in mind that **flame_sfc** and **flame_sfci** are also part of the CLMC Information Model. However, filtering on
these tags is automatically generated and added to all InfluxDB queries by using the metadata values from the
alerts specification. Therefore, including **flame_sfc** and **flame_sfci** in the **resource_type** is considered INVALID.
-For more information on the global tags, please check the [documentation](monitoring.md).
+For more information on the global tags, please check the [documentation](monitoring.md). In addition, this could include
+custom tags which are part of a specific measurement.
* **comparison_operator** - the logical operator to use for comparison - lt (less than), gt (greater than), lte (less than or equal to), etc.
diff --git a/scripts/clmc-service/install-clmc-service.sh b/scripts/clmc-service/install-clmc-service.sh
index 27f60ebc1320272526b62575f0cafe28d62ab427..6b3afbe75b9ee2d44b1eb7bfc09001780be278f4 100755
--- a/scripts/clmc-service/install-clmc-service.sh
+++ b/scripts/clmc-service/install-clmc-service.sh
@@ -66,43 +66,39 @@ fi
# switch the CLMC environment - and check
echo "----> Switching to use CLMC python environment"
workon CLMC
-if [ $? -ne 0 ] ; then
+if [[ $? -ne 0 ]] ; then
echo "Failed: switching to CLMC python environment"
exit 1
fi
-# install tox - and check
-echo "----> Installing TOX"
-pip3 install tox
-tox --version
-if [ $? -ne 0 ] ; then
- echo "Failed: installing tox"
- exit 1
-fi
-
-# navigate to the clmc-webservice - and check
+# navigate to the clmc-service folder and check
echo "----> Moving to CLMC service"
cd ${REPO_ROOT}/src/service
-if [ $? -ne 0 ] ; then
+if [[ $? -ne 0 ]] ; then
echo "Failed: could not find clmc-service"
exit 1
fi
-# running tests using tox
-echo "----> Running tox"
-TOX_OUTPUT="$(tox)"
-# check if tox output contains the 'congratulations :)' bit for tests passed
-if [[ $TOX_OUTPUT != *"congratulations :)"* ]]; then
- echo $TOX_OUTPUT
- echo "CLMC service unit tests failed."
- exit 1
+echo "----> Replicating CLMC Python environment"
+pip3 install -r requirements.txt
+if [[ $? -ne 0 ]] ; then
+ echo "Failed: could not replicate CLMC Python environment from requirements.txt"
+ exit 1
+fi
+
+# running tests using pytest
+echo "----> Running tests"
+pytest -v clmcservice
+if [[ $? -ne 0 ]] ; then
+ echo "Failed: CLMC service unit tests have failed."
+ exit 1
fi
-echo "----> Tox execution of unit tests passed successfully"
+echo "----> Execution of unit tests passed successfully"
# install the service
echo "----> Installing CLMC web service"
pip3 install .
-if [ $? -ne 0 ] ; then
+if [[ $? -ne 0 ]] ; then
echo "Failed: installing clmc-webservice"
exit 1
fi
@@ -114,7 +110,7 @@ mkdir -p /var/log/flame/clmc
# initialise the CLMC service database with the model tables
echo "----> Initialising CLMC database"
initialize_clmcservice_db production.ini
-if [ $? -ne 0 ] ; then
+if [[ $? -ne 0 ]] ; then
echo "Failed: initialising CLMC database"
exit 1
fi
diff --git a/src/service/VERSION b/src/service/VERSION
index b842790079f79338cf4a5f7de3efcabd6f25b0b5..b62a3e51a74a810ec8e8b760cce66cdbfac3c1d6 100644
--- a/src/service/VERSION
+++ b/src/service/VERSION
@@ -1 +1 @@
-__version__ = "2.1.1"
\ No newline at end of file
+__version__ = "2.1.2"
\ No newline at end of file
diff --git a/src/service/clmcservice/alertsapi/alerts_specification_schema.py b/src/service/clmcservice/alertsapi/alerts_specification_schema.py
index 022c2a38f3ff62b7cf021ce672333c8c3e230950..7d15056f5e3183536a765b2c67df52ba82ac024a 100644
--- a/src/service/clmcservice/alertsapi/alerts_specification_schema.py
+++ b/src/service/clmcservice/alertsapi/alerts_specification_schema.py
@@ -64,8 +64,10 @@ URL_REGEX = re.compile(
r'(?:[/?#][^\s]*)?$', # URL path or query parameters
re.IGNORECASE)
-# Global tags allowed to be used for filtering in the trigger condition
-CLMC_INFORMATION_MODEL_GLOBAL_TAGS = {"flame_sfp", "flame_sf", "flame_server", "flame_location"}
+# Global tags allowed to be used for filtering in the trigger condition - removed the restriction, so that all tags could be used for filtering (not just global)
+# CLMC_INFORMATION_MODEL_GLOBAL_TAGS = {"flame_sfp", "flame_sf", "flame_server", "flame_location"}
+
+
# NOTICE that "flame_sfc", "flame_sfci" are not allowed, even though they are part of the CLMC Information Model
# This is because those two tags are automatically added to the InfluxDB queries - the required values are retrieved from the alert spec. metadata
# "flame_sfe" cannot be used as well, because the value of this tag is only known at runtime.
@@ -95,7 +97,7 @@ ALERTS_SPECIFICATION_SCHEMA = Schema({
"granularity": And(int, lambda p: p > 0),
Optional("aggregation_method"): And(str, lambda s: s in INFLUX_QL_FUNCTIONS),
Optional("resource_type"): {
- And(str, lambda s: s in CLMC_INFORMATION_MODEL_GLOBAL_TAGS): str
+ str: str
},
Optional("comparison_operator"): And(str, lambda s: s in COMPARISON_OPERATORS)
},
diff --git a/src/service/clmcservice/alertsapi/views.py b/src/service/clmcservice/alertsapi/views.py
index 729c55d014c129eeb52c61aeb89145e8e6119568..8b39d09a507ce2f9dc5e4277d8d0faaa96572cfb 100644
--- a/src/service/clmcservice/alertsapi/views.py
+++ b/src/service/clmcservice/alertsapi/views.py
@@ -409,7 +409,7 @@ class AlertsConfigurationAPI(object):
response = post(kapacitor_api_tasks_url, json=kapacitor_http_request_body)
response_content = response.json()
# log the response
- log.info(response_content, response.status_code)
+ log.info("{0} - {1}".format(response_content, response.status_code))
# track all reported errors
if response_content.get("error", "") != "": # Kapacitor always returns an error key which is set to an empty string if successful
@@ -475,7 +475,7 @@ class AlertsConfigurationAPI(object):
kapacitor_http_request_body = fill_http_post_handler_vars(handler_id, http_handler_url)
response = post(kapacitor_api_handlers_url, json=kapacitor_http_request_body)
response_content = response.json()
- log.info(response_content, response.status_code)
+ log.info("{0} - {1}".format(response_content, response.status_code))
if response_content.get("error", "") != "":
capture_error = True
@@ -535,8 +535,9 @@ class AlertsConfigurationAPI(object):
raise HTTPBadRequest("Request alert specification file could not be parsed as a valid TOSCA document.")
# validate against CLMC spec.
- valid_alert_spec = validate_clmc_alerts_specification(tosca_tpl.tpl)
+ valid_alert_spec, error = validate_clmc_alerts_specification(tosca_tpl.tpl, include_error=True)
if not valid_alert_spec:
+ log.error("Error encountered while validating alerts specification against CLMC alerts specification schema - {0}".format(error))
raise HTTPBadRequest("Request alert specification file could not be validated as a CLMC TOSCA alerts specification document.")
return tosca_tpl
diff --git a/src/service/requirements.txt b/src/service/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..9cfd29044267440e933889e83ee07608de46ad51
--- /dev/null
+++ b/src/service/requirements.txt
@@ -0,0 +1,65 @@
+PasteDeploy==2.0.1
+schema==0.6.8
+psycopg2==2.7.5
+psutil==5.6.1
+colorama==0.4.1
+pbr==5.2.0
+pluggy==0.11.0
+prettytable==0.7.2
+neobolt==1.7.12
+more-itertools==7.0.0
+py==1.8.0
+certifi==2019.3.9
+attrs==19.1.0
+WebOb==1.8.5
+atomicwrites==1.3.0
+waitress==1.1.0
+Click==7.0
+pycparser==2.19
+PyYAML==3.13
+asn1crypto==0.24.0
+SQLAlchemy==1.2.12
+urllib3==1.24.3
+ipaddress==1.0.22
+six==1.12.0
+wcwidth==0.1.7
+pyperclip==1.7.0
+hupper==1.6.1
+chardet==3.0.4
+toml==0.10.0
+pytz==2019.1
+filelock==3.0.10
+MarkupSafe==1.1.1
+translationstring==1.3
+Pygments==2.3.1
+idna==2.8
+pyparsing==2.4.0
+repoze.lru==0.7
+venusian==1.2.0
+zope.deprecation==4.4.0
+stevedore==1.30.1
+Mako==1.0.9
+cffi==1.12.3
+zope.interface==4.6.0
+neotime==1.7.4
+Babel==2.6.0
+python-dateutil==2.8.0
+plaster==1.0
+cmd2==0.9.12
+requests==2.21.0
+pathlib2==2.3.3
+prompt-toolkit==2.0.9
+pytest==3.8.1
+cryptography==2.6.1
+transaction==2.4.0
+cliff==2.14.1
+plaster-pastedeploy==0.6
+influxdb==5.2.0
+py2neo==4.2.0
+zope.sqlalchemy==1.0
+pyOpenSSL==19.0.0
+tosca-parser==1.1.0
+pyramid==1.9.2
+pyramid-mako==1.0.2
+pyramid-exclog==1.0
+pyramid-debugtoolbar==4.5
\ No newline at end of file
diff --git a/src/service/resources/tosca/test-data/clmc-validator/invalid/alerts_test_config-11.yaml b/src/service/resources/tosca/test-data/clmc-validator/invalid/alerts_test_config-11.yaml
index f18b60b6aef9f53d8afb0ef5ff4d251ba2425279..59bd2f6757e2c8048617474b705e8feee2631bd4 100644
--- a/src/service/resources/tosca/test-data/clmc-validator/invalid/alerts_test_config-11.yaml
+++ b/src/service/resources/tosca/test-data/clmc-validator/invalid/alerts_test_config-11.yaml
@@ -7,6 +7,7 @@ imports:
metadata:
servicefunctionchain: companyA-VR
+ sfc: companyA-VR # correct metadata field is servicefunctionchain
# sfci: companyA-VR-premium
topology_template:
@@ -24,7 +25,6 @@ topology_template:
granularity: 120
aggregation_method: mean
resource_type:
- flame_sfc: companyA-VR # not allowed - sfc tag is automatically added in the filters
flame_location: watershed
comparison_operator: gt
action:
diff --git a/src/service/resources/tosca/test-data/clmc-validator/invalid/alerts_test_config-12.yaml b/src/service/resources/tosca/test-data/clmc-validator/invalid/alerts_test_config-12.yaml
index 8e78967d3555cf84996d473e689f45b98869dd05..0c2a659f528a0da4f51700e8e7c7968575d65366 100644
--- a/src/service/resources/tosca/test-data/clmc-validator/invalid/alerts_test_config-12.yaml
+++ b/src/service/resources/tosca/test-data/clmc-validator/invalid/alerts_test_config-12.yaml
@@ -7,7 +7,7 @@ imports:
metadata:
servicefunctionchain: companyA-VR
-# sfci: companyA-VR-premium
+ sfci: companyA-VR-premium # not allowed - sfc instance tag is automatically added in the filters
topology_template:
@@ -24,7 +24,6 @@ topology_template:
granularity: 120
aggregation_method: mean
resource_type:
- flame_sfci: companyA-VR-premium # not allowed - sfci tag is automatically added in the filters
flame_location: watershed
comparison_operator: gt
action:
diff --git a/src/service/resources/tosca/test-data/clmc-validator/invalid/alerts_test_config-3.yaml b/src/service/resources/tosca/test-data/clmc-validator/invalid/alerts_test_config-3.yaml
index 8b9aac9bfd2e4019f5d23fd04ab511b45ffe20b2..24890361628a94e1a86e9a23c96fc52a57fdbe62 100644
--- a/src/service/resources/tosca/test-data/clmc-validator/invalid/alerts_test_config-3.yaml
+++ b/src/service/resources/tosca/test-data/clmc-validator/invalid/alerts_test_config-3.yaml
@@ -26,7 +26,7 @@ topology_template:
granularity: 60
aggregation_method: last
resource_type:
- flame_sf_package_id: storage # sf_package_id is not the correct tag name, it is sfp
+ status_code: 200 # integers must be quoted as strings when used as tag values, e.g. '200'
flame_sf: storage-users
flame_location: watershed
comparison_operator: lt
diff --git a/src/service/resources/tosca/test-data/clmc-validator/valid/alerts_test_config-2.yaml b/src/service/resources/tosca/test-data/clmc-validator/valid/alerts_test_config-2.yaml
index da3f8d78fa16efa0a4a77d9c213bfad372ddf09a..ee3de7a27e647c208dab43f3dff920ba418eae20 100644
--- a/src/service/resources/tosca/test-data/clmc-validator/valid/alerts_test_config-2.yaml
+++ b/src/service/resources/tosca/test-data/clmc-validator/valid/alerts_test_config-2.yaml
@@ -41,6 +41,7 @@ topology_template:
threshold: -100 # requests have decreased by at least 100
granularity: 120
resource_type:
+ status_code: '200'
flame_sfp: storage
flame_sf: storage-users
flame_location: watershed
diff --git a/src/service/setup.py b/src/service/setup.py
index 10eef05579477ecee93ffeabcf9999fe54128c53..097fb2a91850353e98845f7643a0ad16ab7efdb2 100644
--- a/src/service/setup.py
+++ b/src/service/setup.py
@@ -52,6 +52,7 @@ def get_version(*relative_path):
requires = [
+ 'py2neo==4.2.0',
'plaster_pastedeploy==0.6',
'pyramid==1.9.2',
'pyramid_debugtoolbar==4.5',
@@ -61,7 +62,6 @@ requires = [
'zope.sqlalchemy==1.0',
'psycopg2==2.7.5',
'influxdb==5.2.0',
- 'py2neo==4.2.0',
'pyyaml==3.13',
'tosca-parser==1.1.0',
'schema==0.6.8',
diff --git a/src/service/tox.ini b/src/service/tox.ini
deleted file mode 100644
index 49efb9f07566378a3a7d1af73bc6e73fbd819dad..0000000000000000000000000000000000000000
--- a/src/service/tox.ini
+++ /dev/null
@@ -1,5 +0,0 @@
-[tox]
-envlist = py35
-[testenv]
-deps=pytest
-commands=pytest
\ No newline at end of file
diff --git a/src/test/VERSION b/src/test/VERSION
index b842790079f79338cf4a5f7de3efcabd6f25b0b5..b62a3e51a74a810ec8e8b760cce66cdbfac3c1d6 100644
--- a/src/test/VERSION
+++ b/src/test/VERSION
@@ -1 +1 @@
-__version__ = "2.1.1"
\ No newline at end of file
+__version__ = "2.1.2"
\ No newline at end of file
diff --git a/src/test/clmctest/alerts/test_alerts.py b/src/test/clmctest/alerts/test_alerts.py
index 60c0d598ff13c486dc71e2bc4c5216a9ece8ddaa..fcffc57631d3db40e90e9c7c1ca65906b458cd0e 100644
--- a/src/test/clmctest/alerts/test_alerts.py
+++ b/src/test/clmctest/alerts/test_alerts.py
@@ -32,6 +32,7 @@ from clmctest.alerts.alert_handler_server import LOG_TEST_FOLDER_PATH
NGINX_PORT = 80
+SFEMC = "flame_sfemc"
def is_valid_timestamp(str_timestamp):
@@ -147,19 +148,19 @@ class TestAlerts(object):
"topic_api_endpoint": "/kapacitor/v1/alerts/topics/46fb8800c8a5eeeb04b090d838d475df574a2e6d854b5d678fc981c096eb6c1b",
"topic_handlers_api_endpoint": "/kapacitor/v1/alerts/topics/46fb8800c8a5eeeb04b090d838d475df574a2e6d854b5d678fc981c096eb6c1b/handlers"},
{"policy": "scale_nginx_policy", "trigger": "increase_in_active_requests", "task_identifier": "7a9867f9270dba6635ac3760a3b70bc929f5bd0f3bf582e45d27fbd437f528ca",
- "handlers": ["flame_sfemc", "http://172.40.231.200:9999/"],
+ "handlers": [SFEMC, "http://172.40.231.200:9999/"],
"topic_identifier": "7a9867f9270dba6635ac3760a3b70bc929f5bd0f3bf582e45d27fbd437f528ca",
"task_api_endpoint": "/kapacitor/v1/tasks/7a9867f9270dba6635ac3760a3b70bc929f5bd0f3bf582e45d27fbd437f528ca",
"topic_api_endpoint": "/kapacitor/v1/alerts/topics/7a9867f9270dba6635ac3760a3b70bc929f5bd0f3bf582e45d27fbd437f528ca",
"topic_handlers_api_endpoint": "/kapacitor/v1/alerts/topics/7a9867f9270dba6635ac3760a3b70bc929f5bd0f3bf582e45d27fbd437f528ca/handlers"},
{"policy": "scale_nginx_policy", "trigger": "increase_in_running_processes", "task_identifier": "f5edaeb27fb847116be749c3815d240cbf0d7ba79aee1959daf0b3445a70f2c8",
- "handlers": ["flame_sfemc", "http://172.40.231.200:9999/"],
+ "handlers": [SFEMC, "http://172.40.231.200:9999/"],
"topic_identifier": "f5edaeb27fb847116be749c3815d240cbf0d7ba79aee1959daf0b3445a70f2c8",
"task_api_endpoint": "/kapacitor/v1/tasks/f5edaeb27fb847116be749c3815d240cbf0d7ba79aee1959daf0b3445a70f2c8",
"topic_api_endpoint": "/kapacitor/v1/alerts/topics/f5edaeb27fb847116be749c3815d240cbf0d7ba79aee1959daf0b3445a70f2c8",
"topic_handlers_api_endpoint": "/kapacitor/v1/alerts/topics/f5edaeb27fb847116be749c3815d240cbf0d7ba79aee1959daf0b3445a70f2c8/handlers"},
{"policy": "deadman_policy", "trigger": "no_measurements", "task_identifier": "f7dab6fd53001c812d44533d3bbb6ef45f0d1d39b9441bc3c60402ebda85d320",
- "handlers": ["flame_sfemc", "http://172.40.231.200:9999/"],
+ "handlers": [SFEMC, "http://172.40.231.200:9999/"],
"topic_identifier": "f7dab6fd53001c812d44533d3bbb6ef45f0d1d39b9441bc3c60402ebda85d320",
"task_api_endpoint": "/kapacitor/v1/tasks/f7dab6fd53001c812d44533d3bbb6ef45f0d1d39b9441bc3c60402ebda85d320",
"topic_api_endpoint": "/kapacitor/v1/alerts/topics/f7dab6fd53001c812d44533d3bbb6ef45f0d1d39b9441bc3c60402ebda85d320",
@@ -210,9 +211,9 @@ class TestAlerts(object):
{"policy": "scale_nginx_policy", "trigger": "increase_in_running_processes"}, {"policy": "deadman_policy", "trigger": "no_measurements"}], \
"Incorrect list of deleted alerts"
# sort by handler and trigger to ensure comparison order is correct
- assert sorted(json_response["deleted_handlers"], key=lambda x: (x['handler'], x['trigger'])) == [{"policy": "scale_nginx_policy", "trigger": "increase_in_active_requests", "handler": "flame_sfemc"},
- {"policy": "scale_nginx_policy", "trigger": "increase_in_running_processes", "handler": "flame_sfemc"},
- {"policy": "deadman_policy", "trigger": "no_measurements", "handler": "flame_sfemc"},
+ assert sorted(json_response["deleted_handlers"], key=lambda x: (x['handler'], x['trigger'])) == [{"policy": "scale_nginx_policy", "trigger": "increase_in_active_requests", "handler": SFEMC},
+ {"policy": "scale_nginx_policy", "trigger": "increase_in_running_processes", "handler": SFEMC},
+ {"policy": "deadman_policy", "trigger": "no_measurements", "handler": SFEMC},
{"policy": "scale_nginx_policy", "trigger": "high_requests", "handler": "http://172.40.231.200:9999/"},
{"policy": "scale_nginx_policy", "trigger": "increase_in_active_requests", "handler": "http://172.40.231.200:9999/"},
{"policy": "scale_nginx_policy", "trigger": "increase_in_running_processes", "handler": "http://172.40.231.200:9999/"},