Merge branch 'integration'

2fc90bf4 · Michael Boniface · 2581c211 · e4701891 · 2fc90bf4 · 2fc90bf4
Commit 2fc90bf4 authored 6 years ago by Michael Boniface
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -35,8 +35,8 @@ build:tests:
    - python setup.py sdist --dist-dir=$CI_PROJECT_DIR/build
  artifacts:
    paths:
-    - build/clmctest-1.3.0.tar.gz
-    - build/clmcservice-1.3.0.tar.gz
+    - build/clmctest-1.4.0.tar.gz
+    - build/clmcservice-1.4.0.tar.gz
    expire_in: 1 day

 test:all:
@@ -50,10 +50,10 @@ test:all:
    - echo "REPO_PASS=${REPO_PASS}" >> $CI_PROJECT_DIR/reporc
    - sudo scripts/test/fixture.sh create -f src/test/clmctest/rspec.json -r $CI_PROJECT_DIR -c all
    - sudo mkdir /var/lib/lxd/containers/test-runner/rootfs/opt/clmc/build
-    - sudo cp build/clmctest-1.3.0.tar.gz /var/lib/lxd/containers/test-runner/rootfs/opt/clmc/build
-    - sudo cp build/clmcservice-1.3.0.tar.gz /var/lib/lxd/containers/test-runner/rootfs/opt/clmc/build
-    - sudo lxc exec test-runner -- pip3 install /opt/clmc/build/clmctest-1.3.0.tar.gz
-    - sudo lxc exec test-runner -- pip3 install /opt/clmc/build/clmcservice-1.3.0.tar.gz
+    - sudo cp build/clmctest-1.4.0.tar.gz /var/lib/lxd/containers/test-runner/rootfs/opt/clmc/build
+    - sudo cp build/clmcservice-1.4.0.tar.gz /var/lib/lxd/containers/test-runner/rootfs/opt/clmc/build
+    - sudo lxc exec test-runner -- pip3 install /opt/clmc/build/clmctest-1.4.0.tar.gz
+    - sudo lxc exec test-runner -- pip3 install /opt/clmc/build/clmcservice-1.4.0.tar.gz
    - sudo lxc exec test-runner -- pytest -s --tb=short -rfp --pyargs clmctest
  when: on_success      
  

--- a/IPRREGISTRY.md
+++ b/IPRREGISTRY.md
+# Registry of 3rd Party Intellectual Property Rights used by Cross Layer Management and Control Software
+
+The CLMC depends on 3rd party open source software distributed using approved open source software licenses. 
+
+## Unmodified Libraries
+
+| Library | License | License Link |
+| ------- | ------- | ------------ |
+| pyramid | Repoze Public License (BSD-derived) http://repoze.org/license.html |https://github.com/Pylons/pyramid/blob/master/LICENSE.txt |
+| pyramid_debugtoolbar | Repoze Public License (BSD-derived) http://repoze.org/license.html |	https://github.com/Pylons/pyramid_debugtoolbar/blob/master/LICENSE.txt |
+| pyramid_exclog | Repoze Public License (BSD-derived)  http://repoze.org/license.html |	https://github.com/Pylons/pyramid_exclog/blob/master/LICENSE.txt |
+| plaster_pastedeploy | MIT LICENSE | https://github.com/Pylons/plaster_pastedeploy/blob/master/LICENSE.txt |
+| waitress | ZOPE PUBLIC LICENSE (ZPL) v2.1 | https://github.com/Pylons/waitress/blob/master/LICENSE.txt |
+| sqlalchemy | MIT LICENSE | https://github.com/sqlalchemy/sqlalchemy/blob/master/LICENSE |	
+| zope.sqlalchemy | ZOPE PUBLIC LICENSE (ZPL) v2.1 |https://github.com/zopefoundation/zope.sqlalchemy/blob/master/LICENSE.txt |
+| psycopg2 | GNU LESSER GENERAL PUBLIC LICENSE v3| https://github.com/psycopg/psycopg2/blob/master/LICENSE |
+| influxdb-python | MIT LICENSE	| https://github.com/influxdata/influxdb-python/blob/master/LICENSE |
+| py2neo | APACHE LICENSE v2 | https://github.com/technige/py2neo/blob/py2neo-4.1.3/LICENSE |	
+| pyyaml | MIT LICENSE | https://github.com/yaml/pyyaml/blob/master/LICENSE |	
+| tosca-parser | APACHE LICENSE v2 | https://github.com/openstack/tosca-parser/blob/master/LICENSE |
+| schema | MIT LICENSE | https://github.com/keleshev/schema/blob/master/LICENSE-MIT |	
+| requests | APACHE LICENSE v2 | https://github.com/requests/requests/blob/master/LICENSE |
+| pytest | MIT LICENSE | https://github.com/pytest-dev/pytest/blob/master/LICENSE |	
+| pytest-cov | MIT LICENSE | https://github.com/pytest-dev/pytest-cov/blob/master/LICENSE |
+			
+## Unmodified Service Prerequisites
+
+The following services need to be installed by the recipient as prerequisites for the CLMC
+
+| Library | License | License Link |
+| ------- | ------- | ------------ |
+| InfluxDB | MIT LICENSE | https://github.com/influxdata/influxdb/blob/master/LICENSE |
+| Chronograf | GNU AFFERO GENERAL PUBLIC LICENSE v3 |  https://github.com/influxdata/chronograf/blob/master/LICENSE |
+| Kapacitor | MIT LICENSE | https://github.com/influxdata/kapacitor/blob/master/LICENSE |
+| Neo4j | GNU GENERAL PUBLIC LICENSE v3 | https://neo4j.com/licensing/ |			
+| PostgreSQL | PostgreSQL License | https://www.postgresql.org/about/licence/ |
+			
--- a/LICENSE
+++ b/LICENSE
--- a/Vagrantfile
+++ b/Vagrantfile
+## © University of Southampton IT Innovation Centre, 2018
+##
+## Copyright in this software belongs to University of Southampton
+## IT Innovation Centre of Gamma House, Enterprise Road,
+## Chilworth Science Park, Southampton, SO16 7NS, UK.
+##
+## This software may not be used, sold, licensed, transferred, copied
+## or reproduced in whole or in part in any manner or form or in or
+## on any media by any person other than in accordance with the terms
+## of the Licence Agreement supplied with the software, or otherwise
+## without the prior written consent of the copyright owners.
+##
+## This software is distributed WITHOUT ANY WARRANTY, without even the
+## implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+## PURPOSE, except where stated in the Licence Agreement supplied with
+## the software.
+##
+##      Created By :            Michael Boniface
+##      Created Date :          18-12-2018
+##      Created for Project :   FLAME
+
 $lxc_script = <<-SCRIPT

 apt-get update

--- a/docs/aggregation.md
+++ b/docs/aggregation.md
 <!--
-// © University of Southampton IT Innovation Centre, 2017
+// © University of Southampton IT Innovation Centre, 2018
 //
 // Copyright in this software belongs to University of Southampton
 // IT Innovation Centre of Gamma House, Enterprise Road, 

--- a/docs/dashboards.md
+++ b/docs/dashboards.md
+<!--
+// © University of Southampton IT Innovation Centre, 2018
+//
+// Copyright in this software belongs to University of Southampton
+// IT Innovation Centre of Gamma House, Enterprise Road, 
+// Chilworth Science Park, Southampton, SO16 7NS, UK.
+//
+// This software may not be used, sold, licensed, transferred, copied
+// or reproduced in whole or in part in any manner or form or in or
+// on any media by any person other than in accordance with the terms
+// of the Licence Agreement supplied with the software, or otherwise
+// without the prior written consent of the copyright owners.
+//
+// This software is distributed WITHOUT ANY WARRANTY, without even the
+// implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE, except where stated in the Licence Agreement supplied with
+// the software.
+//
+//      Created By :            Nikolay Stanchev, Michael Boniface
+//      Created Date :          27-04-2018
+//      Created for Project :   FLAME
+-->
+
 # Sample Chronograf Dashboards

 The folder located at src/test/clmctest/dashboards contains several sample dashboards for use in Chronograf.

--- a/docs/flips-integration.md
+++ b/docs/flips-integration.md
+<!--
+// © University of Southampton IT Innovation Centre, 2018
+//
+// Copyright in this software belongs to University of Southampton
+// IT Innovation Centre of Gamma House, Enterprise Road, 
+// Chilworth Science Park, Southampton, SO16 7NS, UK.
+//
+// This software may not be used, sold, licensed, transferred, copied
+// or reproduced in whole or in part in any manner or form or in or
+// on any media by any person other than in accordance with the terms
+// of the Licence Agreement supplied with the software, or otherwise
+// without the prior written consent of the copyright owners.
+//
+// This software is distributed WITHOUT ANY WARRANTY, without even the
+// implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE, except where stated in the Licence Agreement supplied with
+// the software.
+//
+//      Created By :            Michael Boniface
+//      Created Date :          18-12-2018
+//      Created for Project :   FLAME
+-->
+
 # **Service Management and Control Decisions**

 © University of Southampton IT Innovation Centre, 2018

--- a/docs/service-management-decisions.md
+++ b/docs/service-management-decisions.md
+<!--
+// © University of Southampton IT Innovation Centre, 2017
+//
+// Copyright in this software belongs to University of Southampton
+// IT Innovation Centre of Gamma House, Enterprise Road, 
+// Chilworth Science Park, Southampton, SO16 7NS, UK.
+//
+// This software may not be used, sold, licensed, transferred, copied
+// or reproduced in whole or in part in any manner or form or in or
+// on any media by any person other than in accordance with the terms
+// of the Licence Agreement supplied with the software, or otherwise
+// without the prior written consent of the copyright owners.
+//
+// This software is distributed WITHOUT ANY WARRANTY, without even the
+// implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE, except where stated in the Licence Agreement supplied with
+// the software.
+//
+//      Created By :            Michael Boniface
+//      Created Date :          18-12-2017
+//      Created for Project :   FLAME
+-->
+
 # **Service Management and Control Decisions**

 © University of Southampton IT Innovation Centre, 2017

--- a/scripts/clmc-agent/build-telegraf.sh
+++ b/scripts/clmc-agent/build-telegraf.sh
 #!/bin/bash
+#/////////////////////////////////////////////////////////////////////////
+#//
+#// (c) University of Southampton IT Innovation Centre, 2018
+#//
+#// Copyright in this software belongs to University of Southampton
+#// IT Innovation Centre of Gamma House, Enterprise Road,
+#// Chilworth Science Park, Southampton, SO16 7NS, UK.
+#//
+#// This software may not be used, sold, licensed, transferred, copied
+#// or reproduced in whole or in part in any manner or form or in or
+#// on any media by any person other than in accordance with the terms
+#// of the Licence Agreement supplied with the software, or otherwise
+#// without the prior written consent of the copyright owners.
+#//
+#// This software is distributed WITHOUT ANY WARRANTY, without even the
+#// implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+#// PURPOSE, except where stated in the Licence Agreement supplied with
+#// the software.
+#//
+#//      Created By :            Michael Boniface
+#//      Created Date :          13/12/2017
+#//      Updated By :            Nikolay Stanchev
+#//      Updated Date :          30/08/2018
+#//      Created for Project :   FLAME
+#//
+#/////////////////////////////////////////////////////////////////////////

 # install build prequisites
 apt-get install ruby ruby-dev rubygems build-essential rpm -y

--- a/scripts/clmc-agent/configure.sh
+++ b/scripts/clmc-agent/configure.sh
 #!/bin/bash
 #/////////////////////////////////////////////////////////////////////////
 #//
-#// (c) University of Southampton IT Innovation Centre, 2017
+#// (c) University of Southampton IT Innovation Centre, 2018
 #//
 #// Copyright in this software belongs to University of Southampton
 #// IT Innovation Centre of Gamma House, Enterprise Road,

--- a/scripts/clmc-agent/telegraf.conf
+++ b/scripts/clmc-agent/telegraf.conf
-## © University of Southampton IT Innovation Centre, 2017
+## © University of Southampton IT Innovation Centre, 2018
 ##
 ## Copyright in this software belongs to University of Southampton
 ## IT Innovation Centre of Gamma House, Enterprise Road, 

--- a/scripts/clmc-service/install-clmc-service.sh
+++ b/scripts/clmc-service/install-clmc-service.sh
 #!/bin/bash
-
+#/////////////////////////////////////////////////////////////////////////
+#//
+#// (c) University of Southampton IT Innovation Centre, 2018
+#//
+#// Copyright in this software belongs to University of Southampton
+#// IT Innovation Centre of Gamma House, Enterprise Road,
+#// Chilworth Science Park, Southampton, SO16 7NS, UK.
+#//
+#// This software may not be used, sold, licensed, transferred, copied
+#// or reproduced in whole or in part in any manner or form or in or
+#// on any media by any person other than in accordance with the terms
+#// of the Licence Agreement supplied with the software, or otherwise
+#// without the prior written consent of the copyright owners.
+#//
+#// This software is distributed WITHOUT ANY WARRANTY, without even the
+#// implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+#// PURPOSE, except where stated in the Licence Agreement supplied with
+#// the software.
+#//
+#//      Created By :            Michael Boniface, Nikolay Sanchev
+#//      Created Date :          13/12/2018
+#//      Created for Project :   FLAME
+#//
+#/////////////////////////////////////////////////////////////////////////
 apt-get update
 apt-get install libssl-dev -y


--- a/scripts/clmc-service/install-neo4j.sh
+++ b/scripts/clmc-service/install-neo4j.sh
 #!/bin/bash
+#/////////////////////////////////////////////////////////////////////////
+#//
+#// (c) University of Southampton IT Innovation Centre, 2018
+#//
+#// Copyright in this software belongs to University of Southampton
+#// IT Innovation Centre of Gamma House, Enterprise Road,
+#// Chilworth Science Park, Southampton, SO16 7NS, UK.
+#//
+#// This software may not be used, sold, licensed, transferred, copied
+#// or reproduced in whole or in part in any manner or form or in or
+#// on any media by any person other than in accordance with the terms
+#// of the Licence Agreement supplied with the software, or otherwise
+#// without the prior written consent of the copyright owners.
+#//
+#// This software is distributed WITHOUT ANY WARRANTY, without even the
+#// implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+#// PURPOSE, except where stated in the Licence Agreement supplied with
+#// the software.
+#//
+#//      Created By :            Michael Boniface, Nikolay Sanchev
+#//      Created Date :          13/12/2018
+#//      Created for Project :   FLAME
+#//
+#/////////////////////////////////////////////////////////////////////////

 # status of neo4j can be found using: journalctl -e -u neo4j
 # Directories: 

--- a/scripts/clmc-service/install-tick-stack.sh
+++ b/scripts/clmc-service/install-tick-stack.sh
 #!/bin/bash
+#/////////////////////////////////////////////////////////////////////////
+#//
+#// (c) University of Southampton IT Innovation Centre, 2018
+#//
+#// Copyright in this software belongs to University of Southampton
+#// IT Innovation Centre of Gamma House, Enterprise Road,
+#// Chilworth Science Park, Southampton, SO16 7NS, UK.
+#//
+#// This software may not be used, sold, licensed, transferred, copied
+#// or reproduced in whole or in part in any manner or form or in or
+#// on any media by any person other than in accordance with the terms
+#// of the Licence Agreement supplied with the software, or otherwise
+#// without the prior written consent of the copyright owners.
+#//
+#// This software is distributed WITHOUT ANY WARRANTY, without even the
+#// implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+#// PURPOSE, except where stated in the Licence Agreement supplied with
+#// the software.
+#//
+#//      Created By :            Michael Boniface, Nikolay Sanchev
+#//      Created Date :          13/12/2018
+#//      Created for Project :   FLAME
+#//
+#/////////////////////////////////////////////////////////////////////////

 echo "----> Installing Tick Stack"
 # Define tickstack software versions

--- a/scripts/clmc-service/nginx.conf
+++ b/scripts/clmc-service/nginx.conf
+#/////////////////////////////////////////////////////////////////////////
+#//
+#// (c) University of Southampton IT Innovation Centre, 2018
+#//
+#// Copyright in this software belongs to University of Southampton
+#// IT Innovation Centre of Gamma House, Enterprise Road,
+#// Chilworth Science Park, Southampton, SO16 7NS, UK.
+#//
+#// This software may not be used, sold, licensed, transferred, copied
+#// or reproduced in whole or in part in any manner or form or in or
+#// on any media by any person other than in accordance with the terms
+#// of the Licence Agreement supplied with the software, or otherwise
+#// without the prior written consent of the copyright owners.
+#//
+#// This software is distributed WITHOUT ANY WARRANTY, without even the
+#// implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+#// PURPOSE, except where stated in the Licence Agreement supplied with
+#// the software.
+#//
+#//      Created By :            Nikolay Sanchev
+#//      Created Date :          13/12/2018
+#//      Created for Project :   FLAME
+#//
+#/////////////////////////////////////////////////////////////////////////
+
 user www-data;
 worker_processes auto;
 pid /run/nginx.pid;

--- a/scripts/test/deleteallvms.py
+++ b/scripts/test/deleteallvms.py
@@ -19,6 +19,7 @@
 ##      Created By :            Michael Boniface
 ##      Created Date :          28-03-2018
 ##      Created for Project :   FLAME
+
 import subprocess
 import sys


--- a/scripts/test/fixture.sh
+++ b/scripts/test/fixture.sh
 #!/bin/bash
+#/////////////////////////////////////////////////////////////////////////
+#//
+#// (c) University of Southampton IT Innovation Centre, 2018
+#//
+#// Copyright in this software belongs to University of Southampton
+#// IT Innovation Centre of Gamma House, Enterprise Road,
+#// Chilworth Science Park, Southampton, SO16 7NS, UK.
+#//
+#// This software may not be used, sold, licensed, transferred, copied
+#// or reproduced in whole or in part in any manner or form or in or
+#// on any media by any person other than in accordance with the terms
+#// of the Licence Agreement supplied with the software, or otherwise
+#// without the prior written consent of the copyright owners.
+#//
+#// This software is distributed WITHOUT ANY WARRANTY, without even the
+#// implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+#// PURPOSE, except where stated in the Licence Agreement supplied with
+#// the software.
+#//
+#//      Created By :            Michael Boniface
+#//      Created Date :          13/12/2018
+#//      Created for Project :   FLAME
+#//
+#/////////////////////////////////////////////////////////////////////////

 repo_root="/vagrant"
 target_root="/opt/clmc"

--- a/scripts/test/install-git-runner.sh
+++ b/scripts/test/install-git-runner.sh
 #!/bin/bash
-apt-get update
+#/////////////////////////////////////////////////////////////////////////
+#//
+#// (c) University of Southampton IT Innovation Centre, 2018
+#//
+#// Copyright in this software belongs to University of Southampton
+#// IT Innovation Centre of Gamma House, Enterprise Road,
+#// Chilworth Science Park, Southampton, SO16 7NS, UK.
+#//
+#// This software may not be used, sold, licensed, transferred, copied
+#// or reproduced in whole or in part in any manner or form or in or
+#// on any media by any person other than in accordance with the terms
+#// of the Licence Agreement supplied with the software, or otherwise
+#// without the prior written consent of the copyright owners.
+#//
+#// This software is distributed WITHOUT ANY WARRANTY, without even the
+#// implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+#// PURPOSE, except where stated in the Licence Agreement supplied with
+#// the software.
+#//
+#//      Created By :            Michael Boniface
+#//      Created Date :          13/12/2018
+#//      Created for Project :   FLAME
+#//
+#/////////////////////////////////////////////////////////////////////////

+apt-get update

 wget -O /usr/local/bin/gitlab-runner https://gitlab-runner-downloads.s3.amazonaws.com/latest/binaries/gitlab-runner-linux-amd64
 chmod +x /usr/local/bin/gitlab-runner

--- a/src/service/VERSION
+++ b/src/service/VERSION
-__version__ = "1.3.0"
\ No newline at end of file
+__version__ = "1.4.0"
\ No newline at end of file
--- a/src/service/clmcservice/generate_network_measurements.py
+++ b/src/service/clmcservice/generate_network_measurements.py
+#!/usr/bin/python3
+"""
+// © University of Southampton IT Innovation Centre, 2018
+//
+// Copyright in this software belongs to University of Southampton
+// IT Innovation Centre of Gamma House, Enterprise Road,
+// Chilworth Science Park, Southampton, SO16 7NS, UK.
+//
+// This software may not be used, sold, licensed, transferred, copied
+// or reproduced in whole or in part in any manner or form or in or
+// on any media by any person other than in accordance with the terms
+// of the Licence Agreement supplied with the software, or otherwise
+// without the prior written consent of the copyright owners.
+//
+// This software is distributed WITHOUT ANY WARRANTY, without even the
+// implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE, except where stated in the Licence Agreement supplied with
+// the software.
+//
+//      Created By :            Nikolay Stanchev
+//      Created Date :          25-06-2018
+//      Created for Project :   FLAME
+"""
 import getopt
 import sys
 from itertools import permutations
@@ -53,9 +76,9 @@ def report_network_measurements(influx_host, db_name, json_data, neo4j_host, neo
            to_node = Node("ComputeNode", name=to_node_name)
            graph.create(to_node)

-        # query = 'SELECT mean(*) FROM "CLMCMetrics"."autogen"."ping" WHERE host=\'{0}\' and url=\'{1}\' and time>={2} and time<{3}'.format(from_node['name'], to_node['name'], from_timestamp, to_timestamp)
+        # query = 'SELECT mean(*) FROM "{0}"."autogen"."ping" WHERE host=\'{1}\' and url=\'{2}\' and time>={3} and time<{4}'.format(db_name, from_node['name'], to_node['name'], from_timestamp, to_timestamp)
        # In future when latencies are reported continuously, we should put timestamp filtering in the query for network links
-        query = 'SELECT mean(*) FROM "CLMCMetrics"."autogen"."ping" WHERE host=\'{0}\' and url=\'{1}\''.format(from_node['name'], to_node['name'])
+        query = 'SELECT mean(*) FROM "{0}"."autogen"."ping" WHERE host=\'{1}\' and url=\'{2}\''.format(db_name, from_node['name'], to_node['name'])
        print("Executing query: {0}".format(query))

        result = db_client.query(query)  # execute the query