Merge branch 'deploy-refactor' into dev

deploy/nginx/sites-available/pedasi

 server {
-    listen 80;
-    server_name localhost pedasi.* pedasi-dev.* *.iotobservatory.io;
+    listen 80 default_server;
+    server_name _;
 
     merge_slashes off;
 
@@ -13,7 +13,7 @@ server {
     }
 
     location = /report.html {
-        file /var/www/pedasi/report.html;
+        alias /var/www/pedasi/report.html;
         auth_basic "Restricted Content";
         auth_basic_user_file /etc/nginx/.htpasswd;
     }

deploy/nginx/sites-available/pedasi.dev

-server {
-    listen 80;
-
-    merge_slashes off;
-
-    location = /favicon.ico {
-        alias /var/www/pedasi/static/img/favicon.ico;
-    }
-
-    location /static/ {
-        alias /var/www/pedasi/static/;
-    }
-
-    location = /report.html {
-        alias /var/www/pedasi/report.html;
-        auth_basic "Restricted Content";
-        auth_basic_user_file /etc/nginx/.htpasswd;
-    }
-
-    location = /robots.txt {
-        alias /var/www/pedasi/deploy/robots.txt;
-    }
-
-    location / {
-        include     uwsgi_params;
-        uwsgi_pass  unix:/run/uwsgi/pedasi.sock;
-    }
-}

docs/source/conf.py

@@ -43,13 +43,28 @@ release = '0.1.0'
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = [
+    'sphinxcontrib.apidoc',
     'sphinx.ext.autodoc',
-    'sphinx.ext.apidoc',
     'sphinx.ext.todo',
     'sphinx.ext.coverage',
     'sphinx.ext.viewcode',
 ]
 
+# Sphinxcontrib-apidoc settings
+# Add apidoc stage to normal build process
+
+apidoc_module_dir = '../../'
+apidoc_output_dir = 'apidoc'
+apidoc_excluded_paths = [
+    '**/migrations',
+    '**/tests/',
+    '**/tests.py',
+    '**/urls.py',
+    'manage.py',
+    'pedasi/wsgi.py',
+]
+apidoc_separate_modules = True
+
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
 
@@ -83,7 +98,7 @@ pygments_style = 'sphinx'
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 #
-html_theme = 'alabaster'
+html_theme = 'sphinx_rtd_theme'
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the

docs/source/guide_user.rst

+.. _guide_user:
+
+PEDASI User Guide
+=================
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Contents:
+
+
+User Model
+----------
+
+There are four classes of users within PEDASI:
+
+ - *Basic User*: these are anonymous users, able to browse and search the catalogue's public data sources and associated metadata and retrieve public datasets from those data sources. They can also request a *PEDASI User* account. The data access activities of Basic Users are not tracked by PEDASI.
+
+ - *PEDASI User*: in addition to what a Basic User can do these users can also request access to specific data sources, which may be approved by *Data Providers*. These users are also *observed* within PEDASI, with data access activities tracked by the system (for those data sources that opt to track user activities).
+
+Another class of user is the *Provider*, used for administering PEDASI data sources and applications. These build on what PEDASI Users can do:
+
+ - *Data Provider*: these users may register new data sources within PEDASI, update data sources and their metadata, remove data sources, and approve data source access requests.
+
+ - *Application Provider*: similarly, these users may register new applications within PEDASI, update applications and their metadata, and remove applications.
+
+Applications developed for PEDASI also function as PEDASI Users, having their data access activities optionally tracked within the system, depending on the configuration of data sources.
+
+There is also a PEDASI *Central Administrator* role, able to approve PEDASI User account requests, and assign/remove Data and Application Provider roles to users.
+
+
+Obtaining a PEDASI Account
+--------------------------
+
+Why Register for an Account?
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Registering for an account enables you to be authenticated within the system as a PEDASI User and request access to private data sources and use them if approved.
+
+Data Source Activity Tracking
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+One of the research aims of PEDASI is to explore research challenges around linkages between data. As such, a PEDASI User's activities in relation to data sources is recorded within the system to assist this research, to determine usage and provenance relationships between datasets across the PEDASI user/application ecosystem, and help inform Data Providers how PEDASI uses their data sources.
+
+As an authenticated PEDASI User, the following data is recorded for each data access for those data sources that opt to record usage data:
+
+ - The date and time of access
+ - Your unique PEDASI account reference ID (not your personal Google account details)
+ - The type of activity (e.g. access, update, etc.)
+
+These records are held internally within PEDASI and are available to Central Administrators and explicitly approved users (e.g. Data Providers and researchers).
+
+
+Registering for an Account
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The preferred method to register an account with PEDASI is to register your Google account. If you don't have a Google account you can `sign up for one`_. Once you have an account you should contact the PEDASI Central Administrator via their contact details and request that your account be added to the system.
+
+Once approved, log in to the system as a PEDASI User using your Google account by selecting *Google Login* on the front page navigation bar.
+
+.. _`sign up for one`: https://accounts.google.com/signup
+
+
+Browsing the Data Catalogue
+---------------------------
+
+You can view the available data sources within PEDASI by selecting *Data Sources* from the front page navigation bar.
+
+
+Viewing Data Sources Details
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Selecting *Details* for a data source from the Data Sources page shows the following information:
+
+ - *Owner*: the Data Provider who administers this data source
+ - *URL*: a URL to the Application Programming Interface (API) for this data source
+ - *Licence*: the licence under which the data is provided. You must ensure you comply with these licensing conditions when using data provided by this data source
+ - *Metadata*: additional fields, listed by metadata type, for this data (e.g. data query parameters, etc.)
+
+
+Requesting Access to a Datasource
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+If you're logged in as a PEDASI User (and not anonymously), you may request access to a data source by selecting *Manage Access* from a data source overview page. From here you can supply the following details for an access request via a form:
+
+ - The level of access, where a given level also provides access to previous levels:
+
+   - *NONE*: to revoke access to a data source
+   - *VIEW*: to view a data source's high-level details
+   - *META*: to view a data source's metadata
+   - *PROV*: to view a data source's PROV records
+
+ - Push requested: to allow the data source to receive updates and/or new data from the user, where the data source supports it (currently only internal PEDASI data sources)
+ - Reason: the reason for the request
+
+The data source's Data Provider will then consider and optionally approve the request. Subsequent requests can be made by the user to either escalate or de-escalate their access level, each requiring approval.
+
+
+Using the Data Explorer
+^^^^^^^^^^^^^^^^^^^^^^^
+
+The Data Explorer allows you to explore and obtain data from data sources by building and submitting queries based on that data source's API parameters. This is also intended as an aid for developers who wish to understand and construct PEDASI queries within their applications.
+
+Selecting *Data Explorer* from the data source overview page shows an interface to do this using the following process:
+
+ 1. *Select a specific data set if supported*: firstly, if the data source supports multiple data sets, select one from the *Datasets* panel on the bottom right. Dataset-specific metadata will be displayed in the *Metadata* panel on the bottom left.
+
+ 2. *Use the Query Builder to construct a query*: in the *Query Builder* panel on the top left, select a query parameter from the dropdown selector (or if they are not configured for this data source, you can add one manually in the *Parameter* field), assign a value to that parameter, and select *Add to Query*. Repeat this as required to build a complete query. You'll be able to see the query that will be sent to PEDASI in the *Query URL* text box as the query is constructed.
+
+ 3. *Submit the query and see the results*: select *Submit Query* to submit the query, with the results displayed in the *Query Results* panel on the top right.
+
+Results are presented in the Query Results panel precisely as returned by the data source.
+
+
+References
+----------
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`

docs/source/index.rst

-.. PEDASI documentation master file, created by
-   sphinx-quickstart on Fri Aug 17 13:18:08 2018.
-   You can adapt this file completely to your liking, but it should at least
-   contain the root `toctree` directive.
+.. _doc-index:
 
-Welcome to PEDASI's documentation!
-==================================
+PEDASI
+======
 
 .. toctree::
    :maxdepth: 2
    :caption: Contents:
 
 
+Overview
+--------
 
-Indices and tables
-==================
+Developed as a platform and a service to explore research challenges in data security, privacy, and ethics, PEDASI enables providers of data - particularly `Internet of Things`_ data - to share their data securely within a common catalogue for use by application developers and researchers. Data can either be hosted and made accessible directly within PEDASI as an internal data source, or hosted elsewhere and accessible as an external data source through PEDASI.
+
+An initial deployment of the platform is available at https://dev.iotobservatory.io.
+
+.. _`Internet of Things`: https://en.wikipedia.org/wiki/Internet_of_things
+
+
+Key Features
+------------
+
+PEDASI's key features are:
+
+ - Searchable catalogue of supported data sources registered by data owners
+ - Extensible connector interface that currently supports HyperCat and IoTUK Nation Database data sources
+ - Dataset discovery and access via a web interface or via an Applications API
+ - Queryable and extensible metadata associated with datasets
+ - Adoption of W3C PROV-DM specification to track and record dataset creation, update, and access within internal datastore
+ - Internally hosted support for read/write NoSQL datastores
+ - Functions as a reverse proxy to data sources, returning data from requests exactly as supplied by the data source
+
+
+Resources
+---------
+
+Documentation is available for the following stakeholders:
+
+ - :ref:`Researchers and other end-users<guide_user>`: for users wishing to discover, explore, and make use of datasets using the web interface
+ - :ref:`Administrators<guide_administrator>`: for those aiming to deploy PEDASI, or provide data or use applications through PEDASI
+ - :ref:`Application developers<guide_developer>`: for developers wanting to create applications that access data available through PEDASI
+
+This documentation is also available on `Read the Docs`_.
+
+.. _`Read the Docs`: https://pedasi.readthedocs.io/en/dev
+
+
+License
+-------
+
+PEDASI is provided under the MIT licence.
+
+
+References
+----------
 
 * :ref:`genindex`
 * :ref:`modindex`

pedasi/settings.py

@@ -26,6 +26,9 @@ DEBUG
   Run the server in debug mode?
   Default is 'false'.
 
+ALLOWED_HOSTS - required if not in debug mode
+  List of hostnames on which the server is permitted to run
+
 DATABASE_URL
   URL to default SQL database - in `dj-database-url <https://github.com/kennethreitz/dj-database-url>`_ format.
   Default is SQLite3 'db.sqlite3' in project root directory.
@@ -41,7 +44,7 @@ import os
 
 from django.urls import reverse_lazy
 
-from decouple import config
+from decouple import config, Csv
 import dj_database_url
 import mongoengine
 
@@ -54,17 +57,9 @@ SECRET_KEY = config('SECRET_KEY')
 # SECURITY WARNING: don't run with debug turned on in production!
 DEBUG = config('DEBUG', default=False, cast=bool)
 
-if DEBUG:
-    ALLOWED_HOSTS = [
-        '*',
-    ]
-
-else:
-    ALLOWED_HOSTS = [
-        'localhost',
-        'pedasi-dev.eastus.cloudapp.azure.com',
-    ]
-
+ALLOWED_HOSTS = config('ALLOWED_HOSTS',
+                       cast=Csv(),
+                       default='*' if DEBUG else '127.0.0.1,localhost.localdomain,localhost')
 
 # Application definition
 

playbook.yml

@@ -46,7 +46,7 @@
           - python3
           - python3-dev
           - python3-pip
-          - python3-venv
+          - python-virtualenv
           - git
           - mysql-server
           - libmysqlclient-dev
@@ -57,8 +57,6 @@
           # Required for Ansible to setup DB
           - python3-mysqldb
 
-    - name: Setup dev deployment
-      block:
     - name: Check if running under Vagrant
       stat:
         path: /vagrant
@@ -70,70 +68,32 @@
         dest: '{{ project_dir }}'
       when: vagrant_dir.stat.exists == True
 
-      - name: Copy deploy key
-        copy:
-          src: 'deploy/.deployment-key'
-          dest: '~/.deployment-key'
-          mode: 0600
-        when: vagrant_dir.stat.exists == False
-
-      - name: Clone / update dev branch from main repo
+    - name: Clone / update branch from main repo
       git:
-          repo: 'ssh://git@github.com/PEDASI/PEDASI.git'
+        repo: 'https://github.com/PEDASI/PEDASI.git'
         dest: '{{ project_dir }}'
         accept_hostkey: yes
-          key_file: '~/.deployment-key'
-          version: dev
+        version: '{{ branch | default("dev") }}'
       when: vagrant_dir.stat.exists == False
 
-      - name: Copy dev settings
+    - name: Copy settings
       copy:
-          src: 'deploy/.env.dev'
+        src: '{{ env_file | default("deploy/.env") }}'
         dest: '{{ project_dir }}/.env'
         owner: www-data
         group: www-data
         mode: 0600
 
-      when: production is not defined
-
-    - name: Setup production deployment
-      block:
-      - name: Copy deploy key
-        copy:
-          src: 'deploy/.deployment-key'
-          dest: '~/.deployment-key'
-          mode: 0600
-
-      - name: Clone / update master branch
-        git:
-          repo: 'ssh://git@github.com/PEDASI/PEDASI.git'
-          dest: '{{ project_dir }}'
-          accept_hostkey: yes
-          key_file: '~/.deployment-key'
-          version: master
-
-      - name: Copy production settings
-        copy:
-          src: 'deploy/.env.prod'
-          dest: '{{ project_dir }}/.env'
-          owner: www-data
-          group: www-data
-          mode: 0600
-
-      when: production is defined
-
     - name: Set permissions on manage.py
       file:
         path: '{{ project_dir }}/manage.py'
         mode: 0755
 
-    - name: Create virtualenv
-      command: python3 -m venv '{{ venv_dir }}' creates='{{ venv_dir }}'
-
     - name: Install pip requirements
       pip:
         requirements: '{{ project_dir}}/requirements.txt'
         virtualenv: '{{ venv_dir }}'
+        virtualenv_python: python3
 
     - name: Restart and enable MariaDB
       systemd:
@@ -195,23 +155,13 @@
         - { src: '{{ project_dir }}/deploy/uwsgi/sites/pedasi.ini', dest: /etc/uwsgi/sites/pedasi.ini }
         - { src: '{{ project_dir }}/deploy/systemd/system/uwsgi.service', dest: /etc/systemd/system/uwsgi.service }
 
-    - name: Copy web config files - dev
-      copy:
-        src: '{{ project_dir }}/deploy/nginx/sites-available/pedasi.dev'
-        dest: /etc/nginx/sites-available/pedasi
-        remote_src: yes
-        # WARNING: this will not update an existing file
-        force: no
-      when: production is not defined
-
-    - name: Copy web config files - production
+    - name: Copy web config files
       copy:
         src: '{{ project_dir }}/deploy/nginx/sites-available/pedasi'
         dest: /etc/nginx/sites-available/pedasi
         remote_src: yes
         # WARNING: this will not update an existing file
         force: no
-      when: production is defined
 
     - name: Activate Nginx site
       file:
@@ -279,7 +229,6 @@
           SPHINXAPIDOC: '{{ venv_dir }}/bin/sphinx-apidoc'
       loop:
       - clean
-      - apidoc
       - html
 
     - name: Django collect static files

requirements.txt

@@ -29,6 +29,7 @@ mysqlclient==1.3.13
 networkx==2.2
 oauthlib==2.1.0
 packaging==17.1
+pbr==5.1.2
 prov==1.5.2
 pycparser==2.18
 Pygments==2.2.0
@@ -50,6 +51,8 @@ snowballstemmer==1.2.1
 social-auth-app-django==3.0.0
 social-auth-core==2.0.0
 Sphinx==1.7.6
+sphinx-rtd-theme==0.4.2
+sphinxcontrib-apidoc==0.3.0
 sphinxcontrib-websupport==1.1.0
 SQLAlchemy==1.2.14
 typed-ast==1.1.0