diff --git a/deploy/nginx/sites-available/pedasi b/deploy/nginx/sites-available/pedasi index 13d182189328d6a49a1c3769fd0d22c8d0545429..fa280989285d76613702afb3dda2b1e0dd6db0b9 100644 --- a/deploy/nginx/sites-available/pedasi +++ b/deploy/nginx/sites-available/pedasi @@ -1,6 +1,6 @@ server { - listen 80; - server_name localhost pedasi.* pedasi-dev.* *.iotobservatory.io; + listen 80 default_server; + server_name _; merge_slashes off; @@ -13,7 +13,7 @@ server { } location = /report.html { - file /var/www/pedasi/report.html; + alias /var/www/pedasi/report.html; auth_basic "Restricted Content"; auth_basic_user_file /etc/nginx/.htpasswd; } diff --git a/deploy/nginx/sites-available/pedasi.dev b/deploy/nginx/sites-available/pedasi.dev deleted file mode 100644 index 2493ef39e0c4af356f3084889badafcbd33a7700..0000000000000000000000000000000000000000 --- a/deploy/nginx/sites-available/pedasi.dev +++ /dev/null @@ -1,28 +0,0 @@ -server { - listen 80; - - merge_slashes off; - - location = /favicon.ico { - alias /var/www/pedasi/static/img/favicon.ico; - } - - location /static/ { - alias /var/www/pedasi/static/; - } - - location = /report.html { - alias /var/www/pedasi/report.html; - auth_basic "Restricted Content"; - auth_basic_user_file /etc/nginx/.htpasswd; - } - - location = /robots.txt { - alias /var/www/pedasi/deploy/robots.txt; - } - - location / { - include uwsgi_params; - uwsgi_pass unix:/run/uwsgi/pedasi.sock; - } -} diff --git a/docs/source/conf.py b/docs/source/conf.py index 7d119329143489789d82950c0dca2a1ddfcdc5d7..8199c71e1145b721777d2c44f449b133114f2cea 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -43,13 +43,28 @@ release = '0.1.0' # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ + 'sphinxcontrib.apidoc', 'sphinx.ext.autodoc', - 'sphinx.ext.apidoc', 'sphinx.ext.todo', 'sphinx.ext.coverage', 'sphinx.ext.viewcode', ] +# Sphinxcontrib-apidoc settings +# Add apidoc stage to normal build process + +apidoc_module_dir = '../../' +apidoc_output_dir = 'apidoc' +apidoc_excluded_paths = [ + '**/migrations', + '**/tests/', + '**/tests.py', + '**/urls.py', + 'manage.py', + 'pedasi/wsgi.py', +] +apidoc_separate_modules = True + # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] @@ -83,7 +98,7 @@ pygments_style = 'sphinx' # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = 'alabaster' +html_theme = 'sphinx_rtd_theme' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the diff --git a/docs/source/guide_user.rst b/docs/source/guide_user.rst new file mode 100644 index 0000000000000000000000000000000000000000..4785d4780a7db3cbd7fc9d0794c09995954f564e --- /dev/null +++ b/docs/source/guide_user.rst @@ -0,0 +1,119 @@ +.. _guide_user: + +PEDASI User Guide +================= + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + +User Model +---------- + +There are four classes of users within PEDASI: + + - *Basic User*: these are anonymous users, able to browse and search the catalogue's public data sources and associated metadata and retrieve public datasets from those data sources. They can also request a *PEDASI User* account. The data access activities of Basic Users are not tracked by PEDASI. + + - *PEDASI User*: in addition to what a Basic User can do these users can also request access to specific data sources, which may be approved by *Data Providers*. These users are also *observed* within PEDASI, with data access activities tracked by the system (for those data sources that opt to track user activities). + +Another class of user is the *Provider*, used for administering PEDASI data sources and applications. These build on what PEDASI Users can do: + + - *Data Provider*: these users may register new data sources within PEDASI, update data sources and their metadata, remove data sources, and approve data source access requests. + + - *Application Provider*: similarly, these users may register new applications within PEDASI, update applications and their metadata, and remove applications. + +Applications developed for PEDASI also function as PEDASI Users, having their data access activities optionally tracked within the system, depending on the configuration of data sources. + +There is also a PEDASI *Central Administrator* role, able to approve PEDASI User account requests, and assign/remove Data and Application Provider roles to users. + + +Obtaining a PEDASI Account +-------------------------- + +Why Register for an Account? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Registering for an account enables you to be authenticated within the system as a PEDASI User and request access to private data sources and use them if approved. + +Data Source Activity Tracking +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +One of the research aims of PEDASI is to explore research challenges around linkages between data. As such, a PEDASI User's activities in relation to data sources is recorded within the system to assist this research, to determine usage and provenance relationships between datasets across the PEDASI user/application ecosystem, and help inform Data Providers how PEDASI uses their data sources. + +As an authenticated PEDASI User, the following data is recorded for each data access for those data sources that opt to record usage data: + + - The date and time of access + - Your unique PEDASI account reference ID (not your personal Google account details) + - The type of activity (e.g. access, update, etc.) + +These records are held internally within PEDASI and are available to Central Administrators and explicitly approved users (e.g. Data Providers and researchers). + + +Registering for an Account +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The preferred method to register an account with PEDASI is to register your Google account. If you don't have a Google account you can `sign up for one`_. Once you have an account you should contact the PEDASI Central Administrator via their contact details and request that your account be added to the system. + +Once approved, log in to the system as a PEDASI User using your Google account by selecting *Google Login* on the front page navigation bar. + +.. _`sign up for one`: https://accounts.google.com/signup + + +Browsing the Data Catalogue +--------------------------- + +You can view the available data sources within PEDASI by selecting *Data Sources* from the front page navigation bar. + + +Viewing Data Sources Details +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Selecting *Details* for a data source from the Data Sources page shows the following information: + + - *Owner*: the Data Provider who administers this data source + - *URL*: a URL to the Application Programming Interface (API) for this data source + - *Licence*: the licence under which the data is provided. You must ensure you comply with these licensing conditions when using data provided by this data source + - *Metadata*: additional fields, listed by metadata type, for this data (e.g. data query parameters, etc.) + + +Requesting Access to a Datasource +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If you're logged in as a PEDASI User (and not anonymously), you may request access to a data source by selecting *Manage Access* from a data source overview page. From here you can supply the following details for an access request via a form: + + - The level of access, where a given level also provides access to previous levels: + + - *NONE*: to revoke access to a data source + - *VIEW*: to view a data source's high-level details + - *META*: to view a data source's metadata + - *PROV*: to view a data source's PROV records + + - Push requested: to allow the data source to receive updates and/or new data from the user, where the data source supports it (currently only internal PEDASI data sources) + - Reason: the reason for the request + +The data source's Data Provider will then consider and optionally approve the request. Subsequent requests can be made by the user to either escalate or de-escalate their access level, each requiring approval. + + +Using the Data Explorer +^^^^^^^^^^^^^^^^^^^^^^^ + +The Data Explorer allows you to explore and obtain data from data sources by building and submitting queries based on that data source's API parameters. This is also intended as an aid for developers who wish to understand and construct PEDASI queries within their applications. + +Selecting *Data Explorer* from the data source overview page shows an interface to do this using the following process: + + 1. *Select a specific data set if supported*: firstly, if the data source supports multiple data sets, select one from the *Datasets* panel on the bottom right. Dataset-specific metadata will be displayed in the *Metadata* panel on the bottom left. + + 2. *Use the Query Builder to construct a query*: in the *Query Builder* panel on the top left, select a query parameter from the dropdown selector (or if they are not configured for this data source, you can add one manually in the *Parameter* field), assign a value to that parameter, and select *Add to Query*. Repeat this as required to build a complete query. You'll be able to see the query that will be sent to PEDASI in the *Query URL* text box as the query is constructed. + + 3. *Submit the query and see the results*: select *Submit Query* to submit the query, with the results displayed in the *Query Results* panel on the top right. + +Results are presented in the Query Results panel precisely as returned by the data source. + + +References +---------- + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/docs/source/index.rst b/docs/source/index.rst index 54593fd058831f4bcec723c9f8238953649899ab..aa1d9f3dbc3acfbb9604b196847fc60dc8eda63b 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -1,19 +1,59 @@ -.. PEDASI documentation master file, created by - sphinx-quickstart on Fri Aug 17 13:18:08 2018. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. +.. _doc-index: -Welcome to PEDASI's documentation! -================================== +PEDASI +====== .. toctree:: :maxdepth: 2 :caption: Contents: +Overview +-------- -Indices and tables -================== +Developed as a platform and a service to explore research challenges in data security, privacy, and ethics, PEDASI enables providers of data - particularly `Internet of Things`_ data - to share their data securely within a common catalogue for use by application developers and researchers. Data can either be hosted and made accessible directly within PEDASI as an internal data source, or hosted elsewhere and accessible as an external data source through PEDASI. + +An initial deployment of the platform is available at https://dev.iotobservatory.io. + +.. _`Internet of Things`: https://en.wikipedia.org/wiki/Internet_of_things + + +Key Features +------------ + +PEDASI's key features are: + + - Searchable catalogue of supported data sources registered by data owners + - Extensible connector interface that currently supports HyperCat and IoTUK Nation Database data sources + - Dataset discovery and access via a web interface or via an Applications API + - Queryable and extensible metadata associated with datasets + - Adoption of W3C PROV-DM specification to track and record dataset creation, update, and access within internal datastore + - Internally hosted support for read/write NoSQL datastores + - Functions as a reverse proxy to data sources, returning data from requests exactly as supplied by the data source + + +Resources +--------- + +Documentation is available for the following stakeholders: + + - :ref:`Researchers and other end-users<guide_user>`: for users wishing to discover, explore, and make use of datasets using the web interface + - :ref:`Administrators<guide_administrator>`: for those aiming to deploy PEDASI, or provide data or use applications through PEDASI + - :ref:`Application developers<guide_developer>`: for developers wanting to create applications that access data available through PEDASI + +This documentation is also available on `Read the Docs`_. + +.. _`Read the Docs`: https://pedasi.readthedocs.io/en/dev + + +License +------- + +PEDASI is provided under the MIT licence. + + +References +---------- * :ref:`genindex` * :ref:`modindex` diff --git a/pedasi/settings.py b/pedasi/settings.py index 85512f8c20d97a3ebf3acf11b6701f0715fbbb3c..174bf975c4e86d58555365e1cfff9d2f5b891cea 100644 --- a/pedasi/settings.py +++ b/pedasi/settings.py @@ -26,6 +26,9 @@ DEBUG Run the server in debug mode? Default is 'false'. +ALLOWED_HOSTS - required if not in debug mode + List of hostnames on which the server is permitted to run + DATABASE_URL URL to default SQL database - in `dj-database-url <https://github.com/kennethreitz/dj-database-url>`_ format. Default is SQLite3 'db.sqlite3' in project root directory. @@ -41,7 +44,7 @@ import os from django.urls import reverse_lazy -from decouple import config +from decouple import config, Csv import dj_database_url import mongoengine @@ -54,17 +57,9 @@ SECRET_KEY = config('SECRET_KEY') # SECURITY WARNING: don't run with debug turned on in production! DEBUG = config('DEBUG', default=False, cast=bool) -if DEBUG: - ALLOWED_HOSTS = [ - '*', - ] - -else: - ALLOWED_HOSTS = [ - 'localhost', - 'pedasi-dev.eastus.cloudapp.azure.com', - ] - +ALLOWED_HOSTS = config('ALLOWED_HOSTS', + cast=Csv(), + default='*' if DEBUG else '127.0.0.1,localhost.localdomain,localhost') # Application definition diff --git a/playbook.yml b/playbook.yml index 74a342fcdc54335d494c5526a6c1e7196fea0f9f..ba9f61fe839f3c10904e85b47904c1477f00d3ab 100644 --- a/playbook.yml +++ b/playbook.yml @@ -46,7 +46,7 @@ - python3 - python3-dev - python3-pip - - python3-venv + - python-virtualenv - git - mysql-server - libmysqlclient-dev @@ -57,83 +57,43 @@ # Required for Ansible to setup DB - python3-mysqldb - - name: Setup dev deployment - block: - - name: Check if running under Vagrant - stat: - path: /vagrant - register: vagrant_dir - - - name: Clone / update from local repo - git: - repo: '/vagrant' - dest: '{{ project_dir }}' - when: vagrant_dir.stat.exists == True - - - name: Copy deploy key - copy: - src: 'deploy/.deployment-key' - dest: '~/.deployment-key' - mode: 0600 - when: vagrant_dir.stat.exists == False - - - name: Clone / update dev branch from main repo - git: - repo: 'ssh://git@github.com/PEDASI/PEDASI.git' - dest: '{{ project_dir }}' - accept_hostkey: yes - key_file: '~/.deployment-key' - version: dev - when: vagrant_dir.stat.exists == False - - - name: Copy dev settings - copy: - src: 'deploy/.env.dev' - dest: '{{ project_dir }}/.env' - owner: www-data - group: www-data - mode: 0600 - - when: production is not defined - - - name: Setup production deployment - block: - - name: Copy deploy key - copy: - src: 'deploy/.deployment-key' - dest: '~/.deployment-key' - mode: 0600 - - - name: Clone / update master branch - git: - repo: 'ssh://git@github.com/PEDASI/PEDASI.git' - dest: '{{ project_dir }}' - accept_hostkey: yes - key_file: '~/.deployment-key' - version: master - - - name: Copy production settings - copy: - src: 'deploy/.env.prod' - dest: '{{ project_dir }}/.env' - owner: www-data - group: www-data - mode: 0600 - - when: production is defined + - name: Check if running under Vagrant + stat: + path: /vagrant + register: vagrant_dir + + - name: Clone / update from local repo + git: + repo: '/vagrant' + dest: '{{ project_dir }}' + when: vagrant_dir.stat.exists == True + + - name: Clone / update branch from main repo + git: + repo: 'https://github.com/PEDASI/PEDASI.git' + dest: '{{ project_dir }}' + accept_hostkey: yes + version: '{{ branch | default("dev") }}' + when: vagrant_dir.stat.exists == False + + - name: Copy settings + copy: + src: '{{ env_file | default("deploy/.env") }}' + dest: '{{ project_dir }}/.env' + owner: www-data + group: www-data + mode: 0600 - name: Set permissions on manage.py file: path: '{{ project_dir }}/manage.py' mode: 0755 - - name: Create virtualenv - command: python3 -m venv '{{ venv_dir }}' creates='{{ venv_dir }}' - - name: Install pip requirements pip: requirements: '{{ project_dir}}/requirements.txt' virtualenv: '{{ venv_dir }}' + virtualenv_python: python3 - name: Restart and enable MariaDB systemd: @@ -195,23 +155,13 @@ - { src: '{{ project_dir }}/deploy/uwsgi/sites/pedasi.ini', dest: /etc/uwsgi/sites/pedasi.ini } - { src: '{{ project_dir }}/deploy/systemd/system/uwsgi.service', dest: /etc/systemd/system/uwsgi.service } - - name: Copy web config files - dev - copy: - src: '{{ project_dir }}/deploy/nginx/sites-available/pedasi.dev' - dest: /etc/nginx/sites-available/pedasi - remote_src: yes - # WARNING: this will not update an existing file - force: no - when: production is not defined - - - name: Copy web config files - production + - name: Copy web config files copy: src: '{{ project_dir }}/deploy/nginx/sites-available/pedasi' dest: /etc/nginx/sites-available/pedasi remote_src: yes # WARNING: this will not update an existing file force: no - when: production is defined - name: Activate Nginx site file: @@ -279,7 +229,6 @@ SPHINXAPIDOC: '{{ venv_dir }}/bin/sphinx-apidoc' loop: - clean - - apidoc - html - name: Django collect static files diff --git a/requirements.txt b/requirements.txt index 5ec9ca35cae6d7137f67d446713fd0a74e470810..baab955031f0d1fb0acb9a90179b7db8a63dfccd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -29,6 +29,7 @@ mysqlclient==1.3.13 networkx==2.2 oauthlib==2.1.0 packaging==17.1 +pbr==5.1.2 prov==1.5.2 pycparser==2.18 Pygments==2.2.0 @@ -50,6 +51,8 @@ snowballstemmer==1.2.1 social-auth-app-django==3.0.0 social-auth-core==2.0.0 Sphinx==1.7.6 +sphinx-rtd-theme==0.4.2 +sphinxcontrib-apidoc==0.3.0 sphinxcontrib-websupport==1.1.0 SQLAlchemy==1.2.14 typed-ast==1.1.0