diff --git a/.gitignore b/.gitignore
index a3b5a62c365b2eaddbcdec14aad1e4caf1d2a157..b50e1bd11ef62055b7e96a44a09fd0edefce9891 100644
--- a/.gitignore
+++ b/.gitignore
@@ -14,6 +14,12 @@ db.sqlite3
*.pyc
__pycache__/
/report.html
+*.log
+
+# Javascript environment
+node_modules/
+package.json
+package-lock.json
# Editor settings
.idea/
diff --git a/LICENSE b/LICENCE.md
similarity index 100%
rename from LICENSE
rename to LICENCE.md
diff --git a/README.md b/README.md
index 642e0fd4b7c13c340aae01a159e8b3d7b358cd26..9cbc6680f8c14e8b6bbc7f2c0482dbaf82e86171 100644
--- a/README.md
+++ b/README.md
@@ -1,43 +1,35 @@
-# PEDASI
+# PEDASI v0.1.0
-PEDASI is an Internet of Things (IoT) Observatory demonstrator platform.
+Developed as a platform and a service to explore research challenges in data security, privacy, and ethics, PEDASI enables providers of data - particularly [Internet of Things](https://en.wikipedia.org/wiki/Internet_of_things) data - to share their data securely within a common catalogue for use by application developers and researchers. Data can either be hosted and made accessible directly within PEDASI as an internal data source, or hosted elsewhere and accessible as an external data source through PEDASI.
-It functions as middleware between data sources and applications to provide a testbed for investigation of
-research questions in the domain of IoT.
+An initial deployment of the platform is available at https://dev.iotobservatory.io.
+## Key Features
-## Running / Deploying PEDASI
+PEDASI’s key features are:
-### In Development
-To run PEDASI locally during development you will need to install and run:
+ - Searchable catalogue of supported data sources registered by data owners
+ - Extensible connector interface that currently supports HyperCat and IoTUK Nation Database data sources
+ - Dataset discovery and access via a web interface or via an Applications API
+ - Queryable and extensible metadata associated with datasets
+ - Adoption of W3C PROV-DM specification to track and record dataset creation, update, and access within internal datastore
+ - Internally hosted support for read/write NoSQL datastores
+ - Functions as a reverse proxy to data sources, returning data from requests exactly as supplied by the data source
-* MySQL server
- * Create a schema to hold the PEDASI core database tables
-* MongoDB server
- * Create a database to hold the PEDASI PROV collections
+## Release Notes
-It is recommended that you then create a Python virtual environment in which to run PEDASI.
-The Python requirements are described in the `requirements.txt` file.
+This is a public alpha release, and therefore features and functionality may change and the software and documentation may contain technical bugs or other issues. If you discover any issues please consider registering a [GitHub issue](https://github.com/PEDASI/PEDASI/issues).
-PEDASI can be run using the Django development server by `python manage.py runserver`.
+## Documentation
-### In Production
+Documentation is available on [readthedocs](https://pedasi.readthedocs.io/en/master/) for users, system administrators, data and application providers, and application developers, and is also installed within a PEDASI deployment (e.g. at https://dev.iotobservatory.io/static/html).
-This repository contains an Ansible `playbook.yml` file which will perform a full install of PEDASI onto a
-clean host running Ubuntu 18.04 LTS, or update an existing PEDASI instance if it was previously deployed with the same script.
+## Contact Information
-To deploy using production settings you must:
-* Create an Ansible inventory file and set `production=True` for the machine you wish to deploy to
-* Create an SSH key and register it as a deployment key on the PEDASI GitHub project
- * Move the SSH private key file to `deploy/.deployment-key`
-* Create a configuration file (see below) `deploy/.env.prod`
-* Run the Ansible deployment script `ansible-playbook -v -i inventory.yml playbook.yml -u <remote_username>`
+ - Project team: Adrian Cox (a.j.cox@soton.ac.uk), Mark Schueler (m.schueler@soton.ac.uk)
+ - Development team: James Graham (j.graham@soton.ac.uk), Steve Crouch (s.crouch@ecs.soton.ac.uk)
+## Licence
-## Configuring PEDASI
-Both PEDASI and Django are able to be configured via a `.env` file in the project root.
+PEDASI is provided under the MIT licence - see the [LICENCE.md](LICENCE.md) file for details.
-The only required configuration property is the Django SECRET_KEY which should be a randomly generated
-character sequence.
-
-Other configuration properties are described at the top of `pedasi/settings.py`.
diff --git a/api/tests.py b/api/tests.py
index 29c3688fb587fcbb4eb3cc85a690d1558f81de0b..483cf28300763493a739c594f28f991616e4f38a 100644
--- a/api/tests.py
+++ b/api/tests.py
@@ -6,7 +6,7 @@ from django.test import Client, TestCase
from rest_framework.authtoken.models import Token
from rest_framework.test import APIClient
-from datasources import models
+from datasources import connectors, models
class RootApiTest(TestCase):
@@ -482,8 +482,8 @@ class DataSourceApiIoTUKTest(TestCase):
class DataSourceApiHyperCatTest(TestCase):
test_name = 'HyperCat'
plugin_name = 'HyperCat'
- test_url = 'https://api.cityverve.org.uk/v1/cat/polling-station'
- dataset = 'https://api.cityverve.org.uk/v1/entity/polling-station/5'
+ test_url = 'https://api.cityverve.org.uk/v1/cat'
+ dataset = 'https://api.cityverve.org.uk/v1/cat/polling-station'
@classmethod
def setUpTestData(cls):
@@ -499,7 +499,7 @@ class DataSourceApiHyperCatTest(TestCase):
url=cls.test_url,
api_key=cls.api_key,
plugin_name=cls.plugin_name,
- auth_method=models.DataSource.determine_auth_method(cls.test_url, cls.api_key)
+ auth_method=connectors.BaseDataConnector.determine_auth_method(cls.test_url, cls.api_key)
)
def setUp(self):
diff --git a/api/views/datasources.py b/api/views/datasources.py
index 0061d50c7344ccdc9d266e68bb47023ebc58bf7f..09557ee5280111665b9d82b336a3191c4d903459 100644
--- a/api/views/datasources.py
+++ b/api/views/datasources.py
@@ -20,28 +20,28 @@ class DataSourceApiViewset(viewsets.ReadOnlyModelViewSet):
Provides views for:
/api/datasources/
- List all :class:`DataSource`s
+ List all :class:`datasources.models.DataSource`\ s.
/api/datasources/<int>/
- Retrieve a single :class:`DataSource`
+ Retrieve a single :class:`datasources.models.DataSource`.
/api/datasources/<int>/prov/
- Retrieve PROV records related to a :class:`DataSource`
+ Retrieve PROV records related to a :class:`datasources.models.DataSource`.
/api/datasources/<int>/metadata/
- Retrieve :class:`DataSource` metadata via API call to data source URL
+ Retrieve :class:`datasources.models.DataSource` metadata via API call to data source URL.
/api/datasources/<int>/data/
- Retrieve :class:`DataSource` data via API call to data source URL
+ Retrieve :class:`datasources.models.DataSource` data via API call to data source URL.
/api/datasources/<int>/datasets/
- Retrieve :class:`DataSource` list of data sets via API call to data source URL
+ Retrieve :class:`datasources.models.DataSource` list of data sets via API call to data source URL.
/api/datasources/<int>/datasets/<href>/metadata/
- Retrieve :class:`DataSource` metadata for a single dataset via API call to data source URL
+ Retrieve :class:`datasources.models.DataSource` metadata for a single dataset via API call to data source URL.
/api/datasources/<int>/datasets/<href>/metadata/
- Retrieve :class:`DataSource` data for a single dataset via API call to data source URL
+ Retrieve :class:`datasources.models.DataSource` data for a single dataset via API call to data source URL.
"""
queryset = models.DataSource.objects.all()
serializer_class = serializers.DataSourceSerializer
diff --git a/applications/admin.py b/applications/admin.py
index 9f0ea3ec04cf84fbc0d8355c69300cada23996a3..107e3f2c4e8ff4af5d2d66f4a1b7e0e4b620366b 100644
--- a/applications/admin.py
+++ b/applications/admin.py
@@ -14,7 +14,7 @@ class ApplicationAdmin(admin.ModelAdmin):
permission = super().has_change_permission(request, obj)
if obj is not None:
- permission &= obj.owner == request.user
+ permission &= (obj.owner == request.user) or request.user.is_superuser
return permission
@@ -25,7 +25,7 @@ class ApplicationAdmin(admin.ModelAdmin):
permission = super().has_delete_permission(request, obj)
if obj is not None:
- permission &= obj.owner == request.user
+ permission &= (obj.owner == request.user) or request.user.is_superuser
return permission
diff --git a/applications/models.py b/applications/models.py
index 8d1b36f1b6164689154979751cb49e50ab9e9a64..fa32ec28188fb954d46b70325d62bfbb3d8c6d37 100644
--- a/applications/models.py
+++ b/applications/models.py
@@ -1,3 +1,5 @@
+from uuid import uuid4
+
from django.contrib.auth import get_user_model
from django.contrib.auth.models import Group
from django.conf import settings
@@ -8,9 +10,10 @@ from django.utils.text import slugify
from rest_framework.authtoken.models import Token
from core.models import BaseAppDataModel, SoftDeletionManager
+from provenance.models import ProvAbleModel, ProvApplicationModel
-class Application(BaseAppDataModel):
+class Application(ProvAbleModel, ProvApplicationModel, BaseAppDataModel):
"""
Manage the state of and access to an external application.
@@ -126,11 +129,12 @@ class Application(BaseAppDataModel):
if self.proxy_user:
return self.proxy_user
- proxy_username = 'application-proxy-' + slugify(self.name)
+ # Add random UUID to username to allow multiple applications with the same name
+ proxy_username = 'application-proxy-' + slugify(self.name) + str(uuid4())
proxy_user = get_user_model().objects.create_user(proxy_username)
# Create an API access token for the proxy user
- Token.objects.create(user=proxy_user)
+ proxy_user.create_auth_token()
return proxy_user
diff --git a/applications/templates/applications/application/detail.html b/applications/templates/applications/application/detail.html
index 0fbb2f8e611251755affcdc606791b825d690244..37a95c90d09d0ad111a93e0909f5fa4c8beb7651 100644
--- a/applications/templates/applications/application/detail.html
+++ b/applications/templates/applications/application/detail.html
@@ -1,6 +1,10 @@
{% extends "base.html" %}
{% load bootstrap4 %}
+{% block extra_head %}
+ <script src="https://cdn.jsdelivr.net/npm/js-cookie@2.2.0/src/js.cookie.min.js"></script>
+{% endblock %}
+
{% block content %}
<nav aria-label="breadcrumb">
<ol class="breadcrumb">
@@ -56,28 +60,76 @@
</tr>
<tr>
<td>URL</td>
- <td>{{ application.url }}</td>
+ <td>{{ application.url|default:'No URL provided' }}</td>
</tr>
- {% if api_key %}
+ {% if has_edit_permission %}
<tr>
<td>API Key</td>
- <td>{{ api_key }}</td>
+ <td>
+ <script type="application/javascript">
+ function getToken() {
+ $.ajax({
+ dataType: "json",
+ url: "{% url 'applications:token' pk=application.pk %}",
+ data: null,
+ success: function (data) {
+ $('#spanApiToken').text(data.data.token.key);
+
+ document.getElementById("spanApiTokenPresent").style.display = "inline";
+ document.getElementById("spanApiTokenAbsent").style.display = "none";
+ }
+ });
+ }
+
+ function revokeToken() {
+ $.ajax({
+ dataType: "json",
+ url: "{% url 'applications:token' pk=application.pk %}",
+ method: "DELETE",
+ headers: {
+ "X-CSRFToken": Cookies.get("csrftoken")
+ },
+ data: null,
+ success: function (data) {
+ $('#spanApiToken').text("");
+
+ document.getElementById("spanApiTokenPresent").style.display = "none";
+ document.getElementById("spanApiTokenAbsent").style.display = "inline";
+ }
+ });
+ }
+ </script>
+
+ <span id="spanApiTokenPresent" {% if not application.proxy_user.auth_token %}style="display: none;"{% endif %}>
+ <span id="spanApiToken">
+ {{ application.proxy_user.auth_token }}
+ </span>
+
+ <button onclick="revokeToken();" class="btn btn-danger" role="button">Revoke API Token</button>
+ </span>
+
+ <span id="spanApiTokenAbsent" {% if application.proxy_user.auth_token %}style="display: none;"{% endif %}>
+ <button onclick="getToken();" class="btn btn-default" role="button">Generate API Token</button>
+ </span>
+ </td>
</tr>
{% endif %}
</tbody>
</table>
- <div class="row justify-content-center pt-5">
- <div class="col-4">
- <script type="application/javascript">
- function launchApp() {
- const win = window.open("{{ application.url }}", "_blank");
- win.focus();
- }
- </script>
+ {% if application.url %}
+ <div class="row justify-content-center pt-5">
+ <div class="col-4">
+ <script type="application/javascript">
+ function launchApp() {
+ const win = window.open("{{ application.url }}", "_blank");
+ win.focus();
+ }
+ </script>
- <button role="button" onclick="launchApp();" class="btn btn-info btn-lg btn-block">Launch App</button>
+ <button role="button" onclick="launchApp();" class="btn btn-info btn-lg btn-block">Launch App</button>
+ </div>
</div>
- </div>
+ {% endif %}
{% endblock %}
\ No newline at end of file
diff --git a/applications/urls.py b/applications/urls.py
index 1007f30ae0e53ddfa5607a16e98ac58e2de83276..731701cb8c9d654fd7bbc4b44d75644b800e095a 100644
--- a/applications/urls.py
+++ b/applications/urls.py
@@ -25,6 +25,10 @@ urlpatterns = [
views.ApplicationDeleteView.as_view(),
name='application.delete'),
+ path('<int:pk>/token',
+ views.ApplicationManageTokenView.as_view(),
+ name='token'),
+
path('<int:pk>/manage-access',
views.ApplicationManageAccessView.as_view(),
name='application.manage-access'),
diff --git a/applications/views.py b/applications/views.py
index 77dc4788c3a5cc04af7d50b6d29a5e6b3e5f04c3..046a64136ede982dc98da345540d286688af30ee 100644
--- a/applications/views.py
+++ b/applications/views.py
@@ -1,4 +1,9 @@
+"""
+Views to manage and access :class:`Application`s.
+"""
+
from django.contrib.auth.mixins import PermissionRequiredMixin
+from django.http import JsonResponse
from django.urls import reverse_lazy
from django.views.generic.detail import DetailView
from django.views.generic.edit import CreateView, DeleteView, UpdateView
@@ -7,8 +12,8 @@ from django.views.generic.list import ListView
from rest_framework.authtoken.models import Token
from . import models
-from core.permissions import OwnerPermissionMixin
from core.views import ManageAccessView
+from profiles.permissions import OwnerPermissionMixin
class ApplicationListView(ListView):
@@ -41,6 +46,7 @@ class ApplicationUpdateView(OwnerPermissionMixin, UpdateView):
context_object_name = 'application'
fields = '__all__'
+ permission_required = 'applications.change_application'
class ApplicationDeleteView(OwnerPermissionMixin, DeleteView):
@@ -48,6 +54,7 @@ class ApplicationDeleteView(OwnerPermissionMixin, DeleteView):
template_name = 'applications/application/delete.html'
context_object_name = 'application'
+ permission_required = 'applications.delete_application'
success_url = reverse_lazy('applications:application.list')
@@ -67,7 +74,11 @@ class ApplicationDetailView(DetailView):
context['has_edit_permission'] = self.request.user.is_superuser or self.request.user == self.object.owner
if self.request.user == self.object.owner or self.request.user.is_superuser:
- context['api_key'] = Token.objects.get(user=self.object.proxy_user)
+ try:
+ context['api_key'] = self.object.proxy_user.auth_token
+
+ except Token.DoesNotExist:
+ pass
return context
@@ -83,3 +94,41 @@ class ApplicationManageAccessView(OwnerPermissionMixin, ManageAccessView):
model = models.Application
template_name = 'applications/application/manage_access.html'
context_object_name = 'application'
+
+
+class ApplicationManageTokenView(OwnerPermissionMixin, DetailView):
+ """
+ Manage an API Token for an application.
+ """
+ model = models.Application
+
+ def render_to_response(self, context, **response_kwargs):
+ """
+ Get an existing API Token or create a new one for the requested :class:`Application`.
+
+ :return: JSON containing Token key
+ """
+ api_token, created = Token.objects.get_or_create(user=self.object.proxy_user)
+
+ return JsonResponse({
+ 'status': 'success',
+ 'data': {
+ 'token': {
+ 'key': api_token.key
+ }
+ }
+ })
+
+ def delete(self, request, *args, **kwargs):
+ """
+ Revoke an API Token for the requested :class:`Application`.
+ """
+ self.object = self.get_object()
+ self.object.proxy_user.revoke_auth_token()
+
+ return JsonResponse({
+ 'status': 'success',
+ 'data': {
+ 'token': None,
+ }
+ })
diff --git a/core/permissions.py b/core/permissions.py
deleted file mode 100644
index ddd75cb0fb2d056faef6e45de3525a501b4b2876..0000000000000000000000000000000000000000
--- a/core/permissions.py
+++ /dev/null
@@ -1,6 +0,0 @@
-from django.contrib.auth.mixins import UserPassesTestMixin
-
-
-class OwnerPermissionMixin(UserPassesTestMixin):
- def test_func(self):
- return self.request.user == self.get_object().owner or self.request.user.is_superuser
diff --git a/datasources/admin.py b/datasources/admin.py
index f98162f754fafea55bc3d7fccae6ec20095a71e6..c2419c0629b2ba98784c3ecfee0fcd8309da7548 100644
--- a/datasources/admin.py
+++ b/datasources/admin.py
@@ -20,7 +20,7 @@ class DataSourceAdmin(admin.ModelAdmin):
permission = super().has_change_permission(request, obj)
if obj is not None:
- permission &= obj.owner == request.user
+ permission &= (obj.owner == request.user) or request.user.is_superuser
return permission
@@ -31,7 +31,7 @@ class DataSourceAdmin(admin.ModelAdmin):
permission = super().has_delete_permission(request, obj)
if obj is not None:
- permission &= obj.owner == request.user
+ permission &= (obj.owner == request.user) or request.user.is_superuser
return permission
diff --git a/datasources/apps.py b/datasources/apps.py
index 5b1910ab51717e5e2128e8317808630b66a9f232..66bee37e60b07c876716be7aa10d7e215f0e0c89 100644
--- a/datasources/apps.py
+++ b/datasources/apps.py
@@ -1,5 +1,26 @@
+import logging
+
from django.apps import AppConfig
+from django.db.utils import OperationalError, ProgrammingError
+
+
+logger = logging.getLogger(__name__)
class DatasourcesConfig(AppConfig):
name = 'datasources'
+
+ @staticmethod
+ def create_operational_metadata():
+ from datasources.models import MetadataField
+
+ MetadataField.load_inline_fixtures()
+
+ def ready(self):
+ # Runs after app registry is populated - i.e. all models exist and are importable
+ try:
+ self.create_operational_metadata()
+ logging.info('Loaded inline MetadataField fixtures')
+
+ except (OperationalError, ProgrammingError):
+ logging.warning('Could not create MetadataField fixtures, database has not been initialized')
diff --git a/datasources/connectors/base.py b/datasources/connectors/base.py
index 38589688d1e97427b2f5252e9433d73e998c18af..9ce115877702be2cacf0ee82f654472a61718e6a 100644
--- a/datasources/connectors/base.py
+++ b/datasources/connectors/base.py
@@ -100,6 +100,41 @@ class BaseDataConnector(metaclass=plugin.Plugin):
def request_count(self):
return self._request_counter.count()
+ # TODO make normal method
+ @staticmethod
+ def determine_auth_method(url: str, api_key: str) -> AuthMethod:
+ """
+ Determine which authentication method to use to access the data source.
+
+ Test each known authentication method in turn until one succeeds.
+
+ :param url: URL to authenticate against
+ :param api_key: API key to use for authentication
+ :return: First successful authentication method
+ """
+ # If not using an API key - can't require auth
+ if not api_key:
+ return AuthMethod.NONE
+
+ for auth_method_id, auth_function in REQUEST_AUTH_FUNCTIONS.items():
+ try:
+ # Can we get a response using this auth method?
+ if auth_function is None:
+ response = requests.get(url)
+
+ else:
+ response = requests.get(url,
+ auth=auth_function(api_key, ''))
+
+ response.raise_for_status()
+ return auth_method_id
+
+ except requests.exceptions.HTTPError:
+ pass
+
+ # None of the attempted authentication methods was successful
+ raise requests.exceptions.ConnectionError('Could not authenticate against external API')
+
def get_metadata(self,
params: typing.Optional[typing.Mapping[str, str]] = None):
"""
diff --git a/datasources/connectors/csv.py b/datasources/connectors/csv.py
index 2aa9cb2a1c61ed7afd1b168975b1b5b0f9770104..f218b07fbb2b9ceda64e991e6b05c4cd8f1a795d 100644
--- a/datasources/connectors/csv.py
+++ b/datasources/connectors/csv.py
@@ -3,7 +3,6 @@ Connectors for handling CSV data.
"""
import csv
-import json
import typing
from django.http import JsonResponse
@@ -162,16 +161,18 @@ class CsvToMongoConnector(InternalDataConnector, DataSetConnector):
params = {key: _type_convert(val) for key, val in params.items()}
with context_managers.switch_collection(CsvRow, self.location) as collection:
- records = collection.objects.filter(**params)
+ records = collection.objects.filter(**params).exclude('_id')
- # To get dictionary from MongoEngine records we need to go via JSON string
- data = json.loads(records.exclude('_id').to_json())
+ data = list(records.as_pymongo())
# Couldn't store field 'id' in document - recover it
for item in data:
- if self.id_field_alias in item:
+ try:
item['id'] = item.pop(self.id_field_alias)
+ except KeyError:
+ pass
+
return JsonResponse({
'status': 'success',
'data': data,
diff --git a/datasources/forms.py b/datasources/forms.py
index a5477b241e523ab4c1b3376129adff893de15278..e3678de99976f8f6f5c6ad0184d4db9b95e8058d 100644
--- a/datasources/forms.py
+++ b/datasources/forms.py
@@ -24,7 +24,8 @@ class DataSourceForm(forms.ModelForm):
cleaned_data = super().clean()
try:
- cleaned_data['auth_method'] = models.DataSource.determine_auth_method(
+ # TODO construct and actual data connector instance here
+ cleaned_data['auth_method'] = connectors.BaseDataConnector.determine_auth_method(
cleaned_data['url'],
cleaned_data['api_key']
)
diff --git a/datasources/migrations/0031_default_connector_name.py b/datasources/migrations/0031_default_connector_name.py
new file mode 100644
index 0000000000000000000000000000000000000000..13f8306323927ebacafe513a098c2a4acef3edca
--- /dev/null
+++ b/datasources/migrations/0031_default_connector_name.py
@@ -0,0 +1,18 @@
+# Generated by Django 2.0.8 on 2019-02-19 10:27
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('datasources', '0030_rename_licence'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='datasource',
+ name='plugin_name',
+ field=models.CharField(default='DataSetConnector', max_length=63),
+ ),
+ ]
diff --git a/datasources/models.py b/datasources/models.py
index c95627b513885a0ce2149e2e135bac3c5d871c4e..e7c1b70fcfbba5e81ebb70ab6a16b56fd0097c4d 100644
--- a/datasources/models.py
+++ b/datasources/models.py
@@ -1,18 +1,20 @@
+"""
+This module contains the Django models necessary to manage the set of data sources.
+"""
+
import contextlib
import enum
import json
import typing
from django.conf import settings
-from django.contrib.auth.models import Group
from django.core import validators
from django.db import models
from django.urls import reverse
-import requests
-import requests.exceptions
from core.models import BaseAppDataModel, MAX_LENGTH_API_KEY, MAX_LENGTH_NAME, MAX_LENGTH_PATH, SoftDeletionManager
from datasources.connectors.base import AuthMethod, BaseDataConnector, REQUEST_AUTH_FUNCTIONS
+from provenance.models import ProvAbleModel
#: Length of request reason field - must include brief description of project
MAX_LENGTH_REASON = 511
@@ -75,18 +77,19 @@ class MetadataField(models.Model):
blank=False, null=False)
#: Short text identifier for the field
- short_name = models.CharField(max_length=MAX_LENGTH_NAME,
- validators=[
- validators.RegexValidator(
- '^[a-zA-Z][a-zA-Z0-9_]*\Z',
- 'Short name must begin with a letter and consist only of letters, numbers and underscores.',
- 'invalid'
- )
- ],
- unique=True,
- blank=False, null=False)
+ short_name = models.CharField(
+ max_length=MAX_LENGTH_NAME,
+ validators=[
+ validators.RegexValidator(
+ r'^[a-zA-Z][a-zA-Z0-9_]*\Z',
+ 'Short name must begin with a letter and consist only of letters, numbers and underscores.',
+ 'invalid'
+ )
+ ],
+ unique=True,
+ blank=False, null=False
+ )
- # TODO create all operational fields if missing
#: Does the field have an operational effect within PEDASI?
operational = models.BooleanField(default=False,
blank=False, null=False)
@@ -94,6 +97,26 @@ class MetadataField(models.Model):
def __str__(self):
return self.name
+ @classmethod
+ def load_inline_fixtures(cls):
+ """
+ Create any instances required for the functioning of PEDASI.
+
+ This is called from within the AppConfig.
+ """
+ fixtures = (
+ ('data_query_param', 'data_query_param', True),
+ ('indexed_field', 'indexed_field', True),
+ )
+
+ for name, short_name, operational in fixtures:
+ obj, created = cls.objects.get_or_create(
+ name=name,
+ short_name=short_name
+ )
+ obj.operational = operational
+ obj.save()
+
class MetadataItem(models.Model):
"""
@@ -183,7 +206,7 @@ class UserPermissionLink(models.Model):
unique_together = (('user', 'datasource'),)
-class DataSource(BaseAppDataModel):
+class DataSource(ProvAbleModel, BaseAppDataModel):
"""
Manage access to a data source API.
@@ -215,6 +238,7 @@ class DataSource(BaseAppDataModel):
#: Name of plugin which allows interaction with this data source
plugin_name = models.CharField(max_length=MAX_LENGTH_NAME,
+ default='DataSetConnector',
blank=False, null=False)
#: If the data source API requires an API key use this one
@@ -276,6 +300,12 @@ class DataSource(BaseAppDataModel):
super().__init__(*args, **kwargs)
self._data_connector = None
+ def save(self, *args, **kwargs):
+ # TODO avoid determining auth method if existing one still works
+ self.auth_method = self.data_connector_class.determine_auth_method(self.url, self.api_key)
+
+ super().save(*args, **kwargs)
+
def delete(self, using=None, keep_parents=False):
"""
Soft delete this object.
@@ -329,10 +359,18 @@ class DataSource(BaseAppDataModel):
@property
def is_catalogue(self) -> bool:
+ """
+ Is this data source a data catalogue?
+ """
return self.data_connector_class.is_catalogue
@property
def connector_string(self):
+ """
+ Get the string used to locate the resource associated with this data source.
+
+ e.g. URL, SQL table identifier, etc.
+ """
if self._connector_string:
return self._connector_string
return self.url
@@ -349,39 +387,51 @@ class DataSource(BaseAppDataModel):
try:
plugin = BaseDataConnector.get_plugin(self.plugin_name)
- except KeyError as e:
+ except KeyError as exc:
if not self.plugin_name:
- raise ValueError('Data source plugin is not set') from e
+ raise ValueError('Data source plugin is not set') from exc
- raise KeyError('Data source plugin not found') from e
+ raise KeyError('Data source plugin not found') from exc
return plugin
+ def _get_data_connector(self) -> BaseDataConnector:
+ """
+ Construct the data connector for this source.
+
+ :return: Data connector instance
+ """
+ plugin = self.data_connector_class
+
+ if not self.api_key:
+ data_connector = plugin(self.connector_string)
+
+ else:
+ # Is the authentication method set?
+ auth_method = AuthMethod(self.auth_method)
+ if auth_method == AuthMethod.UNKNOWN:
+ auth_method = plugin.determine_auth_method(self.url, self.api_key)
+
+ # Inject function to get authenticated request
+ auth_class = REQUEST_AUTH_FUNCTIONS[auth_method]
+
+ data_connector = plugin(self.connector_string, self.api_key,
+ auth=auth_class)
+
+ return data_connector
+
@property
@contextlib.contextmanager
def data_connector(self) -> BaseDataConnector:
"""
Context manager to construct the data connector for this source.
+ When the context manager is closed, the number of requests to the external API will be added to the total.
+
:return: Data connector instance
"""
if self._data_connector is None:
- plugin = self.data_connector_class
-
- if not self.api_key:
- self._data_connector = plugin(self.connector_string)
-
- else:
- # Is the authentication method set?
- auth_method = AuthMethod(self.auth_method)
- if not auth_method:
- auth_method = self.determine_auth_method(self.url, self.api_key)
-
- # Inject function to get authenticated request
- auth_class = REQUEST_AUTH_FUNCTIONS[auth_method]
-
- self._data_connector = plugin(self.connector_string, self.api_key,
- auth=auth_class)
+ self._data_connector = self._get_data_connector()
try:
# Returns as context manager
@@ -396,6 +446,11 @@ class DataSource(BaseAppDataModel):
@property
def search_representation(self) -> str:
+ """
+ Provide a text representation of this data source to be entered into a search index.
+
+ :return: Text representation of this data source
+ """
lines = [
self.name,
self.owner.get_full_name(),
@@ -403,12 +458,21 @@ class DataSource(BaseAppDataModel):
]
try:
+ # Using the data_connector context manager results in an infinite recursion:
+ # 1. Save data source
+ # 2. Get search representation (this function)
+ # 3. Close data connector context manager
+ # 4. Save data source -> ...
+
+ data_connector = self._get_data_connector()
+ metadata = data_connector.get_metadata()
+
lines.append(json.dumps(
- self.data_connector.get_metadata(),
+ metadata,
indent=4
))
- except:
+ except (KeyError, NotImplementedError, ValueError):
# KeyError: Plugin was not found
# NotImplementedError: Plugin does not support metadata
# ValueError: Plugin was not set
@@ -417,31 +481,6 @@ class DataSource(BaseAppDataModel):
result = '\n'.join(lines)
return result
- @staticmethod
- def determine_auth_method(url: str, api_key: str) -> AuthMethod:
- # If not using an API key - can't require auth
- if not api_key:
- return AuthMethod.NONE
-
- for auth_method_id, auth_function in REQUEST_AUTH_FUNCTIONS.items():
- try:
- # Can we get a response using this auth method?
- if auth_function is None:
- response = requests.get(url)
-
- else:
- response = requests.get(url,
- auth=auth_function(api_key, ''))
-
- response.raise_for_status()
- return auth_method_id
-
- except requests.exceptions.HTTPError:
- pass
-
- # None of the attempted authentication methods was successful
- raise requests.exceptions.ConnectionError('Could not authenticate against external API')
-
def get_absolute_url(self):
return reverse('datasources:datasource.detail',
kwargs={'pk': self.pk})
diff --git a/datasources/search_indexes.py b/datasources/search_indexes.py
index bff3740e06661e11ae3006ce3afe160ebf23ebcd..d24a4442cedfa5099038b58467ba3a1f581207a2 100644
--- a/datasources/search_indexes.py
+++ b/datasources/search_indexes.py
@@ -1,9 +1,22 @@
+"""
+This module contains the search index definitions for the datasource app using Haystack.
+
+See https://django-haystack.readthedocs.io/en/master/ for documentation.
+"""
+
from haystack import indexes
+
from . import models
class DataSourceIndex(indexes.SearchIndex, indexes.Indexable):
+ """
+ The search index definition for a DataSource.
+
+ Uses templates/search/indexes/datasources/datasource_text.txt and
+ :meth:`datasources.models.DataSource.search_representation`.
+ """
text = indexes.CharField(document=True, use_template=True)
def get_model(self):
diff --git a/datasources/templates/datasources/datasource/detail.html b/datasources/templates/datasources/datasource/detail.html
index 67152ed7bf0d97367a3d78e7d2d64636e39c01e5..d2a4c823b24511ab37f10a27af890603d72f5564 100644
--- a/datasources/templates/datasources/datasource/detail.html
+++ b/datasources/templates/datasources/datasource/detail.html
@@ -104,12 +104,16 @@
</td>
</tr>
<tr>
- <td>URL</td>
+ <td>Licence</td>
+ <td>{{ datasource.licence.name }}</td>
+ </tr>
+ <tr>
+ <td>External API URL</td>
<td>{{ datasource.url }}</td>
</tr>
<tr>
- <td>Licence</td>
- <td>{{ datasource.licence.name }}</td>
+ <td>Pedasi API URL</td>
+ <td>{{ api_url }}</td>
</tr>
</tbody>
</table>
diff --git a/datasources/templates/datasources/datasource/explorer.html b/datasources/templates/datasources/datasource/explorer.html
index dff92cde41b694e4db184265fc5f24fffcc0bde9..1f215b0da83fefd809a818e7f179afba92cc1bf2 100644
--- a/datasources/templates/datasources/datasource/explorer.html
+++ b/datasources/templates/datasources/datasource/explorer.html
@@ -156,7 +156,7 @@
</form>
<div class="alert alert-info w-100">
- Query URL: /api/datasources/{{ datasource.pk }}/<span id="datasetUrlSpan"></span>data/?<span id="queryParamSpan"></span>
+ Query URL: {{ api_url }}<span id="datasetUrlSpan"></span>data/?<span id="queryParamSpan"></span>
</div>
<table class="table" id="tableParams">
diff --git a/datasources/templates/datasources/datasource/manage_access.html b/datasources/templates/datasources/datasource/manage_access.html
index 11ce9bdce97f4cf53ac318d7aa07e6c5467234ee..014e49572c01d74888b46ab1054a46c84e5ae9cf 100644
--- a/datasources/templates/datasources/datasource/manage_access.html
+++ b/datasources/templates/datasources/datasource/manage_access.html
@@ -47,7 +47,13 @@
<tr id="requested-user-{{ permission.user.pk }}">
<td>
<p>
- {{ permission.user.username }}
+ {% if permission.user.application_proxy %}
+ {{ permission.user.application_proxy.name }}
+ <a href="{% url 'applications:application.detail' pk=permission.user.application_proxy.pk %}"
+ role="button" class="badge badge-info">App</a>
+ {% else %}
+ {{ permission.user.username }}
+ {% endif %}
</p>
{% if permission.reason %}
<div class="alert alert-secondary" role="note">
@@ -105,7 +111,15 @@
<tbody>
{% for permission in permissions_granted %}
<tr id="approved-user-{{ permission.user.pk }}">
- <td>{{ permission.user.username }}</td>
+ <td>
+ {% if permission.user.application_proxy %}
+ {{ permission.user.application_proxy.name }}
+ <a href="{% url 'applications:application.detail' pk=permission.user.application_proxy.pk %}"
+ role="button" class="badge badge-info">App</a>
+ {% else %}
+ {{ permission.user.username }}
+ {% endif %}
+ </td>
<td>{{ permission.get_requested_display }}</td>
<td>{{ permission.get_granted_display }}</td>
<td>
diff --git a/datasources/tests/test_connectors.py b/datasources/tests/test_connectors.py
index 8527559159265b2990a24ed6ad2120a6b88a9110..00296f0dfcb401edd16d66fa107b6e3654e44dda 100644
--- a/datasources/tests/test_connectors.py
+++ b/datasources/tests/test_connectors.py
@@ -1,6 +1,6 @@
from django.test import TestCase
-from datasources.connectors.base import BaseDataConnector
+from datasources.connectors.base import AuthMethod, BaseDataConnector
class ConnectorPluginTest(TestCase):
@@ -86,6 +86,13 @@ class ConnectorIoTUKTest(TestCase):
self.assertIn('data', result)
self.assertGreater(len(result['data']), 0)
+ def test_determine_auth(self):
+ connection = self._get_connection()
+
+ auth_method = connection.determine_auth_method(connection.location, connection.api_key)
+
+ self.assertEqual(AuthMethod.NONE, auth_method)
+
class ConnectorRestApiTest(TestCase):
url = 'https://api.iotuk.org.uk/'
diff --git a/datasources/tests/test_connectors_hypercat.py b/datasources/tests/test_connectors_hypercat.py
index a34bff6a05a8caaa7aeb4464f9f0cea4132332c7..8c25b63c3c20f540a2cb1075f30c5d0d81113d7d 100644
--- a/datasources/tests/test_connectors_hypercat.py
+++ b/datasources/tests/test_connectors_hypercat.py
@@ -2,9 +2,8 @@ import itertools
import typing
from django.test import TestCase
-from requests.auth import HTTPBasicAuth
-from datasources.connectors.base import BaseDataConnector, HttpHeaderAuth
+from datasources.connectors.base import AuthMethod, BaseDataConnector, HttpHeaderAuth
def _get_item_by_key_value(collection: typing.Iterable[typing.Mapping],
@@ -27,15 +26,15 @@ def _count_items_by_key_value(collection: typing.Iterable[typing.Mapping],
class ConnectorHyperCatTest(TestCase):
- url = 'https://portal.bt-hypercat.com/cat'
-
- # Met Office dataset for weather at Heathrow
- dataset = 'http://api.bt-hypercat.com/sensors/feeds/c7f361c6-7cb7-4ef5-aed9-397a0c0c4088'
+ # TODO find working dataset
+ url = 'https://api.cityverve.org.uk/v1/cat'
+ subcatalogue = 'https://api.cityverve.org.uk/v1/cat/polling-station'
+ dataset = 'https://api.cityverve.org.uk/v1/entity/polling-station/5'
def _get_connection(self) -> BaseDataConnector:
return self.plugin(self.url,
api_key=self.api_key,
- auth=HTTPBasicAuth)
+ auth=HttpHeaderAuth)
def setUp(self):
from decouple import config
@@ -43,7 +42,8 @@ class ConnectorHyperCatTest(TestCase):
BaseDataConnector.load_plugins('datasources/connectors')
self.plugin = BaseDataConnector.get_plugin('HyperCat')
- self.api_key = config('HYPERCAT_BT_API_KEY')
+ self.api_key = config('HYPERCAT_CISCO_API_KEY')
+ self.auth = None
def test_get_plugin(self):
self.assertIsNotNone(self.plugin)
@@ -58,7 +58,14 @@ class ConnectorHyperCatTest(TestCase):
self.assertTrue(connection.is_catalogue)
- def test_plugin_get_metadata(self):
+ def test_determine_auth(self):
+ connection = self._get_connection()
+
+ auth_method = connection.determine_auth_method(connection.location, connection.api_key)
+
+ self.assertEqual(AuthMethod.HEADER, auth_method)
+
+ def test_plugin_get_catalogue_metadata(self):
connection = self._get_connection()
result = connection.get_metadata()
@@ -67,15 +74,19 @@ class ConnectorHyperCatTest(TestCase):
for property in [
'urn:X-hypercat:rels:hasDescription:en',
'urn:X-hypercat:rels:isContentType',
+ 'urn:X-hypercat:rels:hasHomepage',
]:
self.assertIn(property, relations)
- self.assertEqual('BT Hypercat DataHub Catalog',
+ self.assertEqual('CityVerve Public API - master catalogue',
_get_item_by_key_value(result, 'rel', 'urn:X-hypercat:rels:hasDescription:en')['val'])
self.assertEqual('application/vnd.hypercat.catalogue+json',
_get_item_by_key_value(result, 'rel', 'urn:X-hypercat:rels:isContentType')['val'])
+ self.assertEqual('https://developer.cityverve.org.uk',
+ _get_item_by_key_value(result, 'rel', 'urn:X-hypercat:rels:hasHomepage')['val'])
+
def test_plugin_get_datasets(self):
connection = self._get_connection()
@@ -87,8 +98,16 @@ class ConnectorHyperCatTest(TestCase):
self.assertLessEqual(1,
len(datasets))
- self.assertIn(self.dataset,
- datasets)
+ # Only check a couple of expected results are present - there's too many to list here
+ expected = {
+ 'https://api.cityverve.org.uk/v1/cat/accident',
+ 'https://api.cityverve.org.uk/v1/cat/advertising-board',
+ 'https://api.cityverve.org.uk/v1/cat/advertising-post',
+ # And because later tests rely on it...
+ 'https://api.cityverve.org.uk/v1/cat/polling-station',
+ }
+ for exp in expected:
+ self.assertIn(exp, datasets)
def test_plugin_iter_datasets(self):
connection = self._get_connection()
@@ -127,7 +146,7 @@ class ConnectorHyperCatTest(TestCase):
This process is relatively slow so we only do a couple of iterations.
"""
- with self.plugin(self.url, api_key=self.api_key, auth=HTTPBasicAuth) as connection:
+ with self._get_connection() as connection:
for k, v in itertools.islice(connection.items(), 5):
self.assertEqual(str,
type(k))
@@ -138,124 +157,6 @@ class ConnectorHyperCatTest(TestCase):
self.assertEqual(k,
v.location)
- def test_plugin_get_dataset_metadata(self):
- connection = self._get_connection()
-
- result = connection[self.dataset].get_metadata()
-
- relations = [relation['rel'] for relation in result]
- for property in [
- 'urn:X-bt:rels:feedTitle',
- 'urn:X-hypercat:rels:hasDescription:en',
- 'urn:X-bt:rels:feedTag',
- 'urn:X-bt:rels:hasSensorStream',
- 'urn:X-hypercat:rels:isContentType',
- ]:
- self.assertIn(property, relations)
-
- self.assertIn('Met Office',
- _get_item_by_key_value(result, 'rel', 'urn:X-bt:rels:feedTitle')['val'])
-
- self.assertIn('Met Office',
- _get_item_by_key_value(result, 'rel', 'urn:X-hypercat:rels:hasDescription:en')['val'])
-
- self.assertEqual(1,
- _count_items_by_key_value(result, 'rel', 'urn:X-bt:rels:feedTag'))
-
- self.assertGreaterEqual(_count_items_by_key_value(result, 'rel', 'urn:X-bt:rels:hasSensorStream'),
- 1)
-
- def test_plugin_get_dataset_data(self):
- """
- Test that we can get data from a single dataset within the catalogue.
- """
- connection = self._get_connection()
-
- dataset = connection[self.dataset]
- result = dataset.get_data()
-
- self.assertIsInstance(result, str)
- self.assertGreaterEqual(len(result), 1)
- self.assertIn('c7f361c6-7cb7-4ef5-aed9-397a0c0c4088',
- result)
-
-
-class ConnectorHyperCatCiscoTest(TestCase):
- url = 'https://api.cityverve.org.uk/v1/cat'
- subcatalogue = 'https://api.cityverve.org.uk/v1/cat/polling-station'
- dataset = 'https://api.cityverve.org.uk/v1/entity/polling-station/5'
-
- def _get_connection(self) -> BaseDataConnector:
- return self.plugin(self.url,
- api_key=self.api_key,
- auth=HttpHeaderAuth)
-
- def setUp(self):
- from decouple import config
-
- BaseDataConnector.load_plugins('datasources/connectors')
- self.plugin = BaseDataConnector.get_plugin('HyperCat')
-
- self.api_key = config('HYPERCAT_CISCO_API_KEY')
- self.auth = None
-
- def test_get_plugin(self):
- self.assertIsNotNone(self.plugin)
-
- def test_plugin_init(self):
- connection = self._get_connection()
-
- self.assertEqual(connection.location, self.url)
-
- def test_plugin_type(self):
- connection = self._get_connection()
-
- self.assertTrue(connection.is_catalogue)
-
- def test_plugin_get_catalogue_metadata(self):
- connection = self._get_connection()
-
- result = connection.get_metadata()
-
- relations = [relation['rel'] for relation in result]
- for property in [
- 'urn:X-hypercat:rels:hasDescription:en',
- 'urn:X-hypercat:rels:isContentType',
- 'urn:X-hypercat:rels:hasHomepage',
- ]:
- self.assertIn(property, relations)
-
- self.assertEqual('CityVerve Public API - master catalogue',
- _get_item_by_key_value(result, 'rel', 'urn:X-hypercat:rels:hasDescription:en')['val'])
-
- self.assertEqual('application/vnd.hypercat.catalogue+json',
- _get_item_by_key_value(result, 'rel', 'urn:X-hypercat:rels:isContentType')['val'])
-
- self.assertEqual('https://developer.cityverve.org.uk',
- _get_item_by_key_value(result, 'rel', 'urn:X-hypercat:rels:hasHomepage')['val'])
-
- def test_plugin_get_datasets(self):
- connection = self._get_connection()
-
- datasets = connection.get_datasets()
-
- self.assertEqual(list,
- type(datasets))
-
- self.assertLessEqual(1,
- len(datasets))
-
- # Only check a couple of expected results are present - there's too many to list here
- expected = {
- 'https://api.cityverve.org.uk/v1/cat/accident',
- 'https://api.cityverve.org.uk/v1/cat/advertising-board',
- 'https://api.cityverve.org.uk/v1/cat/advertising-post',
- # And because later tests rely on it...
- 'https://api.cityverve.org.uk/v1/cat/polling-station',
- }
- for exp in expected:
- self.assertIn(exp, datasets)
-
def test_plugin_get_subcatalogue_metadata(self):
connection = self._get_connection()
diff --git a/datasources/views/datasource.py b/datasources/views/datasource.py
index 736d4ea966dcb290b236bf71845668872d1cc1bf..1ea6417e764043f439af276d276805c6458c6426 100644
--- a/datasources/views/datasource.py
+++ b/datasources/views/datasource.py
@@ -11,9 +11,9 @@ from rest_framework import serializers
from rest_framework.views import APIView
import requests.exceptions
-from core.permissions import OwnerPermissionMixin
from datasources import forms, models
from datasources.permissions import HasPermissionLevelMixin
+from profiles.permissions import OwnerPermissionMixin
class DataSourceListView(ListView):
@@ -44,6 +44,12 @@ class DataSourceDetailView(DetailView):
except (KeyError, ValueError):
messages.error(self.request, 'This data source is not configured correctly. Please notify the owner.')
+ context['api_url'] = (
+ 'https://' if self.request.is_secure() else 'http://' +
+ self.request.get_host() +
+ '/api/datasources/{0}/'.format(self.object.pk)
+ )
+
return context
@@ -71,6 +77,7 @@ class DataSourceUpdateView(OwnerPermissionMixin, UpdateView):
context_object_name = 'datasource'
form_class = forms.DataSourceForm
+ permission_required = 'datasources.change_datasource'
class DataSourceDeleteView(OwnerPermissionMixin, DeleteView):
@@ -78,6 +85,7 @@ class DataSourceDeleteView(OwnerPermissionMixin, DeleteView):
template_name = 'datasources/datasource/delete.html'
context_object_name = 'datasource'
+ permission_required = 'datasources.delete_datasource'
success_url = reverse_lazy('datasources:datasource.list')
@@ -198,4 +206,10 @@ class DataSourceExplorerView(HasPermissionLevelMixin, DetailView):
field__short_name='data_query_param'
).values_list('value', flat=True)
+ context['api_url'] = (
+ 'https://' if self.request.is_secure() else 'http://' +
+ self.request.get_host() +
+ '/api/datasources/{0}/'.format(self.object.pk)
+ )
+
return context
diff --git a/datasources/views/licence.py b/datasources/views/licence.py
index 4b90a3694fad9e37d22ee0241c19ee8b0e5448f9..078fd5535156dec41d93b911d919b34f1e2c5c11 100644
--- a/datasources/views/licence.py
+++ b/datasources/views/licence.py
@@ -3,7 +3,7 @@ from django.urls import reverse_lazy
from django.views.generic import CreateView, DeleteView, DetailView, ListView, UpdateView
from .. import forms, models
-from core.permissions import OwnerPermissionMixin
+from profiles.permissions import OwnerPermissionMixin
class LicenceListView(ListView):
diff --git a/datasources/views/user_permission_link.py b/datasources/views/user_permission_link.py
index f9d019c66f3b36f94f30e8a89f06b957e169828e..b7d4c4fddb023e68fc1f4c2572f99b2f3c002125 100644
--- a/datasources/views/user_permission_link.py
+++ b/datasources/views/user_permission_link.py
@@ -6,11 +6,11 @@ from django.shortcuts import reverse
from django.views.generic.detail import DetailView
from django.views.generic.edit import UpdateView
-from profiles.permissions import OwnerPermissionRequiredMixin
+from profiles.permissions import OwnerPermissionMixin
from datasources import forms, models
-class DataSourceAccessManageView(OwnerPermissionRequiredMixin, DetailView):
+class DataSourceAccessManageView(OwnerPermissionMixin, DetailView):
model = models.DataSource
template_name = 'datasources/datasource/manage_access.html'
context_object_name = 'datasource'
diff --git a/deploy/nginx/sites-available/pedasi b/deploy/nginx/sites-available/pedasi
index 6d405e53bf3b4e4866a20db25afd339d73de162f..399490107c8da3a36966a93c8b0439bab0f91bf1 100644
--- a/deploy/nginx/sites-available/pedasi
+++ b/deploy/nginx/sites-available/pedasi
@@ -1,8 +1,9 @@
server {
- listen 80;
- server_name localhost pedasi.* pedasi-dev.* *.iotobservatory.io;
+ listen 80 default_server;
+ server_name _;
merge_slashes off;
+ client_max_body_size 100m;
location /favicon.ico {
alias /var/www/pedasi/static/img/favicon.ico;
@@ -13,11 +14,15 @@ server {
}
location = /report.html {
- file /var/www/pedasi/report.html;
+ alias /var/www/pedasi/report.html;
auth_basic "Restricted Content";
auth_basic_user_file /etc/nginx/.htpasswd;
}
+ location = /robots.txt {
+ alias /var/www/pedasi/deploy/robots.txt;
+ }
+
location / {
include uwsgi_params;
uwsgi_pass unix:/run/uwsgi/pedasi.sock;
diff --git a/deploy/nginx/sites-available/pedasi.dev b/deploy/nginx/sites-available/pedasi.dev
deleted file mode 100644
index e7a65daa508ee3282fc157c7a3bd830886fecf74..0000000000000000000000000000000000000000
--- a/deploy/nginx/sites-available/pedasi.dev
+++ /dev/null
@@ -1,24 +0,0 @@
-server {
- listen 80;
-
- merge_slashes off;
-
- location = /favicon.ico {
- alias /var/www/pedasi/static/img/favicon.ico;
- }
-
- location /static/ {
- alias /var/www/pedasi/static/;
- }
-
- location = /report.html {
- alias /var/www/pedasi/report.html;
- auth_basic "Restricted Content";
- auth_basic_user_file /etc/nginx/.htpasswd;
- }
-
- location / {
- include uwsgi_params;
- uwsgi_pass unix:/run/uwsgi/pedasi.sock;
- }
-}
diff --git a/deploy/robots.txt b/deploy/robots.txt
new file mode 100644
index 0000000000000000000000000000000000000000..adc3595e0ed36ab274e1ea13c3a835751103faad
--- /dev/null
+++ b/deploy/robots.txt
@@ -0,0 +1,3 @@
+User-agent: *
+Disallow: /admin/
+Disallow: /api/
diff --git a/deploy/uwsgi/sites/pedasi.ini b/deploy/uwsgi/sites/pedasi.ini
index 48eec162e68090d77ad23ebe14e1ebd8c50daa5d..f98604ca7577a6548e4fc884ba726ffd52a3ec8d 100644
--- a/deploy/uwsgi/sites/pedasi.ini
+++ b/deploy/uwsgi/sites/pedasi.ini
@@ -9,7 +9,7 @@ module = %(project).wsgi:application
logto = %(chdir)/pedasi.log
master = true
-processes = 1
+processes = 2
socket = /run/uwsgi/%(project).sock
chown-socket = %(uid):www-data
diff --git a/docs/source/conf.py b/docs/source/conf.py
index 7d119329143489789d82950c0dca2a1ddfcdc5d7..fa9f14d95b108d46222ee50ff714791e9c7eefce 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -24,7 +24,7 @@ django.setup()
# -- Project information -----------------------------------------------------
project = 'PEDASI'
-copyright = '2018, James Graham, Steve Crouch'
+copyright = '2019 University of Southampton'
author = 'James Graham, Steve Crouch'
# The short X.Y version
@@ -43,13 +43,28 @@ release = '0.1.0'
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
+ 'sphinxcontrib.apidoc',
'sphinx.ext.autodoc',
- 'sphinx.ext.apidoc',
'sphinx.ext.todo',
'sphinx.ext.coverage',
'sphinx.ext.viewcode',
]
+# Sphinxcontrib-apidoc settings
+# Add apidoc stage to normal build process
+
+apidoc_module_dir = '../../'
+apidoc_output_dir = 'apidoc'
+apidoc_excluded_paths = [
+ '**/migrations',
+ '**/tests/',
+ '**/tests.py',
+ '**/urls.py',
+ 'manage.py',
+ 'pedasi/wsgi.py',
+]
+apidoc_separate_modules = True
+
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
@@ -83,7 +98,7 @@ pygments_style = 'sphinx'
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
#
-html_theme = 'alabaster'
+html_theme = 'sphinx_rtd_theme'
# Theme options are theme-specific and customize the look and feel of a theme
# further. For a list of options available for each theme, see the
@@ -106,6 +121,13 @@ html_static_path = ['_static']
#
# html_sidebars = {}
+# Add following as a header for each rst file
+rst_prolog = """
+.. note:: This is a public alpha release, and therefore features and functionality may change and the software and documentation may contain technical bugs or other issues. If you discover any issues please consider registering a `GitHub issue`_.
+
+.. _`GitHub issue`: https://github.com/PEDASI/PEDASI/issues
+"""
+
# -- Options for HTMLHelp output ---------------------------------------------
diff --git a/docs/source/guide_administrator.rst b/docs/source/guide_administrator.rst
new file mode 100644
index 0000000000000000000000000000000000000000..fdfab70500a87cc1224d48828cbfd13e36088872
--- /dev/null
+++ b/docs/source/guide_administrator.rst
@@ -0,0 +1,209 @@
+.. _guide_administrator:
+
+PEDASI System Administrator Guide
+=================================
+
+.. toctree::
+ :maxdepth: 2
+ :caption: Contents:
+
+
+.. note:: Please read the :doc:`User Guide<guide_user>` first to give you an overview of the PEDASI platform and how to use its features before reading this guide.
+
+
+Purpose
+-------
+
+This guide is for system administrators wishing to deploy a PEDASI instance, either for production or locally for development.
+
+
+Production Deployment
+---------------------
+
+Overview
+^^^^^^^^
+
+A deployment of PEDASI is done automatically to a remote Ubuntu server via a preconfigured Ansible script, which performs the following tasks:
+
+ 1. Install prerequisites
+ 2. Configure databases
+ 3. Install PEDASI
+ 4. Configure and start webserver
+
+See the *playbook.yml* Ansible file in the PEDASI repository's root directory for more details.
+
+
+Prerequisites
+^^^^^^^^^^^^^
+
+Ensure you have the following prerequisites before you begin:
+
+ - Ubuntu 18.04 LTS server with:
+
+ - The latest security updates
+ - A static IP address
+ - A user account with external SSH access enabled and with sudo access privileges (e.g. 'ubuntu' - ubuntu will be used throughout this documentation)
+
+ - A Linux or Mac OS X local machine with the following installed:
+
+ - Ansible v2.7.1 or above
+ - Git command line client v2 or above
+
+
+Cloning the PEDASI Repository
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+On your local machine, first clone the PEDASI repository:
+
+.. code-block:: console
+
+ $ git clone https://github.com/PEDASI/PEDASI.git
+
+
+Configuration
+^^^^^^^^^^^^^
+
+It it necessary to provide some configuration before deploying PEDASI.
+
+ 1. Tell Ansible to which machine PEDASI should be deployed:
+
+ .. code-block:: none
+ :caption: inventory.yml
+
+ [default]
+ hostname.domain
+
+ 2. Provide required configuration for Django - the required and optional settings are described in :mod:`pedasi.settings`.
+ The required settings are:
+
+ .. code-block:: none
+ :caption: deploy/.env
+
+ SECRET_KEY=<random string>
+
+ SOCIAL_AUTH_GOOGLE_OAUTH2_KEY=<Google OAuth2 key>
+ SOCIAL_AUTH_GOOGLE_OAUTH2_SECRET=<Google OAuth2 secret>
+
+
+Deployment
+^^^^^^^^^^
+
+You may now deploy PEDASI using the Ansible provisioning script. If you have set up your Ubuntu instance to use SSH passwordless access, do the following:
+
+.. code-block:: console
+
+ $ ansible-playbook -v -i inventory.yml playbook.yml -u <your username on the remote host>
+
+Otherwise, you will need Ansible to prompt for passwords for the remote user and superuser accounts:
+
+.. code-block:: console
+
+ $ ansible-playbook -v -i inventory.yml playbook.yml -u <your username on the remote host> -k -K
+
+
+Create Administrator Account
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+After deploying PEDASI you must create and activate an initial PEDASI Central Administrator account:
+
+.. code-block:: console
+
+ $ sudo -s
+ $ cd /var/www/pedasi
+ $ source env/bin/activate
+ $ python manage.py createsuperuser --username <username> --email <email address>
+
+Once created, you can access your deployment at https://<server_address>.
+
+
+Assigning Data or Application Provider Roles to Accounts
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+To add Data and/or Application Provider roles to an existing PEDASI User account:
+
+ 1. Ensure you are logged in as the PEDASI Central Administrator account
+ 2. Go to the administrator pages at https://<server_address>/admin
+ 3. Select *Users* from the *PROFILES* subsection to display a list of all system users
+ 4. Select the user account you wish to change, to edit that user's profile
+ 5. Under the *Groups* section under *Permissions*, select *Data Provider* or *Application Provider* and select the right arrow to add this role to the user's profile
+ 6. Select *SAVE* at the bottom of the page
+
+
+Development Deployment
+----------------------
+
+Overview
+^^^^^^^^
+
+A development instance of PEDASI can be automatically and rapidly deployed using Vagrant. It uses VirtualBox as a virtual machine (VM) management tool to provision a Vagrant-style VM and provisions a PEDASI instance within that VM.
+
+
+Prerequisites
+^^^^^^^^^^^^^
+
+Ensure you have the following prerequisites before you begin:
+
+ - A Linux or Mac OS X local machine with the following installed:
+
+ - Vagrant v2.2.0 or above
+ - VirtualBox v5.2.0 or above
+ - Git command line client v2.0 or above
+
+
+Cloning the PEDASI Repository
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+On your local machine, from a shell first clone the PEDASI repository:
+
+.. code-block:: console
+
+ $ git clone https://github.com/PEDASI/PEDASI.git
+
+
+Configuring Deployment
+^^^^^^^^^^^^^^^^^^^^^^
+
+First, check the settings in the *Vagrantfile* in the repository's root directory to ensure any provisioned VM will not conflict with any other resources running on the default ports.
+
+Then create a new *.env* file in the repository's *deploy/* directory with the following contents (replacing *some_test_key* with a string of your choice):
+
+::
+
+ SECRET_KEY=some_test_key
+ DEBUG=true
+
+
+Managing Deployment
+^^^^^^^^^^^^^^^^^^^
+
+To deploy the Vagrant instance, provision the instance, and deploy PEDASI within it, within the PEDASI repository root directory:
+
+.. code-block:: console
+
+ $ vagrant up
+
+The Vagrant instance will now be visible within VirtualBox, and the PEDASI service visible from a browser at http://localhost:8888/.
+
+To access the Vagrant instance from the command line:
+
+.. code-block:: console
+
+ $ vagrant ssh
+
+Please see the `Vagrant documentation`_ for more details on how to use Vagrant, including shutting down and destroying Vagrant instances.
+
+.. _`Vagrant documentation`: https://www.vagrantup.com/docs/
+
+
+Creating Administrator and Provider Accounts
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Follow the instructions in the *Create Administrator Account* subsection of the *Production Deployment* section above to set up an administrator user, as well as the *Assigning Data or Application Provider Roles to Accounts* subsection if required for other non-administrator users.
+
+
+References
+----------
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`
diff --git a/docs/source/guide_developer.rst b/docs/source/guide_developer.rst
new file mode 100644
index 0000000000000000000000000000000000000000..092db164cc4721b480ac51abf5a5bbb51bd56f3d
--- /dev/null
+++ b/docs/source/guide_developer.rst
@@ -0,0 +1,128 @@
+.. _guide_developer:
+
+PEDASI Developer Guide
+======================
+
+.. toctree::
+ :maxdepth: 2
+ :caption: Contents:
+
+
+.. note:: Please read the :doc:`User Guide<guide_user>` first to give you an overview of the PEDASI platform before reading this guide.
+
+
+Purpose
+-------
+
+This guide is for application developers wanting to understand how to set up a development environment for PEDASI and develop PEDASI applications.
+
+
+Developing an Application
+-------------------------
+
+The following sections will walk you through using the PEDASI Applications API in your own applications. We will use the JavaScript IoTUK Nation Map Demo application (see https://github.com/Southampton-RSG/app-iotorgs-map) as a basic example. These instructions assume you are aiming to develop against a production deployment of PEDASI managed buy a Central Administrator, but you can also use a local development deployment if you wish. This section assumes you already have a registered PEDASI account,
+
+
+Adding Required Data Sources
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Firstly, you'll need to ensure that the data sources your application aims to use are added to PEDASI, which require Data Provider account privileges. If you do not already have these privileges on your PEDASI account, either:
+
+ - Contact your local PEDASI Data Provider to register the data sources
+ - Contact the PEDASI Central Administrator to register the data sources
+ - Contact the PEDASI Central Administrator to request Data Provider privileges for your account
+
+For simplicity, this guide assumes that the data source is being added with a public level of access (i.e. at the DATA level), so that the application will not need to have its access level approved via a request.
+
+If you now have these privileges and wish to add the data sources yourself, follow the *Adding a Data Source* section in the :doc:`Data and Application Provider Guide<guide_provider>` (which also contains details for adding the IoTUK Nation Data as a data source).
+
+Once these data source(s) have been registered, note the API query URL for each of them that you intend to use in your application. For each data source, this can be obtained by visiting the *Detail* page for a data source from the data sources list, which you can find via the *Data Sources* link in the PEDASI top navigation bar.
+
+
+Obtaining an Application API Key
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+You will first need to register some basic application details to obtain an API key for your application to authenticate with PEDASI, which requires Application Provider account privileges. If you do not have these privileges on your PEDASI account, either:
+
+ - Contact your local PEDASI Application Provider to register the application and obtain an API key
+ - Contact the PEDASI Central Administrator to register the new application and obtain an API key
+ - Contact the PEDASI Central Administrator to request Application Provider privileges for your account
+
+If you now have these privileges and wish to add the application yourself, follow the *Adding an Application* section in the :doc:`Data and Application Provider Guide<guide_provider>` (which also contains details for adding the IoTUK Nation Map Demo application).
+
+
+Code Examples
+^^^^^^^^^^^^^
+
+Once you have the required data sources and application set up in PEDASI, you can begin using the API. For example, with the IoTUK Nations Database, assuming the API query URL for it is https://dev.iotobservatory.io/api/datasources/2/data.
+
+
+Using cURL
+~~~~~~~~~~
+
+An easy way to first test and see the output is using the command line tool *cURL* available under Linux (and installable on Mac OS X), which allows you to make requests from a RESTful API and see the results. Strictly speaking, since we're assuming it is using a publicly accessible data source, we do not need to supply any application API key. However, for completeness and for best practice, we can include it in the request:
+
+.. code-block:: console
+
+ $ curl -H "Authorization: Token <api_key>" "https://dev.iotobservatory.io/api/datasources/2/data/?town=Southampton"
+
+Then we should see something like the following (although without the formatting):
+
+.. code-block:: none
+
+ {
+ "results": 9,
+ "data": {
+ "1": {
+ "organisation_id": "2018_590",
+ "organisation_name": "2IC LIMITED",
+ "organisation_type": "ltd",
+ ...
+
+
+Using Python
+~~~~~~~~~~~~
+
+We can duplicate the cURL request above in Python 3. Creating a new Python file (e.g. *api-test.py*) with the following contents:
+
+.. code-block:: python
+
+ import sys
+ import requests
+
+ url = "https://dev.iotobservatory.io/api/datasources/2/data/?town=Southampton"
+ headers = {
+ 'Authorization': 'Token <api_key>'
+ }
+ response = requests.get(url, headers=headers)
+ print(response)
+
+We can run this example using:
+
+.. code-block:: console
+
+ $ python api-test.py
+
+Which should display the same output as we saw with cURL.
+
+
+Using JavaScript - a Basic Example Application
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+An `example`_ hosted on GitHub is a lightweight JavaScript PEDASI web application that uses the PEDASI Applications API - see the GitHub repository and its README for more details on how to deploy and use it.
+
+.. _`example`: https://github.com/PEDASI/app-iotorgs-map
+
+
+Application API Documentation
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The :doc:`Applications API Reference<ref_applications_api>` documentation covers the API and its endpoints, and how to use it.
+
+
+References
+----------
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`
diff --git a/docs/source/guide_provider.rst b/docs/source/guide_provider.rst
new file mode 100644
index 0000000000000000000000000000000000000000..27b56d20abf0b067297cf02cb06f9c001067dfba
--- /dev/null
+++ b/docs/source/guide_provider.rst
@@ -0,0 +1,178 @@
+.. _guide_provider:
+
+PEDASI Data And Application Provider Guide
+==========================================
+
+.. toctree::
+ :maxdepth: 2
+ :caption: Contents:
+
+
+.. note:: Please read the :doc:`User Guide<guide_user>` first to give you an overview of the PEDASI platform and how to use its features before reading this guide.
+
+
+Purpose
+-------
+
+This guide is for Data or Application Providers wanting to add, update, or remove data sources or applications within a PEDASI instance.
+
+
+Data Providers: Managing Data Sources
+-------------------------------------
+
+In order for users to begin using PEDASI, you should provide access to a range of data sources. The following sections will walk you through adding and managing your first data source. We will use the IoTUK Nation Database API (see https://iotuk.org.uk/iotuk-nation-database-api/) as a basic example.
+
+If you are not a Central Administrator or don't have Data Provider privileges associated with your account, you'll need to obtain these first. Contact the Central Administrator to grant these privileges for your account.
+
+
+Adding a Data Source
+^^^^^^^^^^^^^^^^^^^^
+
+Before adding a new data source, first check if the licence type for that data source already exists. Select *Licences* from the navigation bar to display all current licences, and if the licence you need to use isn't already in the list, you'll need to add it as a new licence type. See *Adding a New Data Source Licence* below.
+
+To add a new data source:
+
+ 1. Select *Data Sources* from the PEDASI navigation bar to see a list of all data sources to which you have access
+ 2. Select *New Data Source* from the Data Sources page, and add in details for each of the following fields:
+
+ - *Name*: add in a unique name for this data source, such as 'UKIoT Nations'
+ - *Description*: optionally add in some specific details concerning this data source, such as its owner any links or references to any specifications or other documentation regarding the data source and format of the data that is delivered on request, e.g. some of the overview text at https://iotuk.org.uk/projects/iotuk-nation-database%E2%80%8B/ and a link to the page, and perhaps a link to the API details at https://github.com/TheDataCity/IoT-UK-Nation-Database-APIs. If the data source provides partially or fully encrypted data, also specify links to contact information and/or any reference material for obtaining a means to encrypt the data (not required for IoTUK Nations)
+ - *Url*: specify the base API URL for this data source, e.g. https://api.iotuk.org.uk/iotOrganisation.
+ - *Api key*: optionally specify an API key if one is required for PEDASI to access this resource
+ - *Plugin name*: select an appropriate data connector to interface with this API, e.g. DataSetConnector
+ - *Licence*: select an appropriate licence from those available in the dropdown list, e.g. Open Database Licence.
+ - *Is encrypted*: select this if the data supplied from the data source is partially or fully encrypted, e.g. leave unselected
+ - *Public permission level*: see the *Requesting Access to a Data Source* section in the :doc:`User Guide<guide_user>` for a breakdown of the different levels of access you can specify for a data source. e.g. DATA (which is the default)
+ - *Prov exempt*: select this if user activity tracking should be enabled for this data source, e.g. leave unselected
+
+ 3. Then select 'Create' to create this data source, and you'll be presented with an overview page for that data source.
+
+The next stage is to add in metadata for each of the parameters that can be used within the API. This step isn't mandatory, since arbitrary parameters can be specified via the Applications API or from within the Data Explorer, but is recommended. From the data source overview page, for each API query parameter (e.g. town, year, postcode for the IoTUK Nation Database data source):
+
+ 1. In the *Metadata* section, select *data_query_param* from the first dropdown
+ 2. Add the name of the API query field to the *Value* textbox, e.g. 'town'
+ 3. Select *Add* to add this query parameter
+
+Once complete, your data source is ready to use.
+
+
+Updating a Data Source
+^^^^^^^^^^^^^^^^^^^^^^
+
+To edit details for an existing data source:
+
+ 1. Select *Data Sources* from the navigation bar to see a list of all data sources to which you have access
+ 2. Select *Detail* for the data source you wish to edit
+ 3. Select *Edit* to edit the data source's details
+ 4. Edit the fields as instructed in the *Adding a Data Source* section
+ 5. Select *Update* to update the data source's details
+
+
+Removing a Data Source
+^^^^^^^^^^^^^^^^^^^^^^
+
+To remove an existing data source:
+
+ 1. Select *Data Sources* from the navigation bar to see a list of all data sources to which you have access
+ 2. Select *Detail* for the data source you wish to remove
+ 3. Select *Delete* to indicate you wish to remove the data source
+ 4. Select *Delete* to confirm you wish to remove this data source
+
+
+Adding a New Data Source Licence
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+If you need to add a new type of licence for a data source:
+
+ 1. Select *Licences* from the navigation bar to display all current licences
+ 2. Select *New License* to create a new licence
+ 3. Add detail for the following fields:
+
+ - *Name*: add a full name for a new licence, e.g. Open Database Licence
+ - *Short name*: add a short name for the licence, typically an acronym, e.g. ODbL
+ - *Version*: add the version number for the licence to use
+ - *URL*: add a link to an online resource describing the full terms of the licence
+
+ 4. Select 'Create' to create the licence, and you'll be presented with an overview page for that licence.
+
+
+Approving Data Access Requests
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+To approve user or application requests for amended access rights to data sources:
+
+ 1. Select *Data Sources* from the navigation bar to see a list of all data sources to which you have access
+ 2. Select *Detail* for the data source you wish to manage access on
+ 3. Select *Manage Access* to list access requests and manage those requests. You'll see the level of access requested for each user, their current access level, and the reason for the request
+ 4. Select either:
+
+ - *Approve*: to approve the request
+ - *Edit*: to amend the request's access privileges and data push rights (if data push is supported for this data source)
+ - *Reject*: to reject the request
+
+
+Application Providers: Managing Applications
+--------------------------------------------
+
+In order for a developer to access PEDASI's capabilities within their application, their application needs to be first registered within PEDASI in order to obtain an API key they can use to authenticate with PEDASI. The following sections will walk you through adding and managing your first application. We will use the IoTUK Nation Map Demo application (see https://github.com/Southampton-RSG/app-iotorgs-map) as a basic example.
+
+If you are not a Central Administrator or don't have Application Provider privileges associated with your account, you'll need to obtain these first. Contact the Central Administrator to grant these privileges for your account.
+
+
+Adding an Application
+^^^^^^^^^^^^^^^^^^^^^
+
+To add a new application:
+
+ 1. Select *Applications* from the PEDASI navigation bar to see a list of all applications to which you have access
+ 2. Select *New Application* from the Applications page, and add in details for each of the following fields:
+
+ - *Name*: add a full name for the application, e.g. IoTUK Nation Map Demo
+ - *Description*: add a brief description of the application, including what it aims to achieve using PEDASI
+ - *Url*: specify a public URL for the deployed application itself if it's web-based, or alternatively a source code repository URL if one exists, e.g. https://github.com/Southampton-RSG/app-iotorgs-map
+ - *Access control*: TODO: add in text here, e.g. leave unselected
+
+ 3. Select *Create* to register the new application within PEDASI, and you'll be presented with an overview page for that application, with a new API key
+
+The API key is what will be used by the developer to authenticate with PEDASI from their application, with the application acting as a user within the system.
+
+
+Updating an Application
+^^^^^^^^^^^^^^^^^^^^^^^
+
+ 1. Select *Applications* from the PEDASI navigation bar to see a list of all applications to which you have access
+ 2. Select *Detail* for the application you wish to edit. From here, you can also:
+
+ - Select *Revoke API Token*: to revoke the current API token which will prohibit its further use within PEDASI
+ - *If an API token has been revoked*, select *Generate API Token* to generate a new API token for this application
+
+ 3. Select *Edit* to edit the application's details
+ 4. Edit the fields as instructed in the *Adding an Application* section
+ 5. Select *Update* to update the data source's details
+
+
+Obtaining Access to Data Sources for Applications
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+As with users, applications also require access rights to data sources that are not public. By default, data sources are created with DATA access permissions and are considered public (awarding access to both data source metadata and data for all users, but not provenance records).
+
+If a data source has a lower level of access than required by the application, a request should be made from the Application Provider on behalf of their application for an appropriate level of access (typically DATA, the default). See *Requesting Access to a Data Source* in the :doc:`User Guide<guide_user>`.
+
+
+Removing an Application
+^^^^^^^^^^^^^^^^^^^^^^^
+
+To remove an application:
+
+ 1. Select *Applications* from the PEDASI navigation bar to see a list of all applications to which you have access
+ 2. Select *Detail* for the application you wish to remove
+ 3. Select *Delete* to indicate you wish to remove this application
+ 4. Select *Delete* to confirm you wish to remove this application
+
+
+References
+----------
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`
diff --git a/docs/source/guide_user.rst b/docs/source/guide_user.rst
new file mode 100644
index 0000000000000000000000000000000000000000..d9f2210223f44aded9ba5af303c139efa16c297b
--- /dev/null
+++ b/docs/source/guide_user.rst
@@ -0,0 +1,135 @@
+.. _guide_user:
+
+PEDASI User Guide
+=================
+
+.. toctree::
+ :maxdepth: 2
+ :caption: Contents:
+
+
+.. include:: release_note.rst
+
+
+Purpose
+-------
+
+This guide is for PEDASI users who want to understand the PEDASI platform and how to use its features.
+
+
+User Model
+----------
+
+There are four classes of users within PEDASI:
+
+ - *Basic User*: these are anonymous users, able to browse and search the catalogue's public data sources and associated metadata and retrieve public datasets from those data sources. They can also request a *PEDASI User* account. The data access activities of Basic Users are not tracked by PEDASI.
+
+ - *PEDASI User*: in addition to what a Basic User can do these users can also request access to specific data sources, which may be approved by *Data Providers*. These users are also *observed* within PEDASI, with data access activities tracked by the system (for those data sources that opt to track user activities).
+
+Another class of user is the *Provider*, used for administering PEDASI data sources and applications. These build on what PEDASI Users can do:
+
+ - *Data Provider*: these users may register new data sources within PEDASI, update data sources and their metadata, remove data sources, and approve data source access requests.
+
+ - *Application Provider*: similarly, these users may register new applications within PEDASI, update applications and their metadata, and remove applications.
+
+Applications developed for PEDASI also function as PEDASI Users, having their data access activities optionally tracked within the system, depending on the configuration of data sources.
+
+There is also a PEDASI *Central Administrator* role, able to approve PEDASI User account requests, and assign/remove Data and Application Provider roles to users.
+
+
+Obtaining a PEDASI Account
+--------------------------
+
+Why Register for an Account?
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Registering for an account enables you to be authenticated within the system as a PEDASI User and request access to private data sources and use them if approved.
+
+Data Source Activity Tracking
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+One of the research aims of PEDASI is to explore research challenges around linkages between data. As such, a PEDASI User's activities in relation to data sources is recorded within the system to assist this research, to determine usage and provenance relationships between datasets across the PEDASI user/application ecosystem, and help inform Data Providers how PEDASI uses their data sources.
+
+As an authenticated PEDASI User, the following data is recorded for each data access for those data sources that opt to record usage data:
+
+ - The date and time of access
+ - Your unique PEDASI account reference ID (not your personal Google account details)
+ - The type of activity (e.g. access, update, etc.)
+
+These records are held internally within PEDASI and are available to Central Administrators and explicitly approved users (e.g. Data Providers and researchers).
+
+
+Registering for an Account
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The preferred method to register an account with PEDASI is to register your Google account. If you don't have a Google account you can `sign up for one`_. Once you have an account you should contact the PEDASI Central Administrator via their contact details and request that your account be added to the system.
+
+Once approved, log in to the system as a PEDASI User using your Google account by selecting *Google Login* on the front page navigation bar.
+
+.. _`sign up for one`: https://accounts.google.com/signup
+
+
+Browsing the Data Catalogue
+---------------------------
+
+You can view the available data sources within PEDASI by selecting *Data Sources* from the front page navigation bar.
+
+
+Viewing Data Sources Details
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Selecting *Details* for a data source from the Data Sources page shows the following information:
+
+ - *Owner*: the Data Provider who administers this data source
+ - *URL*: a URL to the Application Programming Interface (API) for this data source
+ - *Licence*: the licence under which the data is provided. You must ensure you comply with these licensing conditions when using data provided by this data source
+ - *Metadata*: additional fields, listed by metadata type, for this data (e.g. data query parameters, etc.)
+
+
+Requesting Access to a Data Source
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+If you're logged in as a PEDASI User (and not anonymously), you may request access to a data source by selecting *Manage Access* from a data source overview page. Note that if you are also an Application Provider, you can also request access to data sources for your applications using this mechanism. From here you can supply the following details for an access request via a form:
+
+ - *User*: the identity of the user or application that is making this request (if making this request on behalf of an application, remember to set this to the identity of the application)
+ - *Requested*: The level of access, where a given level also provides access to previous levels:
+
+ - *NONE*: to revoke access to a data source
+ - *VIEW*: to view a data source's high-level details
+ - *META*: to view a data source's metadata
+ - *PROV*: to view a data source's PROV records
+
+ - *Push requested*: to allow the data source to receive updates and/or new data from the user or application, where the data source supports it (currently only internal PEDASI data sources)
+ - *Reason*: the reason for the request
+
+The data source's Data Provider will then consider and optionally approve the request. Subsequent requests can be made by the user or Application Provider to either escalate or de-escalate a user's or application's access level, each requiring approval.
+
+
+Using the Data Explorer
+^^^^^^^^^^^^^^^^^^^^^^^
+
+The Data Explorer allows you to explore and obtain data from data sources by building and submitting queries based on that data source's API parameters. This is also intended as an aid for developers who wish to understand and construct PEDASI queries within their applications.
+
+Selecting *Data Explorer* from the data source overview page shows an interface to do this using the following process:
+
+ 1. *Select a specific data set if supported*: firstly, if the data source supports multiple data sets, select one from the *Datasets* panel on the bottom right. Dataset-specific metadata will be displayed in the *Metadata* panel on the bottom left.
+
+ 2. *Use the Query Builder to construct a query*: in the *Query Builder* panel on the top left, select a query parameter from the dropdown selector (or if they are not configured for this data source, you can add one manually in the *Parameter* field), assign a value to that parameter, and select *Add to Query*. Repeat this as required to build a complete query. You'll be able to see the query that will be sent to PEDASI in the *Query URL* text box as the query is constructed.
+
+ 3. *Submit the query and see the results*: select *Submit Query* to submit the query, with the results displayed in the *Query Results* panel on the top right.
+
+Results are presented in the Query Results panel precisely as returned by the data source.
+
+
+Upgrading an Account to Data and/or Application Provider
+--------------------------------------------------------
+
+If you already have a PEDASI User account and would like to add your own data sources or applications to a PEDASI instance, contact the PEDASI site's Central Administrator to obtain these privileges for your account.
+
+
+References
+----------
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`
diff --git a/docs/source/index.rst b/docs/source/index.rst
index 54593fd058831f4bcec723c9f8238953649899ab..754277e6804b1ed50ebe32f08f4b3b9c61724e78 100644
--- a/docs/source/index.rst
+++ b/docs/source/index.rst
@@ -1,19 +1,60 @@
-.. PEDASI documentation master file, created by
- sphinx-quickstart on Fri Aug 17 13:18:08 2018.
- You can adapt this file completely to your liking, but it should at least
- contain the root `toctree` directive.
+.. _doc-index:
-Welcome to PEDASI's documentation!
-==================================
+PEDASI v0.1.0
+=============
.. toctree::
:maxdepth: 2
:caption: Contents:
+Overview
+--------
-Indices and tables
-==================
+Developed as a platform and a service to explore research challenges in data security, privacy, and ethics, PEDASI enables providers of data - particularly `Internet of Things`_ data - to share their data securely within a common catalogue for use by application developers and researchers. Data can either be hosted and made accessible directly within PEDASI as an internal data source, or hosted elsewhere and accessible as an external data source through PEDASI.
+
+An initial deployment of the platform is available at https://dev.iotobservatory.io.
+
+.. _`Internet of Things`: https://en.wikipedia.org/wiki/Internet_of_things
+
+
+Key Features
+------------
+
+PEDASI's key features are:
+
+ - Searchable catalogue of supported data sources registered by data owners
+ - Extensible connector interface that currently supports HyperCat and IoTUK Nation Database data sources
+ - Dataset discovery and access via a web interface or via an Applications API
+ - Queryable and extensible metadata associated with datasets
+ - Adoption of W3C PROV-DM specification to track and record dataset creation, update, and access within internal datastore
+ - Internally hosted support for read/write NoSQL datastores
+ - Functions as a reverse proxy to data sources, returning data from requests exactly as supplied by the data source
+
+
+Resources
+---------
+
+Documentation is available for the following stakeholders:
+
+ - :doc:`Researchers and other end-users<guide_user>`: for users wishing to discover, explore, and make use of datasets using the web interface
+ - :doc:`Data or Application Providers<guide_provider>`: for those aiming to provide data or use applications through PEDASI
+ - :doc:`System Administrators<guide_administrator>`: for those aiming to deploy and manage PEDASI either for development or production
+ - :doc:`Application developers<guide_developer>`: for developers wanting to create applications that access data available through PEDASI
+
+This documentation is also available on `Read the Docs`_.
+
+.. _`Read the Docs`: https://pedasi.readthedocs.io/en/dev
+
+
+Licence
+-------
+
+PEDASI is provided under the MIT licence.
+
+
+References
+----------
* :ref:`genindex`
* :ref:`modindex`
diff --git a/docs/source/ref_applications_api.rst b/docs/source/ref_applications_api.rst
new file mode 100644
index 0000000000000000000000000000000000000000..9b08a772b4a519b25b8c0d083fef0078d89ead95
--- /dev/null
+++ b/docs/source/ref_applications_api.rst
@@ -0,0 +1,101 @@
+.. _ref_applications_api:
+
+Applications API Reference
+==========================
+
+.. toctree::
+ :maxdepth: 2
+ :caption: Contents:
+
+
+.. note:: Please read the :doc:`Developer Guide<guide_developer>` first before reading this reference.
+
+
+Overview
+--------
+
+This document provides a schema reference to the PEDASI Applications API which is used by third-party applications to request data, metadata, or provenance records from a PEDASI instance and its data sources.
+
+
+Using the API
+-------------
+
+
+API Endpoints
+^^^^^^^^^^^^^
+
+
+GET /api/datasources/
+^^^^^^^^^^^^^^^^^^^^^
+
+Params:
+ TODO
+Retrieves the list of all data sources known to PEDASI, that the authenticated user has the ability to see. This will include some sources which they are unable to use, but have not had their details hidden.
+
+
+GET /api/datasources/<int>/
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Retrieves the PEDASI metadata for a given data source, if the authenticated user has the ability to see it.
+
+
+GET /api/datasources/<int>/metadata/
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Params:
+Any supported by data source API
+Retrieves metadata via an API query to a data source. The authenticated user must have permission to use the given data source.
+
+E.g. A HyperCat catalogue
+
+
+GET /api/datasources/<int>/metadata/<int>/
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+GET /api/datasources/<int>/metadata/<URI>/ (maybe)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Retrieves metadata for a single dataset contained within the source via an API query to the data source. The authenticated user must have permission to use the given data source.
+
+E.g. An entry within a HyperCat catalogue
+
+
+GET /api/datasources/<int>/data/
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Params:
+Any supported by data source API
+In the case where a data source represents a single dataset, retrieve the dataset via an API query to the data source. The authenticated user must have permission to use the given data source.
+
+If the data source does not represent a single dataset, error.
+
+
+GET /api/datasources/<int>/data/<int>/
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+GET /api/datasources/<int>/data/<URI>/ (maybe)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Params:
+In the case where a data source represents multiple datasets, retrieve a single dataset via an API query to the data source. The authenticated user must have permission to use the given data source.
+
+If the data source represents a single dataset, error.
+
+
+GET /api/datasources/<int>/prov/
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Params:
+TODO
+Retrieve all PROV records related to a single data source.
+
+
+GET /api/datasources/<int>/prov/<int>/
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Retrieve a single PROV record related to a the data source.
+
+
+References
+----------
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`
diff --git a/pedasi/common/__init__.py b/pedasi/common/__init__.py
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/pedasi/settings.py b/pedasi/settings.py
index 990ea0db8172b2ab1c2ff3b408188fd9b41369cd..c72f67d4d77dda75ca8191f42f61704e8cdb9d15 100644
--- a/pedasi/settings.py
+++ b/pedasi/settings.py
@@ -1,3 +1,4 @@
+# TODO describe all required / optional settings
"""
Django settings for the PEDASI project.
@@ -26,6 +27,9 @@ DEBUG
Run the server in debug mode?
Default is 'false'.
+ALLOWED_HOSTS - required if not in debug mode
+ List of hostnames on which the server is permitted to run
+
DATABASE_URL
URL to default SQL database - in `dj-database-url <https://github.com/kennethreitz/dj-database-url>`_ format.
Default is SQLite3 'db.sqlite3' in project root directory.
@@ -41,7 +45,7 @@ import os
from django.urls import reverse_lazy
-from decouple import config
+from decouple import config, Csv
import dj_database_url
import mongoengine
@@ -54,17 +58,9 @@ SECRET_KEY = config('SECRET_KEY')
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = config('DEBUG', default=False, cast=bool)
-if DEBUG:
- ALLOWED_HOSTS = [
- '*',
- ]
-
-else:
- ALLOWED_HOSTS = [
- 'localhost',
- 'pedasi-dev.eastus.cloudapp.azure.com',
- ]
-
+ALLOWED_HOSTS = config('ALLOWED_HOSTS',
+ cast=Csv(),
+ default='*' if DEBUG else '127.0.0.1,localhost.localdomain,localhost')
# Application definition
@@ -89,7 +85,7 @@ THIRD_PARTY_APPS = [
CUSTOM_APPS = [
'profiles.apps.ProfilesConfig', # Refer to AppConfig directly since we override the .ready() method
'applications',
- 'datasources',
+ 'datasources.apps.DatasourcesConfig',
'provenance',
'core',
'api',
@@ -137,7 +133,7 @@ WSGI_APPLICATION = 'pedasi.wsgi.application'
DATABASES = {
'default': config(
'DATABASE_URL',
- default='sqlite:///' + os.path.join(BASE_DIR, 'db.sqlite3'),
+ default='mysql://pedasi:pedasi@localhost:3306/pedasi',
cast=dj_database_url.parse
),
}
@@ -236,8 +232,8 @@ SOCIAL_AUTH_PIPELINE = [
# 'social_core.pipeline.social_auth.associate_by_email',
# Create a user account if we haven't found one yet.
- 'social_core.pipeline.user.create_user',
- # 'profiles.social_auth.create_user_disabled',
+ # 'social_core.pipeline.user.create_user',
+ 'profiles.social_auth.create_user_disabled',
# Create the record that associates the social account with the user.
'social_core.pipeline.social_auth.associate_user',
@@ -248,6 +244,9 @@ SOCIAL_AUTH_PIPELINE = [
# Update the user record with any changed info from the auth service.
'social_core.pipeline.user.user_details',
+
+ # Email admins to activate the account
+ 'profiles.social_auth.email_admins',
]
SOCIAL_AUTH_LOGIN_REDIRECT_URL = reverse_lazy('index')
@@ -315,3 +314,23 @@ STATICFILES_DIRS = [
os.path.join(BASE_DIR, 'pedasi', 'static'),
os.path.join(BASE_DIR, 'docs', 'build'),
]
+
+
+# Email provider for notification emails
+EMAIL_HOST = config('EMAIL_HOST', default=None)
+EMAIL_PORT = config('EMAIL_PORT', cast=int, default=25)
+
+EMAIL_HOST_USER = config('EMAIL_HOST_USER', default=None)
+EMAIL_HOST_PASSWORD = config('EMAIL_HOST_PASSWORD', default=None)
+DEFAULT_FROM_EMAIL = config('DEFAULT_FROM_EMAIL', default=EMAIL_HOST_USER)
+
+EMAIL_USE_TLS = config('EMAIL_USE_TLS', cast=bool, default=False)
+EMAIL_USE_SSL = config('EMAIL_USE_SSL', cast=bool, default=False)
+
+EMAIL_SUBJECT_PREFIX = '[PEDASI]'
+
+if DEBUG and not EMAIL_HOST:
+ EMAIL_BACKEND = 'django.core.mail.backends.filebased.EmailBackend'
+ EMAIL_FILE_PATH = os.path.join(BASE_DIR, 'mail')
+
+ADMINS = [val.split('|') for val in config('ADMINS', cast=Csv(), default='')]
diff --git a/playbook.yml b/playbook.yml
index 0d00be0307246ffc30f277e877f343dadb1d4d66..b3345535366f688603832879bc15d23a88d4becd 100644
--- a/playbook.yml
+++ b/playbook.yml
@@ -46,7 +46,7 @@
- python3
- python3-dev
- python3-pip
- - python3-venv
+ - python-virtualenv
- git
- mysql-server
- libmysqlclient-dev
@@ -57,83 +57,43 @@
# Required for Ansible to setup DB
- python3-mysqldb
- - name: Setup dev deployment
- block:
- - name: Check if running under Vagrant
- stat:
- path: /vagrant
- register: vagrant_dir
-
- - name: Clone / update from local repo
- git:
- repo: '/vagrant'
- dest: '{{ project_dir }}'
- when: vagrant_dir.stat.exists == True
-
- - name: Copy deploy key
- copy:
- src: 'deploy/.deployment-key'
- dest: '~/.deployment-key'
- mode: 0600
- when: vagrant_dir.stat.exists == False
-
- - name: Clone / update dev branch from main repo
- git:
- repo: 'ssh://git@github.com/Southampton-RSG/PEDASI-IoT.git'
- dest: '{{ project_dir }}'
- accept_hostkey: yes
- key_file: '~/.deployment-key'
- version: dev
- when: vagrant_dir.stat.exists == False
-
- - name: Copy dev settings
- copy:
- src: 'deploy/.env.dev'
- dest: '{{ project_dir }}/.env'
- owner: www-data
- group: www-data
- mode: 0600
-
- when: production is not defined
-
- - name: Setup production deployment
- block:
- - name: Copy deploy key
- copy:
- src: 'deploy/.deployment-key'
- dest: '~/.deployment-key'
- mode: 0600
-
- - name: Clone / update master branch
- git:
- repo: 'ssh://git@github.com/Southampton-RSG/PEDASI-IoT.git'
- dest: '{{ project_dir }}'
- accept_hostkey: yes
- key_file: '~/.deployment-key'
- version: master
-
- - name: Copy production settings
- copy:
- src: 'deploy/.env.prod'
- dest: '{{ project_dir }}/.env'
- owner: www-data
- group: www-data
- mode: 0600
-
- when: production is defined
+ - name: Check if running under Vagrant
+ stat:
+ path: /vagrant
+ register: vagrant_dir
+
+ - name: Clone / update from local repo
+ git:
+ repo: '/vagrant'
+ dest: '{{ project_dir }}'
+ when: vagrant_dir.stat.exists == True
+
+ - name: Clone / update branch from main repo
+ git:
+ repo: 'https://github.com/PEDASI/PEDASI.git'
+ dest: '{{ project_dir }}'
+ accept_hostkey: yes
+ version: '{{ branch | default("dev") }}'
+ when: vagrant_dir.stat.exists == False
+
+ - name: Copy settings
+ copy:
+ src: '{{ env_file | default("deploy/.env") }}'
+ dest: '{{ project_dir }}/.env'
+ owner: www-data
+ group: www-data
+ mode: 0600
- name: Set permissions on manage.py
file:
path: '{{ project_dir }}/manage.py'
mode: 0755
- - name: Create virtualenv
- command: python3 -m venv '{{ venv_dir }}' creates='{{ venv_dir }}'
-
- name: Install pip requirements
pip:
requirements: '{{ project_dir}}/requirements.txt'
virtualenv: '{{ venv_dir }}'
+ virtualenv_python: python3
- name: Restart and enable MariaDB
systemd:
@@ -195,23 +155,18 @@
- { src: '{{ project_dir }}/deploy/uwsgi/sites/pedasi.ini', dest: /etc/uwsgi/sites/pedasi.ini }
- { src: '{{ project_dir }}/deploy/systemd/system/uwsgi.service', dest: /etc/systemd/system/uwsgi.service }
- - name: Copy web config files - dev
- copy:
- src: '{{ project_dir }}/deploy/nginx/sites-available/pedasi.dev'
- dest: /etc/nginx/sites-available/pedasi
- remote_src: yes
- # WARNING: this will not update an existing file
- force: no
- when: production is not defined
+ - name: Deactivate default Nginx site
+ file:
+ path: /etc/nginx/sites-enabled/default
+ state: absent
- - name: Copy web config files - production
+ - name: Copy web config files
copy:
src: '{{ project_dir }}/deploy/nginx/sites-available/pedasi'
dest: /etc/nginx/sites-available/pedasi
remote_src: yes
# WARNING: this will not update an existing file
force: no
- when: production is defined
- name: Activate Nginx site
file:
@@ -279,7 +234,6 @@
SPHINXAPIDOC: '{{ venv_dir }}/bin/sphinx-apidoc'
loop:
- clean
- - apidoc
- html
- name: Django collect static files
diff --git a/profiles/apps.py b/profiles/apps.py
index 51cb0a78fe63eacbca8d06e30a61b06a8d1bae35..ff5f5fddf745fa314e4dbad55d529b1de6b31bd3 100644
--- a/profiles/apps.py
+++ b/profiles/apps.py
@@ -1,7 +1,7 @@
import logging
from django.apps import AppConfig
-from django.db.utils import ProgrammingError
+from django.db.utils import OperationalError, ProgrammingError
logger = logging.getLogger(__name__)
@@ -49,6 +49,7 @@ class ProfilesConfig(AppConfig):
# Runs after app registry is populated - i.e. all models exist and are importable
try:
self.create_groups()
+ logging.info('Loaded inline Group fixtures')
- except ProgrammingError:
+ except (OperationalError, ProgrammingError):
logging.warning('Could not create Group fixtures, database has not been initialized')
diff --git a/profiles/models.py b/profiles/models.py
index 9e97bb33adacd1128297d29bbe086de53769bce8..738fa71aaca9294d9327cb4476253935c4d89972 100644
--- a/profiles/models.py
+++ b/profiles/models.py
@@ -1,6 +1,11 @@
+"""
+Module containing models required for user profiles.
+"""
from django.contrib.auth.models import AbstractUser
from django.urls import reverse
+from rest_framework.authtoken.models import Token
+
class User(AbstractUser):
"""
@@ -14,3 +19,19 @@ class User(AbstractUser):
Used in PROV records.
"""
return reverse('profiles:uri', kwargs={'pk': self.pk})
+
+ def create_auth_token(self) -> Token:
+ """
+ Create an API auth token for this user.
+
+ :return: API auth token instance
+ """
+ token, created = Token.objects.get_or_create(user=self)
+ return token
+
+ def revoke_auth_token(self):
+ """
+ Revoke and API auth token for this user.
+ """
+ self.auth_token.delete()
+
diff --git a/profiles/permissions.py b/profiles/permissions.py
index dd58254727d8a73532135ff5277ff4ef25ae7224..ae136d39f06e519565322a5ebad7734ceb0b49fe 100644
--- a/profiles/permissions.py
+++ b/profiles/permissions.py
@@ -1,7 +1,22 @@
+"""
+Mixins for views to require certain permissions or ownership.
+"""
+
from django.contrib.auth.mixins import UserPassesTestMixin, PermissionRequiredMixin
-class OwnerPermissionRequiredMixin(PermissionRequiredMixin):
+class SelfOrAdminPermissionMixin(UserPassesTestMixin):
+ """
+ Mixin to require that a user is the linked object or an admin.
+
+ To be used e.g. for edit permission on user profiles
+ """
+ def test_func(self) -> bool:
+ user = self.get_object()
+ return user == self.request.user or self.request.user.is_superuser
+
+
+class OwnerPermissionMixin(PermissionRequiredMixin):
"""
Mixin to require that a user has the relevant global permission and is the owner of the relevant object.
diff --git a/profiles/social_auth.py b/profiles/social_auth.py
index b4d320fc414d66500d14aa5320443f2a3174c39c..453f75f90a4d408059b6f123550622842d68cb99 100644
--- a/profiles/social_auth.py
+++ b/profiles/social_auth.py
@@ -1,7 +1,21 @@
+"""
+Module containing customisations to the Python Social Auth pipeline.
+
+See https://python-social-auth.readthedocs.io/en/latest/
+"""
+
+from django.core.mail import mail_admins
+
+
from social_core.pipeline.user import create_user
def create_user_disabled(strategy, details, backend, user=None, *args, **kwargs):
+ """
+ Create a user account for the user being authenticated - but mark it as disabled.
+
+ A new user must have their account enabled by an admin before they are able to log in.
+ """
# Returns dict containing 'is_new' and 'user'
result = create_user(strategy, details, backend, user, *args, **kwargs)
@@ -11,3 +25,21 @@ def create_user_disabled(strategy, details, backend, user=None, *args, **kwargs)
django_user.save()
return result
+
+
+def email_admins(strategy, details, backend, user=None, *args, **kwargs):
+ """
+ Email the PEDASI admins if a new account has been created and requires approval
+ """
+ if kwargs['is_new']:
+ mail_admins(
+ subject='PEDASI Account Created',
+ message=(
+ 'New PEDASI user account: {0}\n\n'
+ 'A new user account has been created by {1} - {2} and requires admin approval'
+ ).format(
+ user.username,
+ user.get_full_name(),
+ user.email
+ )
+ )
diff --git a/profiles/templates/profiles/user/profile.html b/profiles/templates/profiles/user/profile.html
index 52af26156d30256b035cd272c4b05353c51917e5..76999de7163e1ae0c17c3b120d8a43a44274cd5a 100644
--- a/profiles/templates/profiles/user/profile.html
+++ b/profiles/templates/profiles/user/profile.html
@@ -1,6 +1,10 @@
{% extends "base.html" %}
{% load bootstrap4 %}
+{% block extra_head %}
+ <script src="https://cdn.jsdelivr.net/npm/js-cookie@2.2.0/src/js.cookie.min.js"></script>
+{% endblock %}
+
{% block content %}
<nav aria-label="breadcrumb">
<ol class="breadcrumb">
@@ -22,32 +26,58 @@
</thead>
<tbody>
- <tr>
- <td>API Token</td>
- <td>
- <span id="spanApiToken">
- {% if user == request.user and user.auth_token %}
- {{ user.auth_token }}
-
- {% else %}
+ {% if user == request.user or request.user.is_superuser %}
+ <tr>
+ <td>API Token</td>
+ <td>
<script type="application/javascript">
function getToken() {
$.ajax({
dataType: "json",
- url: "{% url 'profiles:token' %}",
+ url: "{% url 'profiles:token' pk=user.pk %}",
data: null,
success: function (data) {
$('#spanApiToken').text(data.data.token.key);
+
+ document.getElementById("spanApiTokenPresent").style.display = "inline";
+ document.getElementById("spanApiTokenAbsent").style.display = "none";
+ }
+ });
+ }
+
+ function revokeToken() {
+ $.ajax({
+ dataType: "json",
+ url: "{% url 'profiles:token' pk=user.pk %}",
+ method: "DELETE",
+ headers: {
+ "X-CSRFToken": Cookies.get("csrftoken")
+ },
+ data: null,
+ success: function (data) {
+ $('#spanApiToken').text("");
+
+ document.getElementById("spanApiTokenPresent").style.display = "none";
+ document.getElementById("spanApiTokenAbsent").style.display = "inline";
}
});
}
</script>
- <button onclick="getToken();" class="btn btn-default" role="button">Generate an API Token</button>
- {% endif %}
- </span>
- </td>
- </tr>
+ <span id="spanApiTokenPresent" {% if not user.auth_token %}style="display: none;"{% endif %}>
+ <span id="spanApiToken">
+ {{ user.auth_token }}
+ </span>
+
+ <button onclick="revokeToken();" class="btn btn-danger" role="button">Revoke API Token</button>
+ </span>
+
+ <span id="spanApiTokenAbsent" {% if user.auth_token %}style="display: none;"{% endif %}>
+ <button onclick="getToken();" class="btn btn-default" role="button">Generate API Token</button>
+ </span>
+ </td>
+ </tr>
+ {% endif %}
</tbody>
</table>
diff --git a/profiles/tests.py b/profiles/tests.py
index 75e3d2c60ce4d5b15446147a96409ce94083fc6a..1cc396ed84364e154c2c7126cfda4068db4545d7 100644
--- a/profiles/tests.py
+++ b/profiles/tests.py
@@ -44,7 +44,7 @@ class UserTest(TestCase):
# User should not already have token
token = Token.objects.get(user=user)
- response = client.get(reverse('profiles:token'))
+ response = client.get(reverse('profiles:token', kwargs={'pk': user.pk}))
self.assertEqual(200, response.status_code)
diff --git a/profiles/urls.py b/profiles/urls.py
index 1c71dc1268c0de04e2ea8bc075da22834d6a701f..d57bf5b141a9ef2eace08b2dffb7591b12e52b20 100644
--- a/profiles/urls.py
+++ b/profiles/urls.py
@@ -19,7 +19,7 @@ urlpatterns = [
views.UserUriView.as_view(),
name='uri'),
- path('token',
- views.UserGetTokenView.as_view(),
+ path('<int:pk>/token',
+ views.UserManageTokenView.as_view(),
name='token'),
]
diff --git a/profiles/views.py b/profiles/views.py
index d6cca5d14ec6f69bedf3d8bdf925e267c71e425f..7546d381dbc9b421cd438eb5e9b693cde5668833 100644
--- a/profiles/views.py
+++ b/profiles/views.py
@@ -1,5 +1,8 @@
+"""
+Views to manage user profiles.
+"""
+
from django.contrib.auth import get_user_model
-from django.contrib.auth.mixins import LoginRequiredMixin
from django.http import JsonResponse
from django.views.generic import TemplateView
from django.views.generic.detail import DetailView
@@ -8,6 +11,7 @@ from rest_framework.authtoken.models import Token
from applications.models import Application
from datasources.models import DataSource
+from profiles.permissions import SelfOrAdminPermissionMixin
class IndexView(TemplateView):
@@ -61,16 +65,15 @@ class UserInactiveView(TemplateView):
template_name = 'profiles/user/inactive.html'
-class UserGetTokenView(LoginRequiredMixin, DetailView):
+class UserManageTokenView(SelfOrAdminPermissionMixin, DetailView):
"""
- Get an API Token for the currently authenticated user.
+ Manage an API Token for the requested user.
"""
- def get_object(self, queryset=None):
- return self.request.user
+ model = get_user_model()
def render_to_response(self, context, **response_kwargs):
"""
- Get an existing API Token or create a new one for the currently authenticated user.
+ Get an existing API Token or create a new one for the requested user.
:return: JSON containing Token key
"""
@@ -84,3 +87,17 @@ class UserGetTokenView(LoginRequiredMixin, DetailView):
}
}
})
+
+ def delete(self, request, *args, **kwargs):
+ """
+ Revoke an API Token for the requested user.
+ """
+ self.object = self.get_object()
+ self.object.revoke_auth_token()
+
+ return JsonResponse({
+ 'status': 'success',
+ 'data': {
+ 'token': None,
+ }
+ })
diff --git a/provenance/models.py b/provenance/models.py
index 0716eac6023c74260a0aa2dbf288f9a98bb37722..df9bd295f11f361b817c8da83f5502f4b19bae7a 100644
--- a/provenance/models.py
+++ b/provenance/models.py
@@ -13,8 +13,7 @@ import uuid
from django import apps
from django.contrib.contenttypes.models import ContentType
-from django.db.models import QuerySet, signals
-from django.dispatch import receiver
+from django.db.models import QuerySet
from django.utils import timezone
from django.utils.text import slugify
@@ -22,27 +21,29 @@ import mongoengine
from mongoengine.queryset.visitor import Q
import prov.model
-from applications.models import Application
from core.models import BaseAppDataModel
-from datasources.models import DataSource
MAX_LENGTH_NAME_FIELD = 100
-class PedasiDummyApplication:
+class ProvApplicationModel:
"""
Dummy application model to fall back to when an action was performed via the PEDASI web interface.
+
+ Also to be used as parent class of :class:`Application` to help with type hinting.
"""
- name = 'PEDASI'
- pk = 'pedasi' # We convert pk to string anyway - so this is fine
+ def __init__(self, *args, **kwargs):
+ self.pk = kwargs.get('pk', 'pedasi')
+ self.name = kwargs.get('name', 'PEDASI')
+
+ super().__init__(*args, **kwargs)
- @staticmethod
- def get_absolute_url():
+ def get_absolute_url(self):
"""
Return the URL at which PEDASI is hosted.
"""
# TODO don't hardcode URL
- return 'http://www.pedasi-iot.org/'
+ return 'https://dev.iotobservatory.io/'
@enum.unique
@@ -66,7 +67,7 @@ class ProvEntry(mongoengine.DynamicDocument):
def create_prov(cls,
instance: BaseAppDataModel,
user_uri: str,
- application: typing.Optional[Application] = None,
+ application: typing.Optional[ProvApplicationModel] = None,
activity_type: typing.Optional[ProvActivity] = ProvActivity.UPDATE) -> 'ProvEntry':
"""
Build a PROV document representing a particular activity within PEDASI.
@@ -80,6 +81,7 @@ class ProvEntry(mongoengine.DynamicDocument):
instance_type = ContentType.objects.get_for_model(instance)
document = prov.model.ProvDocument(namespaces={
+ # TODO set PEDASI PROV namespace
'piot': 'http://www.pedasi-iot.org/',
'foaf': 'http://xmlns.com/foaf/0.1/',
'xsd': 'http://www.w3.org/2001/XMLSchema#',
@@ -107,7 +109,7 @@ class ProvEntry(mongoengine.DynamicDocument):
# Generate a UUID so we can lookup records belonging to a user
# But not identify the user from a given record
# TODO how strongly do we want to prevent user identification?
- # See https://github.com/Southampton-RSG/PEDASI-IoT/issues/10
+ # See https://github.com/PEDASI/PEDASI/issues/10
'piot:u-' + str(uuid.uuid5(uuid.NAMESPACE_URL, user_uri)),
other_attributes={
prov.model.PROV_TYPE: 'prov:Person',
@@ -115,7 +117,7 @@ class ProvEntry(mongoengine.DynamicDocument):
)
if application is None:
- application = PedasiDummyApplication
+ application = ProvApplicationModel()
agent_application = document.agent(
'piot:app-' + str(application.pk),
@@ -201,8 +203,8 @@ class ProvWrapper(mongoengine.Document):
"""
Get all :class:`ProvEntry` documents related to a particular Django model instance.
- :param instance: Model instance for which to get all :class:`ProvEntry`s
- :return: List of :class:`ProvEntry`s
+ :param instance: Model instance for which to get all :class:`ProvEntry`\ s
+ :return: List of :class:`ProvEntry`\ s
"""
instance_type = ContentType.objects.get_for_model(instance)
@@ -216,7 +218,7 @@ class ProvWrapper(mongoengine.Document):
def create_prov(cls,
instance: BaseAppDataModel,
user_uri: str,
- application: typing.Optional[Application] = None,
+ application: typing.Optional[ProvApplicationModel] = None,
activity_type: typing.Optional[ProvActivity] = ProvActivity.UPDATE) -> ProvEntry:
"""
Create a PROV record for a single action.
@@ -250,15 +252,34 @@ class ProvWrapper(mongoengine.Document):
super().delete(signal_kwargs, **write_concern)
-@receiver(signals.post_save, sender=Application)
-@receiver(signals.post_save, sender=DataSource)
-def save_prov(sender, instance, **kwargs):
+class ProvAbleModel:
"""
- Signal receiver to create a :class:`ProvEntry` when a PROV tracked model is saved.
+ Mixin for models which are capable of having updates tracked by PROV records.
+
+ Creates a new PROV record every time the object is modified and saved.
"""
- ProvWrapper.create_prov(
- instance,
- # TODO what if an admin edits a model?
- instance.owner.get_uri(),
- activity_type=ProvActivity.UPDATE
- )
+ def save(self, *args, **kwargs):
+ try:
+ # Have to read existing saved version from database
+ existing = type(self).objects.get(pk=self.pk)
+ changed = False
+
+ for field in self._meta.fields:
+ attr = field.attname
+
+ if getattr(existing, attr) != getattr(self, attr):
+ changed = True
+ break
+
+ except type(self).DoesNotExist:
+ # First time this object has been saved
+ changed = True
+
+ super().save(*args, **kwargs)
+
+ if changed:
+ ProvWrapper.create_prov(
+ self,
+ self.owner.get_uri(),
+ activity_type=ProvActivity.UPDATE
+ )
diff --git a/provenance/tests.py b/provenance/tests.py
index 21029c68d10595a4d0e53e999cfdc41aa6629a75..8ed6f443eb0ccb5001269e04b3883339c148a851 100644
--- a/provenance/tests.py
+++ b/provenance/tests.py
@@ -4,7 +4,6 @@ Tests for PROV tracking functionality and the models required to support it.
import json
import pathlib
-import unittest
from django.conf import settings
from django.contrib.auth import get_user_model
@@ -15,6 +14,7 @@ import jsonschema
import mongoengine
from mongoengine.queryset.visitor import Q
+from applications.models import Application
from datasources.models import DataSource
from provenance import models
@@ -39,7 +39,6 @@ class ProvEntryTest(TestCase):
name='Test Data Source',
url='http://www.example.com',
owner=self.user,
- plugin_name='TEST'
)
def tearDown(self):
@@ -99,7 +98,6 @@ class ProvWrapperTest(TestCase):
name='Test Data Source',
url='http://www.example.com',
owner=self.user,
- plugin_name='TEST'
)
def tearDown(self):
@@ -130,13 +128,12 @@ class ProvWrapperTest(TestCase):
"""
n_provs = self._count_prov(self.datasource)
- self.datasource.plugin_name = 'CHANGED'
+ self.datasource.api_key = 'TEST'
self.datasource.save()
# Another PROV record should be created when model is changed and saved
self.assertEqual(self._count_prov(self.datasource), n_provs + 1)
- @unittest.expectedFailure
def test_prov_datasource_null_update(self):
"""
Test that no new :class:`ProvEntry` is created when a model is saved without changes.
@@ -158,9 +155,91 @@ class ProvWrapperTest(TestCase):
name='Another Test Data Source',
url='http://www.example.com',
owner=self.user,
- plugin_name='TEST'
)
new_prov_entries = models.ProvWrapper.filter_model_instance(new_datasource)
intersection = set(prov_entries).intersection(new_prov_entries)
self.assertFalse(intersection)
+
+
+class ProvApplicationTest(TestCase):
+ """
+ Test the wrapper that allows us to look up :class:`ProvEntry`s for a given Application.
+ """
+ model = Application
+
+ @classmethod
+ def setUpTestData(cls):
+ cls.user_model = get_user_model()
+ cls.user = cls.user_model.objects.create_user('Test Prov User')
+
+ def setUp(self):
+ self.object = self.model.objects.create(
+ name='Test Object',
+ url='http://www.example.com',
+ owner=self.user,
+ )
+
+ def tearDown(self):
+ # Have to delete instance manually since we're not using Django's database manager
+ object_type = ContentType.objects.get_for_model(self.model)
+
+ models.ProvWrapper.objects(
+ Q(app_label=object_type.app_label) &
+ Q(model_name=object_type.model) &
+ Q(related_pk=self.object.pk)
+ ).delete()
+
+ @staticmethod
+ def _count_prov(obj) -> int:
+ """
+ Count PROV records for a given object.
+ """
+ prov_entries = models.ProvWrapper.filter_model_instance(obj)
+ return prov_entries.count()
+
+ def test_prov_application_create(self):
+ """
+ Test that a :class:`ProvEntry` is created when a model is created.
+ """
+ # PROV record should be created when model is created
+ self.assertEqual(self._count_prov(self.object), 1)
+
+ def test_prov_application_update(self):
+ """
+ Test that a new :class:`ProvEntry` is created when a model is updated.
+ """
+ n_provs = self._count_prov(self.object)
+
+ self.object.url = 'http://example.com'
+ self.object.save()
+
+ # Another PROV record should be created when model is changed and saved
+ self.assertEqual(self._count_prov(self.object), n_provs + 1)
+
+ def test_prov_application_null_update(self):
+ """
+ Test that no new :class:`ProvEntry` is created when a model is saved without changes.
+ """
+ n_provs = self._count_prov(self.object)
+
+ self.object.save()
+
+ # No PROV record should be created when saving a model that has not changed
+ self.assertEqual(self._count_prov(self.object), n_provs)
+
+ def test_prov_records_distinct(self):
+ """
+ Test that :class:`ProvEntry`s are not reused.
+ """
+ prov_entries = models.ProvWrapper.filter_model_instance(self.object)
+
+ new_object = self.model.objects.create(
+ name='Another Test Object',
+ url='http://www.example.com',
+ owner=self.user,
+ )
+ new_prov_entries = models.ProvWrapper.filter_model_instance(new_object)
+
+ intersection = set(prov_entries).intersection(new_prov_entries)
+ self.assertFalse(intersection)
diff --git a/requirements.txt b/requirements.txt
index 5ec9ca35cae6d7137f67d446713fd0a74e470810..baab955031f0d1fb0acb9a90179b7db8a63dfccd 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -29,6 +29,7 @@ mysqlclient==1.3.13
networkx==2.2
oauthlib==2.1.0
packaging==17.1
+pbr==5.1.2
prov==1.5.2
pycparser==2.18
Pygments==2.2.0
@@ -50,6 +51,8 @@ snowballstemmer==1.2.1
social-auth-app-django==3.0.0
social-auth-core==2.0.0
Sphinx==1.7.6
+sphinx-rtd-theme==0.4.2
+sphinxcontrib-apidoc==0.3.0
sphinxcontrib-websupport==1.1.0
SQLAlchemy==1.2.14
typed-ast==1.1.0