diff --git a/.gitignore b/.gitignore
index 72c9e73f1f170fe9163385cc9b354d8fe3f641f0..a3b5a62c365b2eaddbcdec14aad1e4caf1d2a157 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,6 +2,8 @@
.env
/deploy/.env*
/deploy/.deployment-key*
+/Makefile
+/inventory.yml
# Database
db.sqlite3
@@ -11,6 +13,7 @@ db.sqlite3
/venv
*.pyc
__pycache__/
+/report.html
# Editor settings
.idea/
@@ -20,6 +23,7 @@ __pycache__/
playbook.retry
ubuntu-bionic-18.04-cloudimg-console.log
/static/
+/whoosh_index/
# Documentation
/docs/build/
diff --git a/.pylintrc b/.pylintrc
new file mode 100644
index 0000000000000000000000000000000000000000..7edff46fb1a40f25b9a0b7ec2b8e5fc1646fcd89
--- /dev/null
+++ b/.pylintrc
@@ -0,0 +1,3 @@
+[FORMAT]
+
+max-line-length=120
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000000000000000000000000000000000000..a3595a90584187fd0d5f77cb6b7099e5c8967935
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 University of Southampton
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/README.md b/README.md
index d3f6e8061053db38e2f01bcb02d1857868ffad3e..642e0fd4b7c13c340aae01a159e8b3d7b358cd26 100644
--- a/README.md
+++ b/README.md
@@ -24,10 +24,14 @@ PEDASI can be run using the Django development server by `python manage.py runse
### In Production
This repository contains an Ansible `playbook.yml` file which will perform a full install of PEDASI onto a
-clean host, or update an existing PEDASI instance if it was previously deployed with the same script.
-
-To deploy using production settings you must create an Ansible inventory file and set `production=True` for
-the machine you wish to deploy to.
+clean host running Ubuntu 18.04 LTS, or update an existing PEDASI instance if it was previously deployed with the same script.
+
+To deploy using production settings you must:
+* Create an Ansible inventory file and set `production=True` for the machine you wish to deploy to
+* Create an SSH key and register it as a deployment key on the PEDASI GitHub project
+ * Move the SSH private key file to `deploy/.deployment-key`
+* Create a configuration file (see below) `deploy/.env.prod`
+* Run the Ansible deployment script `ansible-playbook -v -i inventory.yml playbook.yml -u <remote_username>`
## Configuring PEDASI
diff --git a/prov/__init__.py b/api/__init__.py
similarity index 100%
rename from prov/__init__.py
rename to api/__init__.py
diff --git a/api/apps.py b/api/apps.py
new file mode 100644
index 0000000000000000000000000000000000000000..d87006dd60ff626eefdf7f8ffaa3998d7f99615a
--- /dev/null
+++ b/api/apps.py
@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class ApiConfig(AppConfig):
+ name = 'api'
diff --git a/api/permissions.py b/api/permissions.py
new file mode 100644
index 0000000000000000000000000000000000000000..e296bf823cb0b21b2f05572b3cc9d04939f9df55
--- /dev/null
+++ b/api/permissions.py
@@ -0,0 +1,78 @@
+"""
+Permission check classes to be used with djangorestframework API.
+"""
+
+from rest_framework import permissions
+
+from datasources import models
+
+
+class BaseUserPermission(permissions.BasePermission):
+ """
+ Base permission check. Permissions should override the `permission_level` property.
+ """
+ message = 'You do not have permission to access this resource.'
+ permission_level = models.UserPermissionLevels.NONE
+
+ def has_object_permission(self, request, view, obj):
+ return obj.has_permission_level(request.user, self.permission_level)
+
+
+class ViewPermission(BaseUserPermission):
+ """
+ Assert that a user has the :class:`models.UserPermissionLevels.VIEW` permission.
+ """
+ message = 'You do not have permission to access this resource.'
+ permission_level = models.UserPermissionLevels.VIEW
+
+
+class MetadataPermission(BaseUserPermission):
+ """
+ Assert that a user has the :class:`models.UserPermissionLevels.META` permission.
+ """
+ message = 'You do not have permission to access the metadata of this resource.'
+ permission_level = models.UserPermissionLevels.META
+
+
+class DataPermission(BaseUserPermission):
+ """
+ Assert that a user has the :class:`models.UserPermissionLevels.DATA` permission.
+ """
+ message = 'You do not have permission to access the data of this resource.'
+ permission_level = models.UserPermissionLevels.DATA
+
+
+class ProvPermission(BaseUserPermission):
+ """
+ Assert that a user has the :class:`models.UserPermissionLevels.PROV` permission.
+ """
+ message = 'You do not have permission to access the prov data of this resource.'
+ permission_level = models.UserPermissionLevels.PROV
+
+
+class DataPushPermission(permissions.BasePermission):
+ """
+ Permission mixin to prevent access to POST and PUT methods by users who do not have the correct permission flag.
+ """
+ message = 'You do not have permission to push data to this resource.'
+
+ def has_object_permission(self, request, view, obj):
+ # Bypass if not pushing data
+ if request.method not in {'POST', 'PUT'}:
+ return True
+
+ # Owner always has permission
+ if request.user == obj.owner:
+ return True
+
+ try:
+ permission = models.UserPermissionLink.objects.get(
+ user=request.user,
+ datasource=obj
+ )
+
+ return permission.push_granted
+
+ except models.UserPermissionLink.DoesNotExist:
+ # Permission must have been granted explicitly
+ return False
diff --git a/api/tests.py b/api/tests.py
new file mode 100644
index 0000000000000000000000000000000000000000..29c3688fb587fcbb4eb3cc85a690d1558f81de0b
--- /dev/null
+++ b/api/tests.py
@@ -0,0 +1,585 @@
+import typing
+
+from django.contrib.auth import get_user_model
+from django.test import Client, TestCase
+
+from rest_framework.authtoken.models import Token
+from rest_framework.test import APIClient
+
+from datasources import models
+
+
+class RootApiTest(TestCase):
+ @classmethod
+ def setUpTestData(cls):
+ cls.user = get_user_model().objects.create_user('Test API User', password='Test API Password')
+ cls.token, created = Token.objects.get_or_create(user=cls.user)
+
+ def test_auth_rejected(self):
+ """
+ Test that we get an authentication failure if not providing a token.
+ """
+ client = APIClient()
+
+ response = client.get('/api/')
+
+ self.assertEqual(response.status_code, 401) # 401 Unauthorized
+
+ def test_force_auth(self):
+ """
+ Test simply that we can access the API using forced authentication.
+
+ This 'authentication' method is used for the API tests.
+ """
+ client = APIClient()
+ client.force_authenticate(self.user)
+
+ response = client.get('/api/')
+
+ self.assertEqual(response.status_code, 200)
+
+ def test_session_auth(self):
+ """
+ Test simply that we can access the API using session-based authentication.
+
+ This authentication method is used to access the API explorer within the PEDASI UI.
+ """
+ client = APIClient()
+ client.login(username='Test API User', password='Test API Password')
+
+ response = client.get('/api/')
+
+ self.assertEqual(response.status_code, 200)
+
+ def test_token_auth(self):
+ """
+ Test simply that we can access the API using token based authentication.
+
+ This authentication method is used to access the API from an external application.
+ """
+ client = APIClient()
+ client.credentials(HTTP_AUTHORIZATION='Token ' + self.token.key)
+
+ response = client.get('/api/')
+
+ self.assertEqual(response.status_code, 200)
+
+
+class DataSourceApiTest(TestCase):
+ @classmethod
+ def setUpTestData(cls):
+ cls.user = get_user_model().objects.create_user('Test API User')
+
+ def setUp(self):
+ self.client = APIClient()
+ self.client.force_authenticate(self.user)
+
+ self.test_name = 'Test DataSource'
+ self.test_url = 'https://example.com/test'
+
+ def tearDown(self):
+ try:
+ self.model.delete()
+ except AttributeError:
+ pass
+
+ def test_root_api_datasource_exists(self):
+ """
+ Test that a 'datasources' entry appears in the API root endpoint.
+ """
+ response = self.client.get('/api/')
+ self.assertEqual(response.status_code, 200)
+
+ self.assertIn('datasources', response.json().keys())
+
+ def _assert_datasource_correct(self, datasource: typing.Dict):
+ """
+ Helper function: assert that a :class:`DataSource` received via the API is correct.
+
+ :param datasource: :class:`DataSource` received via API
+ """
+ self.assertIn('name', datasource)
+ self.assertEqual(self.test_name, datasource['name'])
+
+ self.assertIn('description', datasource)
+ self.assertEqual('', datasource['description'])
+
+ self.assertIn('url', datasource)
+ self.assertEqual(self.test_url, datasource['url'])
+
+ def test_api_datasource_list(self):
+ """
+ Test the :class:`DataSource` API list functionality.
+ """
+ response = self.client.get('/api/datasources/')
+ self.assertEqual(response.status_code, 200)
+
+ self.assertEqual(len(response.json()), 0)
+
+ self.model = models.DataSource.objects.create(
+ name=self.test_name,
+ owner=self.user,
+ url=self.test_url
+ )
+
+ response = self.client.get('/api/datasources/')
+
+ self.assertEqual(len(response.json()), 1)
+
+ datasource = response.json()[0]
+ self._assert_datasource_correct(datasource)
+
+ def test_api_datasource_get(self):
+ """
+ Test the :class:`DataSource` API get one functionality.
+ """
+ response = self.client.get('/api/datasources/1/')
+ self.assertEqual(response.status_code, 404)
+
+ self.model = models.DataSource.objects.create(
+ name=self.test_name,
+ owner=self.user,
+ url=self.test_url
+ )
+
+ response = self.client.get('/api/datasources/{}/'.format(self.model.pk))
+ self.assertEqual(response.status_code, 200)
+
+ datasource = response.json()
+ self._assert_datasource_correct(datasource)
+
+
+class DataSourceApiFilterTest(TestCase):
+ datasources = []
+
+ @classmethod
+ def setUpTestData(cls):
+ owner = get_user_model().objects.create_user('Test API Owner')
+ cls.client = APIClient()
+ cls.client.force_login(owner)
+
+ cls.test_name = 'Filter'
+ test_url = 'https://api.iotuk.org.uk/iotOrganisation'
+
+ metadata_field = models.MetadataField.objects.create(
+ name='Filter field',
+ short_name='filter_field'
+ )
+
+ cls.datasources = [
+ models.DataSource.objects.create(
+ name=cls.test_name,
+ owner=owner,
+ url=test_url
+ ),
+ models.DataSource.objects.create(
+ name=cls.test_name + '-yes',
+ owner=owner,
+ url=test_url
+ ),
+ models.DataSource.objects.create(
+ name=cls.test_name + '-no',
+ owner=owner,
+ url=test_url
+ )
+ ]
+
+ cls.datasources[1].metadata_items.create(
+ field=metadata_field,
+ value='yes'
+ )
+
+ cls.datasources[2].metadata_items.create(
+ field=metadata_field,
+ value='no'
+ )
+
+ def test_no_filter(self):
+ response = self.client.get('/api/datasources/')
+ self.assertEqual(response.status_code, 200)
+
+ contents = response.json()
+ self.assertEqual(len(contents), 3)
+
+ def test_filter_yes(self):
+ response = self.client.get('/api/datasources/?filter_field=yes')
+ self.assertEqual(response.status_code, 200)
+
+ contents = response.json()
+ self.assertEqual(len(contents), 1)
+ self.assertEqual(self.test_name + '-yes', contents[0]['name'])
+
+ def test_filter_no(self):
+ response = self.client.get('/api/datasources/?filter_field=no')
+ self.assertEqual(response.status_code, 200)
+
+ contents = response.json()
+ self.assertEqual(len(contents), 1)
+ self.assertEqual(self.test_name + '-no', contents[0]['name'])
+
+
+class DataSourceApiPermissionsTest(TestCase):
+ @classmethod
+ def setUpTestData(cls):
+ cls.user = get_user_model().objects.create_user('Test API User')
+ cls.owner = get_user_model().objects.create_user('Test API Owner')
+
+ def setUp(self):
+ self.client = APIClient()
+ self.client.force_authenticate(self.user)
+
+ self.owner_client = Client()
+ self.owner_client.force_login(self.owner)
+
+ self.test_name = 'Permissions'
+ # TODO don't rely on external URL for testing
+ self.test_url = 'https://api.iotuk.org.uk/iotOrganisation'
+
+ def tearDown(self):
+ try:
+ self.model.delete()
+ except AttributeError:
+ pass
+
+ def _grant_permission(self, level: models.UserPermissionLevels):
+ response = self.owner_client.post('/datasources/{}/access/grant'.format(self.model.pk),
+ data={
+ 'user': self.user.pk,
+ 'granted': level.value,
+ },
+ headers={
+ 'Accept': 'application/json'
+ })
+ # TODO make this return a proper response code for AJAX-like requests
+ self.assertEqual(response.status_code, 302)
+ self.assertNotIn('login', response.url)
+
+ def test_datasource_permission_view(self):
+ """
+ Test that permissions are correctly handled when attempting to view a data source.
+ """
+ self.model = models.DataSource.objects.create(
+ name=self.test_name,
+ owner=self.owner,
+ url=self.test_url,
+ plugin_name='DataSetConnector',
+ public_permission_level=models.UserPermissionLevels.NONE
+ )
+
+ url = '/api/datasources/{}/'.format(self.model.pk)
+
+ response = self.client.get(url)
+ self.assertEqual(response.status_code, 403)
+
+ self._grant_permission(models.UserPermissionLevels.VIEW)
+
+ response = self.client.get(url)
+ self.assertEqual(response.status_code, 200)
+
+ self._grant_permission(models.UserPermissionLevels.META)
+
+ response = self.client.get(url)
+ self.assertEqual(response.status_code, 200)
+
+ self._grant_permission(models.UserPermissionLevels.DATA)
+
+ response = self.client.get(url)
+ self.assertEqual(response.status_code, 200)
+
+ self._grant_permission(models.UserPermissionLevels.PROV)
+
+ response = self.client.get(url)
+ self.assertEqual(response.status_code, 200)
+
+ def test_datasource_permission_meta(self):
+ """
+ Test that permissions are correctly handled when attempting to get metadata from a data source.
+ """
+ self.model = models.DataSource.objects.create(
+ name=self.test_name,
+ owner=self.owner,
+ url=self.test_url,
+ plugin_name='DataSetConnector',
+ public_permission_level=models.UserPermissionLevels.NONE
+ )
+
+ url = '/api/datasources/{}/metadata/'.format(self.model.pk)
+
+ response = self.client.get(url)
+ self.assertEqual(response.status_code, 403)
+
+ self._grant_permission(models.UserPermissionLevels.VIEW)
+
+ response = self.client.get(url)
+ self.assertEqual(response.status_code, 403)
+
+ self._grant_permission(models.UserPermissionLevels.META)
+
+ response = self.client.get(url)
+ # This data connector does not provide metadata
+ self.assertEqual(response.status_code, 400)
+
+ self._grant_permission(models.UserPermissionLevels.DATA)
+
+ response = self.client.get(url)
+ self.assertEqual(response.status_code, 400)
+
+ self._grant_permission(models.UserPermissionLevels.PROV)
+
+ response = self.client.get(url)
+ self.assertEqual(response.status_code, 400)
+
+ def test_datasource_permission_data(self):
+ """
+ Test that permissions are correctly handled when attempting to get data from a data source.
+ """
+ self.model = models.DataSource.objects.create(
+ name=self.test_name,
+ owner=self.owner,
+ url=self.test_url,
+ plugin_name='DataSetConnector',
+ public_permission_level=models.UserPermissionLevels.NONE
+ )
+
+ url = '/api/datasources/{}/data/?year=2018'.format(self.model.pk)
+
+ response = self.client.get(url)
+ self.assertEqual(response.status_code, 403)
+
+ self._grant_permission(models.UserPermissionLevels.VIEW)
+
+ response = self.client.get(url)
+ self.assertEqual(response.status_code, 403)
+
+ self._grant_permission(models.UserPermissionLevels.META)
+
+ response = self.client.get(url)
+ self.assertEqual(response.status_code, 403)
+
+ self._grant_permission(models.UserPermissionLevels.DATA)
+
+ response = self.client.get(url)
+ self.assertEqual(response.status_code, 200)
+
+ self._grant_permission(models.UserPermissionLevels.PROV)
+
+ response = self.client.get(url)
+ self.assertEqual(response.status_code, 200)
+
+ def test_datasource_permission_prov(self):
+ """
+ Test that permissions are correctly handled when attempting to get PROV data from a data source.
+ """
+ self.model = models.DataSource.objects.create(
+ name=self.test_name,
+ owner=self.owner,
+ url=self.test_url,
+ plugin_name='DataSetConnector',
+ public_permission_level=models.UserPermissionLevels.NONE
+ )
+
+ url = '/api/datasources/{}/prov/'.format(self.model.pk)
+
+ response = self.client.get(url)
+ self.assertEqual(response.status_code, 403)
+
+ self._grant_permission(models.UserPermissionLevels.VIEW)
+
+ response = self.client.get(url)
+ self.assertEqual(response.status_code, 403)
+
+ self._grant_permission(models.UserPermissionLevels.META)
+
+ response = self.client.get(url)
+ self.assertEqual(response.status_code, 403)
+
+ self._grant_permission(models.UserPermissionLevels.DATA)
+
+ response = self.client.get(url)
+ self.assertEqual(response.status_code, 403)
+
+ self._grant_permission(models.UserPermissionLevels.PROV)
+
+ response = self.client.get(url)
+ self.assertEqual(response.status_code, 200)
+
+
+class DataSourceApiIoTUKTest(TestCase):
+ @classmethod
+ def setUpTestData(cls):
+ cls.user = get_user_model().objects.create_user('Test API User')
+
+ def setUp(self):
+ self.client = APIClient()
+ self.client.force_authenticate(self.user)
+
+ self.test_name = 'IoTUK'
+ self.test_url = 'https://api.iotuk.org.uk/iotOrganisation'
+
+ def tearDown(self):
+ try:
+ self.model.delete()
+ except AttributeError:
+ pass
+
+ def test_api_datasource_get(self):
+ """
+ Test the :class:`DataSource` API get one functionality.
+ """
+ response = self.client.get('/api/datasources/1/')
+ self.assertEqual(response.status_code, 404)
+
+ self.model = models.DataSource.objects.create(
+ name=self.test_name,
+ owner=self.user,
+ url=self.test_url,
+ plugin_name='DataSetConnector'
+ )
+
+ response = self.client.get('/api/datasources/{}/'.format(self.model.pk))
+ self.assertEqual(response.status_code, 200)
+
+ datasource = response.json()
+
+ self.assertIn('name', datasource)
+ self.assertEqual(self.test_name, datasource['name'])
+
+ self.assertIn('description', datasource)
+ self.assertEqual('', datasource['description'])
+
+ self.assertIn('url', datasource)
+ self.assertEqual(self.test_url, datasource['url'])
+
+ def test_api_datasource_get_data(self):
+ """
+ Test the :class:`DataSource` API functionality to retrieve data.
+ """
+ response = self.client.get('/api/datasources/1/data/')
+ self.assertEqual(response.status_code, 404)
+
+ self.model = models.DataSource.objects.create(
+ name=self.test_name,
+ owner=self.user,
+ url=self.test_url,
+ plugin_name='DataSetConnector'
+ )
+
+ response = self.client.get('/api/datasources/{}/data/'.format(self.model.pk))
+ # Query should fail since IoTUK requires query filters
+ self.assertEqual(response.status_code, 400)
+
+ response = self.client.get('/api/datasources/{}/data/?year=2017'.format(self.model.pk))
+ self.assertEqual(response.status_code, 200)
+
+ data = response.json()
+
+ self.assertIn('results', data)
+ self.assertLessEqual(1, data['results'])
+ self.assertIn('data', data)
+ self.assertLessEqual(1, len(data['data']))
+
+
+class DataSourceApiHyperCatTest(TestCase):
+ test_name = 'HyperCat'
+ plugin_name = 'HyperCat'
+ test_url = 'https://api.cityverve.org.uk/v1/cat/polling-station'
+ dataset = 'https://api.cityverve.org.uk/v1/entity/polling-station/5'
+
+ @classmethod
+ def setUpTestData(cls):
+ from decouple import config
+
+ cls.user = get_user_model().objects.create_user('Test API User')
+
+ cls.api_key = config('HYPERCAT_CISCO_API_KEY')
+
+ cls.model = models.DataSource.objects.create(
+ name=cls.test_name,
+ owner=cls.user,
+ url=cls.test_url,
+ api_key=cls.api_key,
+ plugin_name=cls.plugin_name,
+ auth_method=models.DataSource.determine_auth_method(cls.test_url, cls.api_key)
+ )
+
+ def setUp(self):
+ self.client = APIClient()
+ self.client.force_authenticate(self.user)
+
+ def test_api_datasource_get(self):
+ """
+ Test the :class:`DataSource` API get one functionality.
+ """
+ response = self.client.get('/api/datasources/{}/'.format(self.model.pk))
+ self.assertEqual(response.status_code, 200)
+
+ datasource = response.json()
+
+ self.assertIn('name', datasource)
+ self.assertEqual(self.test_name, datasource['name'])
+
+ self.assertIn('description', datasource)
+ self.assertEqual('', datasource['description'])
+
+ self.assertIn('url', datasource)
+ self.assertEqual(self.test_url, datasource['url'])
+
+ def test_api_datasource_get_metadata(self):
+ """
+ Test the :class:`DataSource` API functionality to retrieve metadata.
+ """
+ response = self.client.get('/api/datasources/{}/metadata/'.format(self.model.pk))
+ self.assertEqual(response.status_code, 200)
+
+ data = response.json()
+
+ self.assertIn('status', data)
+ self.assertEqual('success', data['status'])
+ self.assertIn('data', data)
+ self.assertLessEqual(1, len(data['data']))
+ # TODO test contents of 'data' list
+
+ def test_api_datasource_get_datasets(self):
+ """
+ Test the :class:`DataSource` API functionality to retrieve the list of datasets.
+ """
+ response = self.client.get('/api/datasources/{}/datasets/'.format(self.model.pk))
+ self.assertEqual(response.status_code, 200)
+
+ data = response.json()
+
+ self.assertIn('status', data)
+ self.assertEqual('success', data['status'])
+ self.assertIn('data', data)
+ self.assertLessEqual(1, len(data['data']))
+ # TODO test contents of 'data' list
+
+ def test_api_datasource_get_dataset_metadata(self):
+ """
+ Test the :class:`DataSource` API functionality to retrieve dataset metadata.
+ """
+ response = self.client.get('/api/datasources/{}/datasets/{}/metadata/'.format(self.model.pk, self.dataset))
+ self.assertEqual(response.status_code, 200)
+
+ data = response.json()
+
+ self.assertIn('status', data)
+ self.assertEqual('success', data['status'])
+ self.assertIn('data', data)
+ self.assertLessEqual(1, len(data['data']))
+ # TODO test contents of 'data' list
+
+ def test_api_datasource_get_dataset_data(self):
+ """
+ Test the :class:`DataSource` API functionality to retrieve dataset data.
+ """
+ response = self.client.get('/api/datasources/{}/datasets/{}/data/'.format(self.model.pk, self.dataset))
+ self.assertEqual(response.status_code, 200)
+
+ data = response.json()
+ self.assertTrue(data)
+ # TODO test content
+
+
+if __name__ == '__main__':
+ TestCase.run()
diff --git a/api/urls.py b/api/urls.py
new file mode 100644
index 0000000000000000000000000000000000000000..9f39481a759622de22ae9645d632b145cae4ba29
--- /dev/null
+++ b/api/urls.py
@@ -0,0 +1,16 @@
+from django.urls import include, path
+
+from rest_framework import routers
+
+from .views import datasources as datasource_views
+
+app_name = 'api'
+
+# Register ViewSets
+router = routers.DefaultRouter()
+router.register('datasources', datasource_views.DataSourceApiViewset)
+
+urlpatterns = [
+ path('',
+ include(router.urls)),
+]
diff --git a/prov/migrations/__init__.py b/api/views/__init__.py
similarity index 100%
rename from prov/migrations/__init__.py
rename to api/views/__init__.py
diff --git a/api/views/datasources.py b/api/views/datasources.py
new file mode 100644
index 0000000000000000000000000000000000000000..0061d50c7344ccdc9d266e68bb47023ebc58bf7f
--- /dev/null
+++ b/api/views/datasources.py
@@ -0,0 +1,318 @@
+"""
+This module contains the API endpoint viewset defining the PEDASI Application API.
+"""
+
+import csv
+import json
+import typing
+
+from django.db.models import ObjectDoesNotExist
+from django.http import HttpResponse, JsonResponse
+from rest_framework import decorators, request, response, viewsets
+
+from .. import permissions
+from datasources import models, serializers
+from provenance import models as prov_models
+
+
+class DataSourceApiViewset(viewsets.ReadOnlyModelViewSet):
+ """
+ Provides views for:
+
+ /api/datasources/
+ List all :class:`DataSource`s
+
+ /api/datasources/<int>/
+ Retrieve a single :class:`DataSource`
+
+ /api/datasources/<int>/prov/
+ Retrieve PROV records related to a :class:`DataSource`
+
+ /api/datasources/<int>/metadata/
+ Retrieve :class:`DataSource` metadata via API call to data source URL
+
+ /api/datasources/<int>/data/
+ Retrieve :class:`DataSource` data via API call to data source URL
+
+ /api/datasources/<int>/datasets/
+ Retrieve :class:`DataSource` list of data sets via API call to data source URL
+
+ /api/datasources/<int>/datasets/<href>/metadata/
+ Retrieve :class:`DataSource` metadata for a single dataset via API call to data source URL
+
+ /api/datasources/<int>/datasets/<href>/metadata/
+ Retrieve :class:`DataSource` data for a single dataset via API call to data source URL
+ """
+ queryset = models.DataSource.objects.all()
+ serializer_class = serializers.DataSourceSerializer
+ permission_classes = [permissions.ViewPermission]
+
+ def _create_prov_entry(self, instance: models.DataSource) -> None:
+ """
+ Create a PROV entry linking the data source and the authenticated user.
+ """
+ # TODO should PROV distinguish between data and metadata accesses?
+ # TODO should we create PROV records for requests that failed
+
+ try:
+ # Is the user actually a proxy for an application?
+ application = self.request.user.application_proxy
+
+ prov_models.ProvWrapper.create_prov(
+ instance,
+ self.request.user.get_uri(),
+ application=application,
+ activity_type=prov_models.ProvActivity.ACCESS
+ )
+
+ except ObjectDoesNotExist:
+ # Normal (non-proxy) user
+ prov_models.ProvWrapper.create_prov(
+ instance,
+ self.request.user.get_uri(),
+ activity_type=prov_models.ProvActivity.ACCESS
+ )
+
+ except AttributeError:
+ # No logged in user - but has passed permission checks - data source must be public
+ pass
+
+ def _filter_by_metadata(self, queryset):
+ """
+ Query filter to filter data sources by variable metadata.
+
+ Query parameters are key value pairs of the metadata field short name and the metadata value.
+
+ :return: Filtered queryset
+ """
+ for key, value in self.request.query_params.items():
+ # The key 'search' is used to activate filters.SearchFilter - don't interfere with it
+ if key == 'search':
+ continue
+
+ queryset = queryset.filter(metadata_items__field__short_name=key,
+ metadata_items__value=value)
+
+ return queryset
+
+ def try_passthrough_response(self,
+ map_response: typing.Callable[..., HttpResponse],
+ error_message: str,
+ dataset: str = None) -> HttpResponse:
+ """
+ Attempt to pass a response from the data connector using the function `map_response`.
+
+ If the data connectors raises an error (AttributeError or NotImplementedError) then return an error response.
+
+ :param map_response: Function to get response from data connector - must return HttpResponse
+ :param error_message: Error message in case data connector raises an error
+ :param dataset: Dataset to access within data source
+ :return: HttpResponse from data connector or error response
+ """
+ instance = self.get_object()
+
+ with instance.data_connector as data_connector:
+ # Are there any query params to pass on?
+ params = self.request.query_params
+ if not params:
+ params = None
+
+ if dataset is not None:
+ data_connector = data_connector[dataset]
+
+ # Record this action in PROV
+ if not instance.prov_exempt:
+ self._create_prov_entry(instance)
+
+ try:
+ return map_response(data_connector, params)
+
+ except (AttributeError, NotImplementedError):
+ data = {
+ 'status': 'error',
+ 'message': error_message,
+ }
+ return response.Response(data, status=400)
+
+ def list(self, request, *args, **kwargs):
+ """
+ List the queryset after filtering by request query parameters for data source metadata.
+ """
+ queryset = self._filter_by_metadata(self.get_queryset())
+
+ page = self.paginate_queryset(queryset)
+ if page is not None:
+ serializer = self.get_serializer(page, many=True)
+ return self.get_paginated_response(serializer.data)
+
+ serializer = self.get_serializer(queryset, many=True)
+ return response.Response(serializer.data)
+
+ @decorators.action(detail=True, permission_classes=[permissions.ProvPermission])
+ def prov(self, request, pk=None):
+ """
+ View for /api/datasources/<int>/prov/
+
+ Retrieve PROV records related to a :class:`DataSource`.
+ """
+ instance = self.get_object()
+
+ data = {
+ # Get all ProvEntry's related to this instance and encode them as JSON
+ 'prov': [json.loads(record.to_json()) for record in prov_models.ProvWrapper.filter_model_instance(instance)]
+ }
+
+ # Record this action in PROV
+ if not instance.prov_exempt:
+ self._create_prov_entry(instance)
+
+ return response.Response(data, status=200)
+
+ @decorators.action(detail=True, permission_classes=[permissions.MetadataPermission])
+ def metadata(self, request, pk=None):
+ """
+ View for /api/datasources/<int>/metadata/
+
+ Retrieve :class:`DataSource` metadata via API call to data source URL.
+ """
+ def map_response(data_connector, params):
+ data = {
+ 'status': 'success',
+ 'data': data_connector.get_metadata(params=params)
+ }
+ return response.Response(data, status=200)
+
+ return self.try_passthrough_response(map_response,
+ 'Data source does not provide metadata')
+
+ @decorators.action(detail=True, methods=['GET'],
+ permission_classes=[permissions.DataPermission, permissions.DataPushPermission])
+ def data(self, request, pk=None):
+ """
+ View for /api/datasources/<int>/data/
+
+ Retrieve :class:`DataSource` data via API call to data source URL.
+ """
+ def map_response(data_connector, params):
+ r = data_connector.get_response(params=params)
+ try:
+ return HttpResponse(r.text, status=r.status_code,
+ content_type=r.headers.get('content-type'))
+
+ except AttributeError:
+ # Should be a Django response already
+ return r
+
+ return self.try_passthrough_response(map_response,
+ 'Data source does not provide data')
+
+ @data.mapping.post
+ def post_data(self, request: request.Request, pk=None):
+ """
+ Add data to this data source. Only applicable to internal data sources.
+
+ Data can be added either as JSON body text or as a POSTed CSV file.
+ """
+ instance = self.get_object()
+
+ try:
+ with instance.data_connector as data_connector:
+ if request.FILES:
+ for filename, f in request.FILES.items():
+ # TODO read in chunks
+ # TODO don't assume utf-8
+ data = f.read().decode('utf-8').splitlines()
+ reader = csv.DictReader(data)
+
+ data_connector.post_data(reader)
+
+ else:
+ data_connector.post_data(request.data)
+
+ # Rebuild index
+ index_fields = instance.metadata_items.filter(field__short_name='indexed_field').values_list('value')
+ if index_fields:
+ data_connector.clean_data(index_fields=index_fields)
+
+ # Record this action in PROV
+ if not instance.prov_exempt:
+ self._create_prov_entry(instance)
+
+ except AttributeError:
+ # Connector has no 'post_data' method
+ return JsonResponse({
+ 'status': 'error',
+ 'message': 'Data source does not support writing of data'
+ }, status=405)
+
+ return JsonResponse({
+ 'status': 'success',
+ 'data': None,
+ })
+
+ @data.mapping.put
+ def put_data(self, request: request.Request, pk=None):
+ instance = self.get_object()
+
+ with instance.data_connector as data_connector:
+ # Remove all existing data
+ data_connector.clear_data()
+
+ return self.post_data(request, pk)
+
+ @decorators.action(detail=True, permission_classes=[permissions.MetadataPermission])
+ def datasets(self, request, pk=None):
+ """
+ View for /api/datasources/<int>/datasets/
+
+ Retrieve :class:`DataSource` list of data sets via API call to data source URL.
+ """
+ def map_response(data_connector, params):
+ data = {
+ 'status': 'success',
+ 'data': data_connector.get_datasets(params=params)
+ }
+ return response.Response(data, status=200)
+
+ return self.try_passthrough_response(map_response,
+ 'Data source does not contain datasets')
+
+ # TODO URL pattern here uses pre Django 2 format
+ @decorators.action(detail=True,
+ url_path='datasets/(?P<href>.*)/metadata',
+ permission_classes=[permissions.MetadataPermission])
+ def dataset_metadata(self, request, pk=None, **kwargs):
+ """
+ View for /api/datasources/<int>/datasets/<href>/metadata/
+
+ Retrieve :class:`DataSource` metadata for a single dataset via API call to data source URL.
+ """
+ def map_response(data_connector, params):
+ data = {
+ 'status': 'success',
+ 'data': data_connector.get_metadata(params=params)
+ }
+ return response.Response(data, status=200)
+
+ return self.try_passthrough_response(map_response,
+ 'Data source does not provide metadata',
+ dataset=self.kwargs['href'])
+
+ # TODO URL pattern here uses pre Django 2 format
+ @decorators.action(detail=True,
+ url_path='datasets/(?P<href>.*)/data',
+ permission_classes=[permissions.DataPermission])
+ def dataset_data(self, request, pk=None, **kwargs):
+ """
+ View for /api/datasources/<int>/datasets/<href>/data/
+
+ Retrieve :class:`DataSource` data for a single dataset via API call to data source URL.
+ """
+ def map_response(data_connector, params):
+ r = data_connector.get_response(params=params)
+ return HttpResponse(r.text, status=r.status_code,
+ content_type=r.headers.get('content-type'))
+
+ return self.try_passthrough_response(map_response,
+ 'Data source does not provide data',
+ dataset=self.kwargs['href'])
diff --git a/applications/migrations/0004_application_access_control.py b/applications/migrations/0004_application_access_control.py
new file mode 100644
index 0000000000000000000000000000000000000000..9dd0e1622053a5a7a32e38eb59279d89a7deffec
--- /dev/null
+++ b/applications/migrations/0004_application_access_control.py
@@ -0,0 +1,18 @@
+# Generated by Django 2.0.8 on 2018-09-25 12:26
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('applications', '0003_remove_owner_from_forms'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='application',
+ name='access_control',
+ field=models.BooleanField(default=False),
+ ),
+ ]
diff --git a/applications/migrations/0005_one_to_one_group.py b/applications/migrations/0005_one_to_one_group.py
new file mode 100644
index 0000000000000000000000000000000000000000..5ef4ebfbadd2bd92122a98e9cf05c2fedd063a0d
--- /dev/null
+++ b/applications/migrations/0005_one_to_one_group.py
@@ -0,0 +1,25 @@
+# Generated by Django 2.0.8 on 2018-09-25 12:35
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('auth', '0009_alter_user_last_name_max_length'),
+ ('applications', '0004_application_access_control'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='application',
+ name='users_group',
+ field=models.OneToOneField(blank=True, editable=False, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='auth.Group'),
+ ),
+ migrations.AddField(
+ model_name='application',
+ name='users_group_requested',
+ field=models.OneToOneField(blank=True, editable=False, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='auth.Group'),
+ ),
+ ]
diff --git a/applications/migrations/0006_unset_url_required.py b/applications/migrations/0006_unset_url_required.py
new file mode 100644
index 0000000000000000000000000000000000000000..d2adeb094ba578e2735ddfbea34bc01822b40621
--- /dev/null
+++ b/applications/migrations/0006_unset_url_required.py
@@ -0,0 +1,18 @@
+# Generated by Django 2.0.8 on 2018-10-29 13:34
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('applications', '0005_one_to_one_group'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='application',
+ name='url',
+ field=models.URLField(blank=True),
+ ),
+ ]
diff --git a/applications/migrations/0007_application_proxy_user.py b/applications/migrations/0007_application_proxy_user.py
new file mode 100644
index 0000000000000000000000000000000000000000..8d39c09ad7f02857fa8d8a8e1930e57dcbb035ab
--- /dev/null
+++ b/applications/migrations/0007_application_proxy_user.py
@@ -0,0 +1,21 @@
+# Generated by Django 2.0.8 on 2018-11-01 11:43
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+ ('applications', '0006_unset_url_required'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='application',
+ name='proxy_user',
+ field=models.ForeignKey(blank=True, editable=False, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='application_proxy', to=settings.AUTH_USER_MODEL),
+ ),
+ ]
diff --git a/applications/migrations/0008_proxy_user_onetoone.py b/applications/migrations/0008_proxy_user_onetoone.py
new file mode 100644
index 0000000000000000000000000000000000000000..c76f649cd47c4df4299a7c13c7f84812b2e8bb78
--- /dev/null
+++ b/applications/migrations/0008_proxy_user_onetoone.py
@@ -0,0 +1,20 @@
+# Generated by Django 2.0.8 on 2018-11-01 15:37
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('applications', '0007_application_proxy_user'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='application',
+ name='proxy_user',
+ field=models.OneToOneField(blank=True, editable=False, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='application_proxy', to=settings.AUTH_USER_MODEL),
+ ),
+ ]
diff --git a/applications/migrations/0009_application_is_deleted.py b/applications/migrations/0009_application_is_deleted.py
new file mode 100644
index 0000000000000000000000000000000000000000..64d2a82b91fc306543af8a01dc8b2fdd2343ac3b
--- /dev/null
+++ b/applications/migrations/0009_application_is_deleted.py
@@ -0,0 +1,18 @@
+# Generated by Django 2.0.8 on 2019-01-11 14:42
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('applications', '0008_proxy_user_onetoone'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='application',
+ name='is_deleted',
+ field=models.BooleanField(default=False, editable=False),
+ ),
+ ]
diff --git a/applications/models.py b/applications/models.py
index a9cd7e8826ecad0bd5386d15eae2fd051d46a664..8d1b36f1b6164689154979751cb49e50ab9e9a64 100644
--- a/applications/models.py
+++ b/applications/models.py
@@ -1,8 +1,13 @@
+from django.contrib.auth import get_user_model
+from django.contrib.auth.models import Group
from django.conf import settings
from django.db import models
from django.urls import reverse
+from django.utils.text import slugify
-from pedasi.common.base_models import BaseAppDataModel
+from rest_framework.authtoken.models import Token
+
+from core.models import BaseAppDataModel, SoftDeletionManager
class Application(BaseAppDataModel):
@@ -14,6 +19,11 @@ class Application(BaseAppDataModel):
* A data / metadata visualisation tool
* A data / metadata analysis pipeline
"""
+ objects = SoftDeletionManager()
+
+ #: Address at which the API may be accessed
+ url = models.URLField(blank=True, null=False)
+
#: User who has responsibility for this application
owner = models.ForeignKey(settings.AUTH_USER_MODEL,
limit_choices_to={
@@ -24,6 +34,106 @@ class Application(BaseAppDataModel):
editable=False,
blank=False, null=False)
+ #: Proxy user which this application will act as
+ proxy_user = models.OneToOneField(settings.AUTH_USER_MODEL,
+ on_delete=models.PROTECT,
+ related_name='application_proxy',
+ editable=False,
+ blank=True, null=True)
+
+ #: Group of users who have read / use access to this data source / application
+ users_group = models.OneToOneField(Group,
+ on_delete=models.SET_NULL,
+ related_name='+',
+ editable=False,
+ blank=True, null=True)
+
+ #: Groups of users who have requested access to this data source / application
+ users_group_requested = models.OneToOneField(Group,
+ on_delete=models.SET_NULL,
+ related_name='+',
+ editable=False,
+ blank=True, null=True)
+
+ #: Do users require explicit permission to use this application?
+ access_control = models.BooleanField(default=False,
+ blank=False, null=False)
+
+ #: Has this object been soft deleted?
+ is_deleted = models.BooleanField(default=False,
+ editable=False, blank=False, null=False)
+
+ def delete(self, using=None, keep_parents=False):
+ """
+ Soft delete this object.
+ """
+ self.is_deleted = True
+ self.save()
+
+ @property
+ def _access_group_name(self):
+ return str(type(self)) + ' ' + self.name + ' Users'
+
+ def save(self, **kwargs):
+ # Create access control groups if they do not exist
+ # Make sure their names match self.name if they do exist
+ if self.access_control:
+ if self.users_group:
+ # Update existing group name
+ self.users_group.name = self._access_group_name
+ self.users_group.save()
+
+ else:
+ self.users_group, created = Group.objects.get_or_create(
+ name=self._access_group_name
+ )
+
+ if self.users_group_requested:
+ # Update existing group name
+ self.users_group_requested.name = self._access_group_name + ' Requested'
+ self.users_group_requested.save()
+
+ else:
+ self.users_group_requested, created = Group.objects.get_or_create(
+ name=self._access_group_name + ' Requested'
+ )
+
+ if not self.proxy_user:
+ self.proxy_user = self._get_proxy_user()
+
+ super().save(**kwargs)
+
+ def has_view_permission(self, user: settings.AUTH_USER_MODEL) -> bool:
+ """
+ Does a user have permission to use this application?
+
+ :param user: User to check
+ :return: User has permission?
+ """
+ if not self.access_control:
+ return True
+ if self.owner == user:
+ return True
+
+ return self.users_group.user_set.filter(pk=user.pk).exists()
+
+ def _get_proxy_user(self) -> settings.AUTH_USER_MODEL:
+ """
+ Create a new proxy user for this application or return the existing one.
+
+ :return: Instance of user model
+ """
+ if self.proxy_user:
+ return self.proxy_user
+
+ proxy_username = 'application-proxy-' + slugify(self.name)
+ proxy_user = get_user_model().objects.create_user(proxy_username)
+
+ # Create an API access token for the proxy user
+ Token.objects.create(user=proxy_user)
+
+ return proxy_user
+
def get_absolute_url(self):
return reverse('applications:application.detail',
kwargs={'pk': self.pk})
diff --git a/applications/search_indexes.py b/applications/search_indexes.py
new file mode 100644
index 0000000000000000000000000000000000000000..8885278fdd0dced7f8fd13276106231619dd40d5
--- /dev/null
+++ b/applications/search_indexes.py
@@ -0,0 +1,10 @@
+from haystack import indexes
+
+from . import models
+
+
+class ApplicationIndex(indexes.SearchIndex, indexes.Indexable):
+ text = indexes.CharField(document=True, use_template=True)
+
+ def get_model(self):
+ return models.Application
diff --git a/applications/templates/applications/application/create.html b/applications/templates/applications/application/create.html
new file mode 100644
index 0000000000000000000000000000000000000000..941514fd37ada74e79d676afe75cd44211ee5501
--- /dev/null
+++ b/applications/templates/applications/application/create.html
@@ -0,0 +1,29 @@
+{% extends "base.html" %}
+{% load bootstrap4 %}
+
+{% block content %}
+ <nav aria-label="breadcrumb">
+ <ol class="breadcrumb">
+ <li class="breadcrumb-item" aria-current="page">
+ <a href="{% url 'index' %}">Home</a>
+ </li>
+ <li class="breadcrumb-item" aria-current="page">
+ <a href="{% url 'applications:application.list' %}">Applications</a>
+ </li>
+ <li class="breadcrumb-item active" aria-current="page">
+ New Application
+ </li>
+ </ol>
+ </nav>
+
+ <h2 class="pb-3">New Application</h2>
+
+ <form class="form" method="post" action="">
+ {% csrf_token %}
+
+ {% bootstrap_form form %}
+
+ <input type="submit" class="btn btn-success" value="Create">
+ </form>
+
+{% endblock %}
diff --git a/applications/templates/applications/application/delete.html b/applications/templates/applications/application/delete.html
new file mode 100644
index 0000000000000000000000000000000000000000..83d00cde27c4efb78184af047985b97a89a61ac3
--- /dev/null
+++ b/applications/templates/applications/application/delete.html
@@ -0,0 +1,70 @@
+{% extends "base.html" %}
+{% load bootstrap4 %}
+
+{% block content %}
+ <nav aria-label="breadcrumb">
+ <ol class="breadcrumb">
+ <li class="breadcrumb-item" aria-current="page">
+ <a href="{% url 'index' %}">Home</a>
+ </li>
+ <li class="breadcrumb-item" aria-current="page">
+ <a href="{% url 'applications:application.list' %}">Applications</a>
+ </li>
+ <li class="breadcrumb-item" aria-current="page">
+ <a href="{% url 'applications:application.detail' pk=application.pk %}">{{ application.name }}</a>
+ </li>
+ <li class="breadcrumb-item active" aria-current="page">
+ Delete
+ </li>
+ </ol>
+ </nav>
+
+ <div class="row">
+ <div class="col-md-10 col-sm-8">
+ <h2>{{ application.name }}</h2>
+
+ {% if application.description %}
+ {{ application.description|linebreaks }}
+ {% endif %}
+ </div>
+ </div>
+
+ <table class="table">
+ <thead>
+ <th scope="col" class="w-25 border-0"></th>
+ <th scope="col" class="border-0"></th>
+ </thead>
+
+ <tbody>
+ <tr>
+ <td>Owner</td>
+ <td>
+ {{ application.owner }}
+ </td>
+ </tr>
+ <tr>
+ <td>URL</td>
+ <td>{{ application.url }}</td>
+ </tr>
+ {% if api_key %}
+ <tr>
+ <td>API Key</td>
+ <td>{{ api_key }}</td>
+ </tr>
+ {% endif %}
+ </tbody>
+ </table>
+
+ <div class="alert alert-danger">
+ <p><b>Are you sure you want to delete this application?</b></p>
+
+ <form class="form" method="post">
+ {% csrf_token %}
+
+ <input type="submit" role="button" class="btn btn-danger" value="Delete">
+
+ <a role="button" class="btn btn-info"
+ href="{% url 'applications:application.detail' pk=application.pk %}">Cancel</a>
+ </form>
+ </div>
+{% endblock %}
\ No newline at end of file
diff --git a/applications/templates/applications/application/detail-no-access.html b/applications/templates/applications/application/detail-no-access.html
new file mode 100644
index 0000000000000000000000000000000000000000..5b4e257c4f908865c02640cf170fd040ca9fd6bc
--- /dev/null
+++ b/applications/templates/applications/application/detail-no-access.html
@@ -0,0 +1,98 @@
+{% extends "base.html" %}
+{% load bootstrap4 %}
+
+{% block extra_head %}
+ <script src="https://cdn.jsdelivr.net/npm/js-cookie@2.2.0/src/js.cookie.min.js"></script>
+{% endblock %}
+
+{% block content %}
+ <nav aria-label="breadcrumb">
+ <ol class="breadcrumb">
+ <li class="breadcrumb-item" aria-current="page">
+ <a href="{% url 'index' %}">Home</a>
+ </li>
+ <li class="breadcrumb-item" aria-current="page">
+ <a href="{% url 'applications:application.list' %}">Applications</a>
+ </li>
+ <li class="breadcrumb-item active" aria-current="page">
+ {{ application.name }}
+ </li>
+ </ol>
+ </nav>
+
+ <div class="row">
+ <div class="col-md-10 col-sm-8">
+ <h2>{{ application.name }}</h2>
+
+ {% if application.description %}
+ {{ application.description|linebreaks }}
+ {% endif %}
+ </div>
+
+ <div class="col-md-2 col-sm-4">
+ {% if application.users_group_requested in request.user.groups.all %}
+ <button id="btn-request-access" disabled
+ class="btn btn-block btn-primary" role="button">Access Requested</button>
+
+ {% elif not request.user.is_authenticated %}
+ You must be logged in to request access.
+
+ {% else %}
+ <button id="btn-request-access" onclick="requestAccess()"
+ class="btn btn-block btn-primary" role="button">Request Access</button>
+
+ <script type="application/javascript">
+ function requestAccess(){
+ $.ajax({
+ url: '{% url 'applications:application.manage-access.user' pk=application.pk user_pk=request.user.pk %}',
+ headers: {
+ 'X-CSRFToken': Cookies.get('csrftoken')
+ },
+ method: 'PUT',
+ success: function(result, status, xhr){
+ const btn = document.getElementById('btn-request-access');
+ btn.textContent = 'Access Requested';
+ btn.disabled = true;
+ }
+ })
+ }
+ </script>
+ {% endif %}
+ </div>
+ </div>
+
+ <div class="alert alert-warning">
+ You do not have permission to access this resource.
+
+ {% if not request.user.is_authenticated %}
+ You must be logged in to request access.
+
+ {% elif request.user in application.users_group_requested.user_set.all %}
+ Please wait for your request to be approved.
+
+ {% else %}
+ You may request access using the 'Request Access' button.
+ {% endif %}
+ </div>
+
+ <table class="table">
+ <thead>
+ <th scope="col" class="w-25 border-0"></th>
+ <th scope="col" class="border-0"></th>
+ </thead>
+
+ <tbody>
+ <tr>
+ <td>Owner</td>
+ <td>
+ {{ application.owner }}
+ </td>
+ </tr>
+ <tr>
+ <td>URL</td>
+ <td>{{ application.url }}</td>
+ </tr>
+ </tbody>
+ </table>
+
+{% endblock %}
diff --git a/applications/templates/applications/application/detail.html b/applications/templates/applications/application/detail.html
index 2d1ee3af757fa25ca40ce8ef9af6c58d60fb0bf4..0fbb2f8e611251755affcdc606791b825d690244 100644
--- a/applications/templates/applications/application/detail.html
+++ b/applications/templates/applications/application/detail.html
@@ -16,19 +16,68 @@
</ol>
</nav>
- <h2>View Application - {{ application.name }}</h2>
+ <div class="row">
+ <div class="col-md-10 col-sm-8">
+ <h2>{{ application.name }}</h2>
- <p>
- Owner: <a href="#" role="link">{{ application.owner }}</a>
- </p>
+ {% if application.description %}
+ {{ application.description|linebreaks }}
+ {% endif %}
+ </div>
- {% if application.description %}
- <p>{{ application.description }}</p>
- {% endif %}
+ <div class="col-md-2 col-sm-4">
+ {% if application.access_control %}
+ <a href="{% url 'applications:application.manage-access' pk=application.pk %}"
+ class="btn btn-block btn-primary" role="button">Manage Access</a>
+ {% endif %}
- <a href="{% url 'admin:applications_application_change' application.id %}"
- class="btn btn-success" role="button">Edit</a>
- <a href="{% url 'admin:applications_application_delete' application.id %}"
- class="btn btn-danger" role="button">Delete</a>
+ {% if has_edit_permission %}
+ <a href="{% url 'applications:application.edit' pk=application.pk %}"
+ class="btn btn-block btn-success" role="button">Edit</a>
+
+ <a href="{% url 'applications:application.delete' pk=application.pk %}"
+ class="btn btn-block btn-danger" role="button">Delete</a>
+ {% endif %}
+ </div>
+ </div>
+
+ <table class="table">
+ <thead>
+ <th scope="col" class="w-25 border-0"></th>
+ <th scope="col" class="border-0"></th>
+ </thead>
+
+ <tbody>
+ <tr>
+ <td>Owner</td>
+ <td>
+ {{ application.owner }}
+ </td>
+ </tr>
+ <tr>
+ <td>URL</td>
+ <td>{{ application.url }}</td>
+ </tr>
+ {% if api_key %}
+ <tr>
+ <td>API Key</td>
+ <td>{{ api_key }}</td>
+ </tr>
+ {% endif %}
+ </tbody>
+ </table>
+
+ <div class="row justify-content-center pt-5">
+ <div class="col-4">
+ <script type="application/javascript">
+ function launchApp() {
+ const win = window.open("{{ application.url }}", "_blank");
+ win.focus();
+ }
+ </script>
+
+ <button role="button" onclick="launchApp();" class="btn btn-info btn-lg btn-block">Launch App</button>
+ </div>
+ </div>
{% endblock %}
\ No newline at end of file
diff --git a/applications/templates/applications/application/list.html b/applications/templates/applications/application/list.html
index 5b7c3c6d37ccb1d2a526c1c2d6f661c455621205..afa921813e5bb05e75f2d22cbe8607bdca7d54eb 100644
--- a/applications/templates/applications/application/list.html
+++ b/applications/templates/applications/application/list.html
@@ -15,34 +15,55 @@
<h2>Applications</h2>
- <a href="{% url 'admin:applications_application_add' %}"
- class="btn btn-success" role="button">Create Application</a>
+ {% if perms.applications.add_application %}
+ <div class="row">
+ <div class="col-md-6 mx-auto">
+ <a href="{% url 'applications:application.add' %}"
+ class="btn btn-block btn-success" role="button">New Application</a>
+ </div>
+ </div>
+ {% endif %}
<div class="mt-3"></div>
- <table class="table">
+ <table class="table table-hover">
<thead class="thead">
<tr>
- <th>Name</th>
- <th>
- <i class="fa fa-user" aria-hidden="true"></i>
+ <th class="w-75">Name</th>
+ <th class="w-auto">
+ <i class="fas fa-user" aria-hidden="true"></i>
</th>
- <th></th>
+ <th class="w-auto">Access</th>
+ <th class="w-auto"></th>
</tr>
</thead>
<tbody>
{% for application in applications %}
<tr>
- <td>{{ application.name }}</td>
<td>
+ <p>
+ <b>{{ application.name }}</b>
+ </p>
+ <p class="pl-5">
+ {{ application.description|truncatechars:120 }}
+ </p>
+ </td>
+ <td class="align-middle">
{% if application.owner == request.user %}
- <i class="fa fa-user" aria-hidden="true"></i>
+ <i class="fas fa-user" aria-hidden="true"
+ data-toggle="tooltip" data-placement="top" title="My application"></i>
{% endif %}
</td>
- <td>
+ <td class="align-middle">
+ {% if application.access_control %}
+ <i class="fas fa-lock fa-lg"
+ data-toggle="tooltip" data-placement="top" title="Application has access controls"></i>
+ {% endif %}
+ </td>
+ <td class="align-middle">
<a href="{% url 'applications:application.detail' pk=application.pk %}"
- class="btn btn-primary" role="button">Detail</a>
+ class="btn btn-block btn-secondary" role="button">Detail</a>
</td>
</tr>
{% empty %}
@@ -52,4 +73,12 @@
{% endfor %}
</tbody>
</table>
-{% endblock %}
\ No newline at end of file
+{% endblock %}
+
+{% block extra_body %}
+ <script type="application/javascript">
+ $(function () {
+ $("[data-toggle='tooltip']").tooltip()
+ })
+ </script>
+{% endblock %}
diff --git a/applications/templates/applications/application/manage_access.html b/applications/templates/applications/application/manage_access.html
new file mode 100644
index 0000000000000000000000000000000000000000..c5d393664632d646fe8f72da7da3395047d7888a
--- /dev/null
+++ b/applications/templates/applications/application/manage_access.html
@@ -0,0 +1,145 @@
+{% extends "base.html" %}
+{% load bootstrap4 %}
+
+{% block extra_head %}
+ <script src="https://cdn.jsdelivr.net/npm/js-cookie@2.2.0/src/js.cookie.min.js"></script>
+{% endblock %}
+
+{% block content %}
+ <nav aria-label="breadcrumb">
+ <ol class="breadcrumb">
+ <li class="breadcrumb-item" aria-current="page">
+ <a href="{% url 'index' %}">Home</a>
+ </li>
+ <li class="breadcrumb-item" aria-current="page">
+ <a href="{% url 'applications:application.list' %}">Applications</a>
+ </li>
+ <li class="breadcrumb-item" aria-current="page">
+ <a href="{% url 'applications:application.detail' pk=application.pk %}">{{ application.name }}</a>
+ </li>
+ <li class="breadcrumb-item active" aria-current="page">
+ Manage Access
+ </li>
+ </ol>
+ </nav>
+
+ <h2>{{ application.name }}</h2>
+
+ {% if application.description %}
+ <p>{{ application.description }}</p>
+ {% endif %}
+
+ <hr/>
+
+ <h2>Requests for Access</h2>
+
+ <table id="requested-user-table" class="table">
+ <thead class="thead">
+ <tr>
+ <th>Username</th>
+ <th></th>
+ <th></th>
+ </tr>
+ </thead>
+
+ <tbody>
+ {% for user in application.users_group_requested.user_set.all %}
+ <tr id="requested-user-{{ user.pk }}">
+ <td>{{ user.username }}</td>
+ <td>
+ <button onclick="userGrantAccess(
+ '{% url 'applications:application.manage-access.user' pk=application.pk user_pk=user.pk %}',
+ {{ user.pk }}
+ )"
+ class="btn btn-success"
+ role="button">Approve</button>
+ </td>
+ <td>
+ <button onclick="userRemoveAccess(
+ '{% url 'applications:application.manage-access.user' pk=application.pk user_pk=user.pk %}',
+ {{ user.pk }}
+ )"
+ class="btn btn-danger"
+ role="button">Reject</button>
+ </td>
+ </tr>
+ {% empty %}
+ <tr><td>No requests</td></tr>
+ {% endfor %}
+ </tbody>
+ </table>
+
+ <hr/>
+
+ <h2>Approved Users</h2>
+
+ <table id="approved-user-table" class="table">
+ <thead class="thead">
+ <tr>
+ <th>Username</th>
+ <th></th>
+ <th></th>
+ </tr>
+ </thead>
+
+ <tbody>
+ {% for user in application.users_group.user_set.all %}
+ <tr id="approved-user-{{ user.pk }}">
+ <td>{{ user.username }}</td>
+ <td></td>
+ <td>
+ <button onclick="userRemoveAccess(
+ '{% url 'applications:application.manage-access.user' pk=application.pk user_pk=user.pk %}',
+ {{ user.pk }}
+ )"
+ class="btn btn-danger"
+ role="button">Remove</button>
+ </td>
+ </tr>
+ {% empty %}
+ <tr><td>No approved users</td></tr>
+ {% endfor %}
+ </tbody>
+ </table>
+
+ <script type="application/javascript">
+ function userGrantAccess(url, userPk){
+ $.ajax({
+ {#url: '{% url 'applications:application.manage-access.user' pk=application.pk user_pk=request.user.pk %}',#}
+ url: url,
+ headers: {
+ 'X-CSRFToken': Cookies.get('csrftoken')
+ },
+ method: 'PUT',
+ success: function(result, status, xhr){
+ document.getElementById('requested-user-' + userPk.toString()).remove();
+
+ // TODO if table is empty add 'table is empty' row
+ }
+ })
+ }
+
+ function userRemoveAccess(url, userPk){
+ $.ajax({
+ {#url: '{% url 'applications:application.manage-access.user' pk=application.pk user_pk=request.user.pk %}',#}
+ url: url,
+ headers: {
+ 'X-CSRFToken': Cookies.get('csrftoken')
+ },
+ method: 'DELETE',
+ success: function(result, status, xhr){
+ try {
+ document.getElementById('approved-user-' + userPk.toString()).remove();
+ } catch (err) {}
+
+ try {
+ document.getElementById('requested-user-' + userPk.toString()).remove();
+ } catch (err) {}
+ }
+
+ // TODO if table is empty add 'table is empty' row
+ })
+ }
+ </script>
+
+{% endblock %}
\ No newline at end of file
diff --git a/applications/templates/applications/application/update.html b/applications/templates/applications/application/update.html
new file mode 100644
index 0000000000000000000000000000000000000000..63efd0ded6201f5814743877a0faf4ac2f986754
--- /dev/null
+++ b/applications/templates/applications/application/update.html
@@ -0,0 +1,32 @@
+{% extends "base.html" %}
+{% load bootstrap4 %}
+
+{% block content %}
+ <nav aria-label="breadcrumb">
+ <ol class="breadcrumb">
+ <li class="breadcrumb-item" aria-current="page">
+ <a href="{% url 'index' %}">Home</a>
+ </li>
+ <li class="breadcrumb-item" aria-current="page">
+ <a href="{% url 'applications:application.list' %}">Applications</a>
+ </li>
+ <li class="breadcrumb-item" aria-current="page">
+ <a href="{% url 'applications:application.detail' pk=application.pk %}">{{ application.name }}</a>
+ </li>
+ <li class="breadcrumb-item active" aria-current="page">
+ Edit Application
+ </li>
+ </ol>
+ </nav>
+
+ <h2 class="pb-3">Edit Application</h2>
+
+ <form class="form" method="post" action="">
+ {% csrf_token %}
+
+ {% bootstrap_form form %}
+
+ <input type="submit" class="btn btn-success" value="Update">
+ </form>
+
+{% endblock %}
diff --git a/applications/templates/search/indexes/applications/application_text.txt b/applications/templates/search/indexes/applications/application_text.txt
new file mode 100644
index 0000000000000000000000000000000000000000..f0c6d75533b053a1ed74cef679afdeffdb975e33
--- /dev/null
+++ b/applications/templates/search/indexes/applications/application_text.txt
@@ -0,0 +1,3 @@
+{{ object.name }}
+{{ object.owner.get_full_name }}
+{{ object.description }}
diff --git a/applications/tests.py b/applications/tests.py
index 5630863647d15857b0de33f33a599c6150c522c9..730a6bd570056d2fafd32fae79c7f0fc04f1d768 100644
--- a/applications/tests.py
+++ b/applications/tests.py
@@ -1,9 +1,30 @@
+from django.contrib.auth import get_user_model
from django.test import TestCase
from . import models
class ApplicationModelTest(TestCase):
+ @classmethod
+ def setUpTestData(cls):
+ cls.user_model = get_user_model()
+ cls.user = cls.user_model.objects.create_user('test')
+
def test_string_representation(self):
- application = models.Application(name='Test Application')
+ application = models.Application(name='Test Application',
+ owner=self.user)
self.assertEqual(str(application), application.name)
+
+ def test_db_store(self):
+ """
+ Test that Applications are able to be stored in a database.
+
+ Catches regression when making changes to database routers.
+ """
+ application = models.Application.objects.create(name='Test Application',
+ owner=self.user)
+
+ application_read, created = models.Application.objects.get_or_create(pk=application.pk)
+
+ self.assertFalse(created)
+ self.assertEqual(application, application_read)
diff --git a/applications/urls.py b/applications/urls.py
index a0b216e6364a9a24eb73910241c4f5bd3ebe3c86..1007f30ae0e53ddfa5607a16e98ac58e2de83276 100644
--- a/applications/urls.py
+++ b/applications/urls.py
@@ -9,7 +9,27 @@ urlpatterns = [
views.ApplicationListView.as_view(),
name='application.list'),
+ path('add',
+ views.ApplicationCreateView.as_view(),
+ name='application.add'),
+
path('<int:pk>/',
views.ApplicationDetailView.as_view(),
name='application.detail'),
+
+ path('<int:pk>/edit',
+ views.ApplicationUpdateView.as_view(),
+ name='application.edit'),
+
+ path('<int:pk>/delete',
+ views.ApplicationDeleteView.as_view(),
+ name='application.delete'),
+
+ path('<int:pk>/manage-access',
+ views.ApplicationManageAccessView.as_view(),
+ name='application.manage-access'),
+
+ path('<int:pk>/manage-access/users/<int:user_pk>',
+ views.ApplicationManageAccessView.as_view(),
+ name='application.manage-access.user'),
]
diff --git a/applications/views.py b/applications/views.py
index 17ee205de6b64318050759fb655bcaef4ea10d0c..77dc4788c3a5cc04af7d50b6d29a5e6b3e5f04c3 100644
--- a/applications/views.py
+++ b/applications/views.py
@@ -1,7 +1,14 @@
+from django.contrib.auth.mixins import PermissionRequiredMixin
+from django.urls import reverse_lazy
from django.views.generic.detail import DetailView
+from django.views.generic.edit import CreateView, DeleteView, UpdateView
from django.views.generic.list import ListView
+from rest_framework.authtoken.models import Token
+
from . import models
+from core.permissions import OwnerPermissionMixin
+from core.views import ManageAccessView
class ApplicationListView(ListView):
@@ -10,7 +17,69 @@ class ApplicationListView(ListView):
context_object_name = 'applications'
+class ApplicationCreateView(PermissionRequiredMixin, CreateView):
+ model = models.Application
+ template_name = 'applications/application/create.html'
+ context_object_name = 'application'
+
+ fields = '__all__'
+ permission_required = 'applications.add_application'
+
+ def form_valid(self, form):
+ try:
+ owner = form.instance.owner
+
+ except models.Application.owner.RelatedObjectDoesNotExist:
+ form.instance.owner = self.request.user
+
+ return super().form_valid(form)
+
+
+class ApplicationUpdateView(OwnerPermissionMixin, UpdateView):
+ model = models.Application
+ template_name = 'applications/application/update.html'
+ context_object_name = 'application'
+
+ fields = '__all__'
+
+
+class ApplicationDeleteView(OwnerPermissionMixin, DeleteView):
+ model = models.Application
+ template_name = 'applications/application/delete.html'
+ context_object_name = 'application'
+
+ success_url = reverse_lazy('applications:application.list')
+
+
class ApplicationDetailView(DetailView):
model = models.Application
template_name = 'applications/application/detail.html'
context_object_name = 'application'
+
+ def get_template_names(self):
+ if not self.object.has_view_permission(self.request.user):
+ return ['applications/application/detail-no-access.html']
+ return super().get_template_names()
+
+ def get_context_data(self, **kwargs):
+ context = super().get_context_data(**kwargs)
+
+ context['has_edit_permission'] = self.request.user.is_superuser or self.request.user == self.object.owner
+
+ if self.request.user == self.object.owner or self.request.user.is_superuser:
+ context['api_key'] = Token.objects.get(user=self.object.proxy_user)
+
+ return context
+
+
+class ApplicationManageAccessView(OwnerPermissionMixin, ManageAccessView):
+ """
+ Manage a user's access to a Application.
+
+ On GET request will display the access management page.
+ Accepts PUT and DELETE requests to add a user to, or remove a user from the access group.
+ Request responses follow JSend specification (see http://labs.omniti.com/labs/jsend).
+ """
+ model = models.Application
+ template_name = 'applications/application/manage_access.html'
+ context_object_name = 'application'
diff --git a/core/__init__.py b/core/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/core/apps.py b/core/apps.py
new file mode 100644
index 0000000000000000000000000000000000000000..26f78a8e673340121f68a92930e2830bc58d269d
--- /dev/null
+++ b/core/apps.py
@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class CoreConfig(AppConfig):
+ name = 'core'
diff --git a/core/models.py b/core/models.py
new file mode 100644
index 0000000000000000000000000000000000000000..b4b3378fc764471fe2e47141030eae086e92b461
--- /dev/null
+++ b/core/models.py
@@ -0,0 +1,61 @@
+"""
+This module contains models for functionality common to both Application and DataSource models.
+"""
+
+import abc
+
+from django.db import models
+
+
+#: Length of CharFields used to hold the names of objects
+MAX_LENGTH_NAME = 63
+
+MAX_LENGTH_API_KEY = 127
+
+MAX_LENGTH_PATH = 255
+
+
+class SoftDeletionManager(models.Manager):
+ """
+ Manager for soft-deletable objects. Filters out objects which have `is_deleted` set.
+ """
+ def get_queryset(self):
+ return super().get_queryset().filter(is_deleted=False)
+
+
+class BaseAppDataModel(models.Model):
+ """
+ The parent class of the Application and DataSource models - providing common functionality.
+
+ This class is an abstract model and will not create a corresponding DB table.
+ """
+ #: Friendly name of this application
+ name = models.CharField(max_length=MAX_LENGTH_NAME,
+ blank=False, null=False)
+
+ #: A brief description
+ description = models.TextField(blank=True, null=False)
+
+ # TODO replace this with an admin group
+ @property
+ @abc.abstractmethod
+ def owner(self):
+ """
+ User responsible for this data source / application.
+ """
+ raise NotImplementedError
+
+ @abc.abstractmethod
+ def get_absolute_url(self):
+ """
+ Return URL at which this object may be viewed.
+
+ Method must be implemented by inheriting classes.
+ """
+ raise NotImplementedError
+
+ def __str__(self):
+ return self.name
+
+ class Meta:
+ abstract = True
diff --git a/core/permissions.py b/core/permissions.py
new file mode 100644
index 0000000000000000000000000000000000000000..ddd75cb0fb2d056faef6e45de3525a501b4b2876
--- /dev/null
+++ b/core/permissions.py
@@ -0,0 +1,6 @@
+from django.contrib.auth.mixins import UserPassesTestMixin
+
+
+class OwnerPermissionMixin(UserPassesTestMixin):
+ def test_func(self):
+ return self.request.user == self.get_object().owner or self.request.user.is_superuser
diff --git a/pedasi/common/plugin.py b/core/plugin.py
similarity index 86%
rename from pedasi/common/plugin.py
rename to core/plugin.py
index 8235e0f55ec9312772ee5523ce0c98a0d4881f86..76b1fe1bbe8c2c0b4e332c549a6c9222e736021f 100644
--- a/pedasi/common/plugin.py
+++ b/core/plugin.py
@@ -1,3 +1,7 @@
+"""
+This module contains functionality for configurable plugins.
+"""
+
import abc
import importlib
import typing
@@ -29,7 +33,7 @@ class Plugin(abc.ABCMeta):
else:
cls._plugins[name] = cls
- def get_plugin(cls, class_name: str) -> type:
+ def get_plugin(cls, class_name: str) -> typing.Type:
"""
Get a plugin class by name.
@@ -56,4 +60,6 @@ class Plugin(abc.ABCMeta):
# This causes a call to the metaclass __init__ method which registers the plugin
importlib.import_module(str(plugin_dir).replace('/', '.') + '.' + module_name)
-
+ @property
+ def plugin_choices(cls) -> typing.List[typing.Tuple[str, str]]:
+ return [(name, name) for name, plugin in cls._plugins.items()]
diff --git a/core/views.py b/core/views.py
new file mode 100644
index 0000000000000000000000000000000000000000..08adea1765d4a2206885ddefef8a0ec5dcd40d42
--- /dev/null
+++ b/core/views.py
@@ -0,0 +1,100 @@
+"""
+This module contains views for behaviour common to both Application and DataSource models.
+"""
+
+from django.contrib.auth import get_user_model
+from django.http import JsonResponse, HttpResponseBadRequest, HttpResponseForbidden
+from django.views.generic.detail import DetailView
+
+
+class ManageAccessView(DetailView):
+ """
+ Manage a user's access to a resource.
+
+ On GET request will display the access management page.
+ Accepts PUT and DELETE requests to add a user to, or remove a user from the access group.
+ Request responses follow JSend specification (see http://labs.omniti.com/labs/jsend).
+ """
+ def put(self, request, *args, **kwargs):
+ """
+ Add a user to the access group for a resource.
+
+ If the request is performed by the resource owner or by staff: add them directly to the access group,
+ If the request is performed by the user themselves: add them to the 'access requested' group,
+ Else reject the request.
+
+ :param request:
+ :param args:
+ :param kwargs:
+ :return:
+ """
+ self.request = request
+ self.object = self.get_object()
+
+ user = get_user_model().objects.get(pk=kwargs['user_pk'])
+ access_group = self.object.users_group
+ request_group = self.object.users_group_requested
+
+ if self.request.user == self.object.owner or self.request.user.is_superuser:
+ # If request is from resource owner or admin, add user to access group
+ access_group.user_set.add(user)
+ request_group.user_set.remove(user)
+
+ elif self.request.user == user:
+ if access_group.user_set.filter(id=user.id).exists():
+ return HttpResponseBadRequest(
+ JsonResponse({
+ 'status': 'fail',
+ 'message': 'You already have access to this resource',
+ })
+ )
+
+ # If user is requesting for themselves, add them to 'access requested' group
+ request_group.user_set.add(user)
+
+ else:
+ return HttpResponseForbidden(
+ JsonResponse({
+ 'status': 'fail',
+ 'message': 'You do not have permission to set access for this user',
+ })
+ )
+
+ return JsonResponse({
+ 'status': 'success',
+ 'data': {
+ 'user': {
+ 'pk': user.pk
+ },
+ },
+ })
+
+ def delete(self, request, *args, **kwargs):
+ self.request = request
+ self.object = self.get_object()
+
+ user = get_user_model().objects.get(pk=kwargs['user_pk'])
+ access_group = self.object.users_group
+ request_group = self.object.users_group_requested
+
+ if self.request.user == user or self.request.user == self.object.owner or self.request.user.is_staff:
+ # Users can remove themselves, be removed by the resource owner, or by staff
+ access_group.user_set.remove(user)
+ request_group.user_set.remove(user)
+
+ else:
+ return HttpResponseForbidden(
+ JsonResponse({
+ 'status': 'fail',
+ 'message': 'You do not have permission to set access for this user',
+ })
+ )
+
+ return JsonResponse({
+ 'status': 'success',
+ 'data': {
+ 'user': {
+ 'pk': user.pk
+ },
+ },
+ })
diff --git a/datasources/admin.py b/datasources/admin.py
index e794f215a1d27040ff700fafaec379f40bdf626a..f98162f754fafea55bc3d7fccae6ec20095a71e6 100644
--- a/datasources/admin.py
+++ b/datasources/admin.py
@@ -1,11 +1,17 @@
from django.contrib import admin
-from . import models
+from . import forms, models
+
+
+@admin.register(models.MetadataField)
+class MetadataFieldAdmin(admin.ModelAdmin):
+ pass
@admin.register(models.DataSource)
class DataSourceAdmin(admin.ModelAdmin):
readonly_fields = ['owner']
+ form = forms.DataSourceForm
def has_change_permission(self, request, obj=None) -> bool:
"""
@@ -35,6 +41,7 @@ class DataSourceAdmin(admin.ModelAdmin):
"""
try:
owner = form.instance.owner
+
except models.DataSource.owner.RelatedObjectDoesNotExist:
form.instance.owner = request.user
diff --git a/datasources/connectors/__init__.py b/datasources/connectors/__init__.py
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..38be2bb0a3b75b6eee596797991dd4a522f57777 100644
--- a/datasources/connectors/__init__.py
+++ b/datasources/connectors/__init__.py
@@ -0,0 +1 @@
+from .base import BaseDataConnector, DataCatalogueConnector, DataSetConnector
\ No newline at end of file
diff --git a/datasources/connectors/base.py b/datasources/connectors/base.py
index 50005c0a74c17068ddf5e97b4e082069cc12245f..38589688d1e97427b2f5252e9433d73e998c18af 100644
--- a/datasources/connectors/base.py
+++ b/datasources/connectors/base.py
@@ -1,8 +1,74 @@
+"""
+This module contains base classes for data connectors.
+
+Data connectors are the component of PEDASI which interacts directly with data provider APIs.
+"""
+
import abc
+from collections import abc as collections_abc
+from collections import OrderedDict
+import enum
import typing
+import requests
+import requests.auth
+
+from core import plugin
+
+
+@enum.unique
+class AuthMethod(enum.IntEnum):
+ """
+ Authentication method to be used when performing a request to the external API.
+ """
+ # Does not require authentication
+ NONE = -1
+
+ # Unknown - assume no authentication if a request is sent
+ UNKNOWN = 0
+
+ # HTTPBasicAuth from Requests
+ BASIC = 1
+
+ # Same as HTTPBasicAuth but key is already b64 encoded
+ HEADER = 2
+
+ @classmethod
+ def choices(cls):
+ return tuple((i.value, i.name) for i in cls)
+
+
+class HttpHeaderAuth(requests.auth.HTTPBasicAuth):
+ """
+ Requests Auth provider.
+
+ The same as HttpBasicAuth - but don't convert to base64
+
+ Used for e.g. Cisco HyperCat API
+ """
+ def __call__(self, r):
+ r.headers['Authorization'] = self.username
+ return r
+
+
+REQUEST_AUTH_FUNCTIONS = OrderedDict([
+ (AuthMethod.NONE, None),
+ (AuthMethod.UNKNOWN, None),
+ (AuthMethod.BASIC, requests.auth.HTTPBasicAuth),
+ (AuthMethod.HEADER, HttpHeaderAuth),
+])
+
+
+class RequestCounter:
+ def __init__(self, count: int = 0):
+ self._count = count
-from pedasi.common import plugin
+ def __iadd__(self, other: int):
+ self._count += other
+ return self
+
+ def count(self):
+ return self._count
class BaseDataConnector(metaclass=plugin.Plugin):
@@ -13,63 +79,174 @@ class BaseDataConnector(metaclass=plugin.Plugin):
* A single dataset
* A data catalogue - a collection of datasets
+ """
+ #: Does this data connector represent a data catalogue containing multiple datasets?
+ is_catalogue = None
- TODO:
+ #: Help string to be shown when a data provider is choosing a connector
+ description = None
- * Should this connector interface be able to handle data catalogues and datasets
- or should we create a new connector for datasets within a catalogue?
- * What other operations do we need?
- """
def __init__(self, location: str,
- api_key: typing.Optional[str] = None):
+ api_key: typing.Optional[str] = None,
+ auth: typing.Optional[typing.Callable] = None,
+ **kwargs):
self.location = location
self.api_key = api_key
+ self.auth = auth
- @abc.abstractmethod
- def get_data(self,
- dataset: typing.Optional[str] = None,
- query_params: typing.Optional[typing.Mapping[str, str]] = None):
+ self._request_counter = RequestCounter()
+
+ @property
+ def request_count(self):
+ return self._request_counter.count()
+
+ def get_metadata(self,
+ params: typing.Optional[typing.Mapping[str, str]] = None):
"""
- Get data from this source using the appropriate API.
+ Get metadata from this source using the appropriate API.
- :param dataset: Optional dataset for which to get data
- :param query_params: Optional query parameter filters
- :return: Requested data
+ :param params: Optional query parameter filters
+ :return: Requested metadata
+ """
+ try:
+ if self._metadata is not None:
+ return self._metadata
+
+ except AttributeError:
+ pass
+
+ raise NotImplementedError('This data connector does not provide metadata')
+
+ def get_response(self,
+ params: typing.Optional[typing.Mapping[str, str]] = None):
+ """
+ Transparently return a response from a source API.
+
+ :param params: Optional query parameter filters
+ :return: Requested data / metadata - response is passed transparently
+ """
+ return self._get_auth_request(self.location,
+ params=params)
+
+ def _get_auth_request(self, url, **kwargs):
+ self._request_counter += 1
+
+ if self.auth is None:
+ return requests.get(url, **kwargs)
+
+ return requests.get(url,
+ auth=self.auth(self.api_key, ''),
+ **kwargs)
+
+
+class ReadOnlyInternalDataConnector(abc.ABC):
+ """
+ Abstract mixin representing a connector for an internally hosted data source which is read only.
+ """
+ @abc.abstractmethod
+ def clean_data(self, **kwargs):
+ """
+ Clean, validate or otherwise structure the data contained within this data source.
"""
raise NotImplementedError
-class DataConnectorContainsDatasets:
+class InternalDataConnector(ReadOnlyInternalDataConnector):
"""
- Mixin class indicating that the plugin represents a data source containing
- multiple datasets.
+ Abstract mixin representing a connector for an internally hosted data source which is able to be written to.
"""
@abc.abstractmethod
- def get_datasets(self,
- query_params: typing.Optional[typing.Mapping[str, str]] = None):
+ def clear_data(self):
+ """
+ Clear all data from this data source.
+ """
+ raise NotImplementedError
+
+ @abc.abstractmethod
+ def post_data(self, data: typing.Union[typing.MutableMapping[str, str],
+ typing.List[typing.MutableMapping[str, str]]]):
"""
- Get the list of all dataset identifiers contained within this source
- using the appropriate API.
+ Add data to this data source.
- :param query_params: Optional query parameter filters
- :return: All dataset identifiers
+ :param data: Data to add
"""
raise NotImplementedError
-class DataConnectorHasMetadata:
+class DataCatalogueConnector(BaseDataConnector, collections_abc.Mapping):
"""
- Mixin class indicating the the plugin represents a data source with metadata.
+ Base class of data connectors which provide access to a data catalogue.
"""
+ #: Does this data connector represent a data catalogue containing multiple datasets?
+ is_catalogue = True
+
+ def get_data(self,
+ params: typing.Optional[typing.Mapping[str, str]] = None):
+ raise TypeError('Data catalogues contain only metadata. You must select a dataset.')
+
@abc.abstractmethod
- def get_metadata(self,
- dataset: typing.Optional[str] = None,
- query_params: typing.Optional[typing.Mapping[str, str]] = None):
+ def get_datasets(self,
+ params: typing.Optional[typing.Mapping[str, str]] = None) -> typing.List[str]:
"""
- Get metadata from this source using the appropriate API.
+ Get the list of datasets provided by this catalogue.
- :param dataset: Optional dataset for which to get metadata
- :param query_params: Optional query parameter filters
- :return: Requested metadata
+ :param params: Query parameters to pass to data source API
+ :return: List of datasets provided by this catalogue
"""
raise NotImplementedError
+
+ @abc.abstractmethod
+ def __getitem__(self, item: str) -> BaseDataConnector:
+ raise NotImplementedError
+
+ def __iter__(self):
+ return iter(self.get_datasets())
+
+ def __len__(self):
+ return len(self.get_datasets())
+
+
+class DataSetConnector(BaseDataConnector):
+ """
+ Base class of data connectors which provide access to a single dataset.
+
+ Metadata may be passed to the constructor if it has been collected from a previous source
+ otherwise attempting to retrieve metadata will raise NotImplementedError.
+
+ If you wish to connect to a source that provides metadata itself, you must create a new
+ connector class which inherits from this one.
+ """
+ #: Does this data connector represent a data catalogue containing multiple datasets?
+ is_catalogue = False
+
+ #: Help string to be shown when a data provider is choosing a connector
+ description = (
+ 'This connector is the default option and should be used when accessing an API at a single endpoint '
+ 'which may or may not accept query parameters.'
+ )
+
+ def __init__(self, location: str,
+ api_key: typing.Optional[str] = None,
+ auth: typing.Optional[typing.Callable] = None,
+ metadata: typing.Optional = None):
+ super().__init__(location, api_key, auth=auth)
+
+ self._metadata = metadata
+
+ def get_data(self,
+ params: typing.Optional[typing.Mapping[str, str]] = None):
+ """
+ Retrieve the data from this source.
+
+ If the data is JSON formatted it will be parsed into a dictionary - otherwise it will
+ be passed as plain text.
+
+ :param params: Query parameters to be passed through to the data source API
+ :return: Data source data
+ """
+ response = self.get_response(params)
+
+ if 'json' in response.headers['Content-Type']:
+ return response.json()
+
+ return response.text
diff --git a/datasources/connectors/csv.py b/datasources/connectors/csv.py
new file mode 100644
index 0000000000000000000000000000000000000000..2aa9cb2a1c61ed7afd1b168975b1b5b0f9770104
--- /dev/null
+++ b/datasources/connectors/csv.py
@@ -0,0 +1,178 @@
+"""
+Connectors for handling CSV data.
+"""
+
+import csv
+import json
+import typing
+
+from django.http import JsonResponse
+
+import mongoengine
+from mongoengine import context_managers
+
+from .base import DataSetConnector, InternalDataConnector
+
+
+class CsvConnector(DataSetConnector):
+ """
+ Data connector for retrieving data from CSV files.
+ """
+ def get_metadata(self,
+ params: typing.Optional[typing.Mapping[str, str]] = None):
+ """
+ Return a JSON response from a CSV file.
+
+ :param params: Query params - ignored
+ :return: Metadata
+ """
+ with open(self.location, 'r') as csvfile:
+ # Requires a header row
+ reader = csv.DictReader(csvfile)
+ return reader.fieldnames
+
+ def get_response(self,
+ params: typing.Optional[typing.Mapping[str, str]] = None):
+ """
+ Return a JSON response from a CSV file.
+
+ CSV file must have a header row with column titles.
+
+ :param params: Optional query parameter filters
+ :return: Requested data
+ """
+ try:
+ with open(self.location, 'r') as csvfile:
+ # Requires a header row
+ reader = csv.DictReader(csvfile)
+
+ if params is None:
+ params = {}
+
+ rows = []
+ for row in reader:
+ for key, value in params.items():
+ try:
+ if row[key].strip() != value.strip():
+ break
+
+ except KeyError:
+ # The filter field isn't in the data so no row can satisfy it
+ break
+
+ else:
+ # All filters match
+ rows.append(dict(row))
+
+ return JsonResponse({
+ 'status': 'success',
+ 'data': rows,
+ })
+
+ except UnicodeDecodeError:
+ return JsonResponse({
+ 'status': 'error',
+ 'message': 'Invalid CSV file',
+ }, status=500)
+
+
+class CsvRow(mongoengine.DynamicDocument):
+ """
+ MongoDB dynamic document to store CSV data.
+
+ Store in own database - distinct from PROV data.
+ Collection must be changed manually when managing CsvRows since all connectors use the same backing model.
+ """
+ meta = {
+ 'db_alias': 'internal_data',
+ }
+
+
+def _type_convert(val):
+ """
+ Attempt to convert a value into a numeric type.
+
+ :param val: Value to attempt to convert
+ :return: Converted value or unmodified value if conversion was not possible
+ """
+ for conv in (int, float):
+ try:
+ return conv(val)
+
+ except ValueError:
+ pass
+
+ return val
+
+
+class CsvToMongoConnector(InternalDataConnector, DataSetConnector):
+ """
+ Data connector representing an internally hosted data source, backed by MongoDB.
+
+ This connector allows data to be pushed as well as retrieved.
+ """
+ id_field_alias = '__id'
+
+ def clean_data(self, **kwargs):
+ index_fields = kwargs.get('index_fields', None)
+
+ if index_fields is None:
+ return
+
+ if isinstance(index_fields, str):
+ index_fields = [index_fields]
+
+ with context_managers.switch_collection(CsvRow, self.location) as collection:
+ for index_field in index_fields:
+ collection.create_index(index_field, background=True)
+
+ def clear_data(self):
+ with context_managers.switch_collection(CsvRow, self.location) as collection:
+ collection.objects.delete()
+
+ def post_data(self, data: typing.Union[typing.MutableMapping[str, str],
+ typing.List[typing.MutableMapping[str, str]]]):
+ def create_document(row: typing.MutableMapping[str, str]):
+ kwargs = {key: _type_convert(val) for key, val in row.items()}
+
+ # Can't store field 'id' in document - rename it
+ if 'id' in kwargs:
+ kwargs[self.id_field_alias] = kwargs.pop('id')
+
+ return kwargs
+
+ # Put data in collection belonging to this data source
+ with context_managers.switch_collection(CsvRow, self.location) as collection:
+ collection = collection._get_collection()
+
+ try:
+ # Data is a dictionary - a single row
+ collection.insert_one(create_document(data))
+
+ except AttributeError:
+ # Data is a list of dictionaries - multiple rows
+ documents = (create_document(row) for row in data)
+ collection.insert_many(documents)
+
+ def get_response(self,
+ params: typing.Optional[typing.Mapping[str, str]] = None):
+ # TODO accept parameters provided twice as an inclusive OR
+ if params is None:
+ params = {}
+ params = {key: _type_convert(val) for key, val in params.items()}
+
+ with context_managers.switch_collection(CsvRow, self.location) as collection:
+ records = collection.objects.filter(**params)
+
+ # To get dictionary from MongoEngine records we need to go via JSON string
+ data = json.loads(records.exclude('_id').to_json())
+
+ # Couldn't store field 'id' in document - recover it
+ for item in data:
+ if self.id_field_alias in item:
+ item['id'] = item.pop(self.id_field_alias)
+
+ return JsonResponse({
+ 'status': 'success',
+ 'data': data,
+ })
diff --git a/datasources/connectors/entity.py b/datasources/connectors/entity.py
new file mode 100644
index 0000000000000000000000000000000000000000..e66f621815a4e987e1079de475835a455aeffd14
--- /dev/null
+++ b/datasources/connectors/entity.py
@@ -0,0 +1,123 @@
+"""
+This module contains connectors for receiving data via Cisco's Entity API.
+"""
+
+import typing
+
+from .base import BaseDataConnector, DataCatalogueConnector, DataSetConnector
+
+
+class CiscoEntityConnector(DataCatalogueConnector):
+ """
+ Data connector for retrieving data or metadata from Cisco's Entity API.
+ """
+ dataset_connector_class = DataSetConnector
+
+ def __init__(self, location: str,
+ api_key: typing.Optional[str] = None,
+ auth: typing.Optional[typing.Callable] = None,
+ metadata: typing.Optional[typing.Mapping] = None):
+ super().__init__(location, api_key=api_key, auth=auth)
+
+ self._response = None
+ self._metadata = metadata
+
+ def __getitem__(self, item: str) -> BaseDataConnector:
+ params = {
+ 'href': item
+ }
+
+ # Use cached response if we have one
+ response = self._get_response(params=params)
+
+ dataset_item = self._get_item_by_key_value(
+ response,
+ 'uri',
+ item
+ )
+
+ if 'timeseries' in item:
+ return self.dataset_connector_class(item, self.api_key,
+ auth=self.auth,
+ metadata=dataset_item)
+
+ return type(self)(item, self.api_key,
+ auth=self.auth,
+ metadata=dataset_item)
+
+ def items(self,
+ params=None) -> typing.ItemsView:
+ """
+ Get key-value pairs of dataset ID to dataset connector for datasets contained within this catalogue.
+
+ :param params: Query parameters to be passed through to the data source API
+ :return: Dictionary ItemsView over datasets
+ """
+ # Use cached response if we have one
+ response = self._get_response(params)
+
+ connector_dict = {
+ item['uri']: self.dataset_connector_class(item['uri'], self.api_key,
+ auth=self.auth,
+ metadata=item)
+ # Response JSON is a list of entities
+ for item in response
+ }
+
+ return connector_dict.items()
+
+ def get_metadata(self,
+ params: typing.Optional[typing.Mapping[str, str]] = None):
+ if self._metadata is None:
+ raise NotImplementedError
+
+ return self._metadata
+
+ def get_datasets(self,
+ params: typing.Optional[typing.Mapping[str, str]] = None) -> typing.List[str]:
+ # Use cached response if we have one
+ response = self._get_response(params=params)
+
+ datasets = []
+ if len(response) == 1 and 'timeseries' in response[0]:
+ # Response is one entity which should contain data series?
+ # TODO is it always 'timeseries'?
+ for item in response[0]['timeseries']:
+ datasets.append(item['uri'])
+
+ else:
+ for item in response:
+ datasets.append(item['uri'])
+
+ return datasets
+
+ @staticmethod
+ def _get_item_by_key_value(collection: typing.Iterable[typing.Mapping],
+ key: str, value) -> typing.Mapping:
+ matches = [item for item in collection if item[key] == value]
+
+ if not matches:
+ raise KeyError
+ elif len(matches) > 1:
+ raise ValueError('Multiple items were found')
+
+ return matches[0]
+
+ def _get_response(self, params: typing.Optional[typing.Mapping[str, str]] = None) -> typing.Mapping:
+ # Use cached response if we have one
+ # TODO should we use cached responses?
+ if self._response is not None and params is None:
+ # Ignore params - they only filter - we already have everything
+ response = self._response
+ else:
+ response = self._get_auth_request(self.location,
+ params=params)
+ response.raise_for_status()
+ return response.json()
+
+ def __enter__(self):
+ self._response = self._get_response()
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ pass
diff --git a/datasources/connectors/hypercat.py b/datasources/connectors/hypercat.py
index 8616c0ff014bc703eed8eef38f2be75095f7dc95..33bf26a091caa4fe66407d0001f92715794e0d05 100644
--- a/datasources/connectors/hypercat.py
+++ b/datasources/connectors/hypercat.py
@@ -1,81 +1,104 @@
+"""
+This module contains data connector classes for retrieving data from HyperCat catalogues.
+"""
+
import typing
-import requests
-import requests.auth
+from .base import BaseDataConnector, DataCatalogueConnector, DataSetConnector
-from datasources.connectors.base import BaseDataConnector, DataConnectorContainsDatasets, DataConnectorHasMetadata
+class HyperCat(DataCatalogueConnector):
+ """
+ Data connector for retrieving data or metadata from a HyperCat catalogue.
+ """
+ dataset_connector_class = DataSetConnector
-class HyperCat(DataConnectorContainsDatasets, DataConnectorHasMetadata, BaseDataConnector):
def __init__(self, location: str,
- api_key: typing.Optional[str] = None):
- super().__init__(location, api_key=api_key)
+ api_key: typing.Optional[str] = None,
+ auth: typing.Optional[typing.Callable] = None):
+ super().__init__(location, api_key=api_key, auth=auth)
self._response = None
- def get_data(self,
- dataset: typing.Optional[str] = None,
- query_params: typing.Optional[typing.Mapping[str, str]] = None):
- if dataset is None:
- raise ValueError('When requesting data from a HyperCat catalogue you must provide a dataset href.')
-
- location = dataset
- r = requests.get(location,
- params=query_params,
- auth=requests.auth.HTTPBasicAuth(self.api_key, ''))
- return r.text
-
- def get_datasets(self,
- query_params: typing.Optional[typing.Mapping[str, str]] = None):
- response = self._response
- if response is None:
- response = self._get_response(query_params)
-
- return [item['href'] for item in response['items']]
-
- # TODO should this be able to return metadata for multiple datasets at once?
- # TODO should there be a different method for getting catalogue metadata?
+ def __getitem__(self, item: str) -> BaseDataConnector:
+ params = {
+ 'href': item
+ }
+
+ response = self._get_response(params)
+
+ dataset_item = self._get_item_by_key_value(
+ response['items'],
+ 'href',
+ item
+ )
+ metadata = dataset_item['item-metadata']
+
+ try:
+ try:
+ content_type = self._get_item_by_key_value(
+ metadata,
+ 'rel',
+ 'urn:X-hypercat:rels:isContentType'
+ )['val']
+
+ except KeyError:
+ content_type = self._get_item_by_key_value(
+ metadata,
+ 'rel',
+ 'urn:X-hypercat:rels:containsContentType'
+ )['val']
+
+ if content_type == 'application/vnd.hypercat.catalogue+json':
+ return type(self)(location=item,
+ api_key=self.api_key,
+ auth=self.auth)
+
+ except (KeyError, ValueError):
+ # Has no or multiple values for content type - is not a catalogue
+ pass
+
+ return self.dataset_connector_class(item, self.api_key,
+ auth=self.auth,
+ metadata=metadata)
+
+ def items(self,
+ params=None) -> typing.ItemsView:
+ """
+ Get key-value pairs of dataset ID to dataset connector for datasets contained within this catalogue.
+
+ :param params: Query parameters to be passed through to the data source API
+ :return: Dictionary ItemsView over datasets
+ """
+ response = self._get_response(params)
+
+ return {
+ item['href']: self.dataset_connector_class(item['href'], self.api_key,
+ auth=self.auth,
+ metadata=item['item-metadata'])
+ for item in response['items']
+ }.items()
+
+ # TODO this gets the entire HyperCat contents so is slow on the BT HyperCat API - ~1s
def get_metadata(self,
- dataset: typing.Optional[str] = None,
- query_params: typing.Optional[typing.Mapping[str, str]] = None):
- if query_params is None:
- query_params = {}
-
- if dataset is not None:
- # Copy so we can update without side effect
- query_params = dict(query_params)
- query_params['href'] = dataset
-
- # Use cached response if we have one
- response = self._response
- if response is None:
- response = self._get_response(query_params)
-
- if dataset is None:
- metadata = response['catalogue-metadata']
+ params: typing.Optional[typing.Mapping[str, str]] = None):
+ response = self._get_response(params)
- else:
- dataset_item = self._get_item_by_key_value(
- response['items'],
- 'href',
- dataset
- )
- metadata = dataset_item['item-metadata']
+ return response['catalogue-metadata']
- metadata_dict = {}
- for item in metadata:
- relation = item['rel']
- value = item['val']
+ def get_datasets(self,
+ params: typing.Optional[typing.Mapping[str, str]] = None) -> typing.List[str]:
+ response = self._get_response(params=params)
- if relation not in metadata_dict:
- metadata_dict[relation] = []
- metadata_dict[relation].append(value)
+ datasets = []
+ for item in response['items']:
+ datasets.append(item['href'])
- return metadata_dict
+ return datasets
@staticmethod
def _get_item_by_key_value(collection: typing.Iterable[typing.Mapping],
- key: str, value) -> typing.Mapping:
+ key: str, value: typing.Any) -> typing.Mapping:
matches = [item for item in collection if item[key] == value]
if not matches:
@@ -85,39 +108,21 @@ class HyperCat(DataConnectorContainsDatasets, DataConnectorHasMetadata, BaseData
return matches[0]
- def _get_response(self, query_params: typing.Optional[typing.Mapping[str, str]] = None):
- # r = requests.get(self.location, params=query_params)
- r = self._get_auth_request(self.location,
- query_params=query_params)
- return r.json()
-
- def _get_auth_request(self, url, **kwargs):
- return requests.get(url,
- auth=requests.auth.HTTPBasicAuth(self.api_key, ''),
- **kwargs)
+ def _get_response(self, params: typing.Optional[typing.Mapping[str, str]] = None) -> typing.Mapping:
+ # Use cached response if we have one
+ # TODO should we use cached responses?
+ if self._response is not None and params is None:
+ # Ignore params - they only filter - we already have everything
+ response = self._response
+ else:
+ response = self._get_auth_request(self.location,
+ params=params)
+ response.raise_for_status()
+ return response.json()
def __enter__(self):
- self._response = self._get_response()
+ self._response = self._get_auth_request(self.location)
return self
def __exit__(self, exc_type, exc_val, exc_tb):
pass
-
-
-class HyperCatCisco(HyperCat):
- def __init__(self, location: str,
- api_key: typing.Optional[str] = None,
- entity_url: str = None):
- super().__init__(location, api_key=api_key)
-
- self.entity_url = entity_url
-
- def get_entities(self):
- r = self._get_auth_request(self.entity_url)
- return r.json()
-
- def _get_auth_request(self, url, **kwargs):
- return requests.get(url,
- # Doesn't accept HttpBasicAuth
- headers={'Authorization': self.api_key},
- **kwargs)
diff --git a/datasources/connectors/iotuk.py b/datasources/connectors/iotuk.py
deleted file mode 100644
index 67a8f485edf8233fd044ef70fd615c6bca312619..0000000000000000000000000000000000000000
--- a/datasources/connectors/iotuk.py
+++ /dev/null
@@ -1,20 +0,0 @@
-import typing
-
-import requests
-
-from .base import BaseDataConnector
-
-
-class IoTUK(BaseDataConnector):
- def get_data(self,
- dataset: typing.Optional[str] = None,
- query_params: typing.Optional[typing.Mapping[str, str]] = None):
- """
- Get data from a source using the IoTUK Nation API.
-
- :param dataset: Optional dataset for which to get data
- :param query_params: Optional query parameter filters
- :return: Requested data
- """
- r = requests.get(self.location, params=query_params)
- return r.json()
diff --git a/datasources/connectors/postcode_lookup.py b/datasources/connectors/postcode_lookup.py
new file mode 100644
index 0000000000000000000000000000000000000000..6d4c95a670bab9b224c256d198d0ff618d935467
--- /dev/null
+++ b/datasources/connectors/postcode_lookup.py
@@ -0,0 +1,106 @@
+"""
+This module contains a connector for UK postcode lookup.
+
+Run as module with --import <csv file> to import a postcode database CSV.
+"""
+
+import csv
+import sys
+import typing
+
+from django.http import JsonResponse
+
+from decouple import config
+import sqlalchemy
+from sqlalchemy.exc import NoSuchTableError
+import sqlalchemy.orm
+
+from .base import DataSetConnector
+
+
+class OnsPostcodeDirectoryConnector(DataSetConnector):
+ """
+ Connector for UK postcode lookup, backed by an SQL table.
+ """
+ _table_name = 'connector_postcode'
+
+ def __init__(self, location: str,
+ api_key: typing.Optional[str] = None,
+ auth: typing.Optional[typing.Callable] = None):
+ super().__init__(location, api_key=api_key, auth=auth)
+
+ self._engine = sqlalchemy.create_engine(config('DATABASE_URL'))
+ self._session_maker = sqlalchemy.orm.sessionmaker(bind=self._engine)
+
+ try:
+ self._table_meta = sqlalchemy.MetaData(self._engine)
+ self._table = sqlalchemy.Table(self._table_name, self._table_meta, autoload=True)
+
+ except NoSuchTableError as exc:
+ raise FileNotFoundError('Postcode table is not present') from exc
+
+ def get_response(self,
+ params: typing.Optional[typing.Mapping[str, str]] = None):
+ if params is None or 'postcode' not in params:
+ return JsonResponse({
+ 'status': 'fail',
+ 'data': {
+ 'postcode': 'Field \'postcode\' is a required field',
+ },
+ }, status=400)
+
+ query = sqlalchemy.select(
+ [self._table]
+ ).where(self._table.c.postcode == params['postcode'].replace(' ', '').upper())
+
+ result = self._session_maker().execute(query).fetchone()
+
+ try:
+ return JsonResponse(dict(result), json_dumps_params={'default': str})
+
+ except TypeError:
+ # Did not return a valid result
+ return JsonResponse({
+ 'status': 'fail',
+ 'data': {
+ 'postcode': 'No record matching postcode \'{0}\' found'.format(params['postcode']),
+ },
+ }, status=404)
+
+ @classmethod
+ def setup(cls, filename):
+ engine = sqlalchemy.create_engine(config('DATABASE_URL'))
+
+ metadata = sqlalchemy.MetaData(engine)
+ postcodes = sqlalchemy.Table(
+ cls._table_name, metadata,
+ sqlalchemy.Column('postcode', sqlalchemy.String(length=10), index=True, nullable=False, primary_key=True),
+ sqlalchemy.Column('lat', sqlalchemy.Float, nullable=False),
+ sqlalchemy.Column('long', sqlalchemy.Float, nullable=False)
+ )
+
+ try:
+ postcodes.create()
+
+ except sqlalchemy.exc.OperationalError:
+ pass
+
+ conn = engine.connect()
+
+ with open(filename, 'r') as csvfile:
+ reader = csv.DictReader(csvfile)
+
+ # TODO this fails if any row already exists - but checking each row in turn is slow - find solution
+ conn.execute(postcodes.insert(), [
+ {'postcode': row['pcds'].replace(' ', '').upper(),
+ 'lat': row['lat'],
+ 'long': row['long']} for row in reader
+ ])
+
+
+if __name__ == '__main__':
+ if len(sys.argv) == 3 and sys.argv[1] == '--import':
+ OnsPostcodeDirectoryConnector.setup(sys.argv[2])
+
+ else:
+ print(__doc__)
diff --git a/datasources/connectors/rest_api.py b/datasources/connectors/rest_api.py
new file mode 100644
index 0000000000000000000000000000000000000000..0632752ef09b4592b736821ad2c5d22f73e28b2c
--- /dev/null
+++ b/datasources/connectors/rest_api.py
@@ -0,0 +1,66 @@
+"""
+This module contains a data connector class for retrieving data via a REST API.
+"""
+
+import typing
+import urllib.parse
+
+from .base import BaseDataConnector, DataCatalogueConnector
+
+
+class RestApiConnector(DataCatalogueConnector):
+ """
+ Data connector for retrieving data from a REST API.
+ """
+ def __init__(self, location: str,
+ api_key: typing.Optional[str] = None,
+ auth: typing.Optional[typing.Callable] = None):
+ super().__init__(location, api_key=api_key, auth=auth)
+
+ self._response = None
+
+ def __getitem__(self, item: str) -> BaseDataConnector:
+ url = urllib.parse.urljoin(self.location, item)
+
+ return type(self)(location=url,
+ api_key=self.api_key,
+ auth=self.auth)
+
+ def get_metadata(self,
+ params: typing.Optional[typing.Mapping[str, str]] = None):
+ """
+ Metadata is not supported by this connector.
+ """
+ raise NotImplementedError('This data connector does not provide metadata')
+
+ def get_data(self,
+ params: typing.Optional[typing.Mapping[str, str]] = None):
+ """
+ Retrieve the data from this source.
+
+ If the data is JSON formatted it will be parsed into a dictionary - otherwise it will
+ be passed as plain text.
+
+ :param params: Query parameters to be passed through to the data source API
+ :return: Data source data
+ """
+ response = self.get_response(params)
+
+ if 'json' in response.headers['Content-Type']:
+ return response.json()
+
+ return response.text
+
+ def get_datasets(self,
+ params: typing.Optional[typing.Mapping[str, str]] = None) -> typing.List[str]:
+ """
+ Listing datasets is not supported by this connector.
+ """
+ raise NotImplementedError('This data connector does not provide a list of datasets')
+
+ def __enter__(self):
+ self._response = self._get_auth_request(self.location)
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ pass
diff --git a/datasources/forms.py b/datasources/forms.py
new file mode 100644
index 0000000000000000000000000000000000000000..a5477b241e523ab4c1b3376129adff893de15278
--- /dev/null
+++ b/datasources/forms.py
@@ -0,0 +1,108 @@
+from django import forms
+
+from requests.exceptions import ConnectionError
+
+from . import connectors, models
+
+
+# TODO arrange such that this doesn't need to be loaded each time
+connectors.BaseDataConnector.load_plugins('datasources/connectors')
+
+
+class DataSourceForm(forms.ModelForm):
+ """
+ Form class for creating / updating DataSource.
+ """
+ plugin_name = forms.ChoiceField(choices=connectors.BaseDataConnector.plugin_choices,
+ initial='DataSetConnector')
+
+ class Meta:
+ model = models.DataSource
+ exclude = ['auth_method', 'owner', 'users']
+
+ def clean(self):
+ cleaned_data = super().clean()
+
+ try:
+ cleaned_data['auth_method'] = models.DataSource.determine_auth_method(
+ cleaned_data['url'],
+ cleaned_data['api_key']
+ )
+
+ except ConnectionError:
+ raise forms.ValidationError('Could not authenticate against URL with provided API key.')
+
+ return cleaned_data
+
+ def clean_encrypted_docs_url(self):
+ """
+ Make sure that 'is_encrypted' and 'encrypted_docs_url' are always present as a pair.
+ """
+ if self.cleaned_data['encrypted_docs_url'] and not self.cleaned_data['is_encrypted']:
+ raise forms.ValidationError('You may not provide an encryption documentation URL if the data is not encrypted')
+
+ if self.cleaned_data['is_encrypted'] and not self.cleaned_data['encrypted_docs_url']:
+ raise forms.ValidationError('You must provide an encryption documentation URL is the data is encrypted')
+
+ return self.cleaned_data['encrypted_docs_url']
+
+
+class PermissionRequestForm(forms.ModelForm):
+ class Meta:
+ model = models.UserPermissionLink
+ fields = ['user', 'requested', 'push_requested', 'reason']
+ widgets = {
+ 'reason': forms.Textarea
+ }
+ help_texts = {
+ 'user': 'You may request permission for yourself or on behalf of any applications for which you are responsible.',
+ 'push_requested': 'Do you also require permission to push data?',
+ }
+
+ def __init__(self, *args, **kwargs):
+ user_choices = kwargs.pop('user_choices')
+ user_initial = kwargs.pop('user_initial')
+ user_field_hidden = kwargs.pop('user_field_hidden')
+
+ super().__init__(*args, **kwargs)
+
+ self.fields['user'].choices = user_choices
+ self.fields['user'].initial = user_initial
+
+ if user_field_hidden:
+ self.fields['user'].widget = forms.HiddenInput()
+
+ def save(self, commit=True):
+ """
+ Save this permission request.
+
+ Because (user, datasource) are unique, if the user is changed we need to get the corresponding record.
+ """
+ record, created = models.UserPermissionLink.objects.get_or_create(
+ user=self.instance.user,
+ datasource=self.instance.datasource
+ )
+
+ record.requested = self.instance.requested
+ record.push_requested = self.instance.push_requested
+ record.reason = self.instance.reason
+
+ record.save()
+
+
+class PermissionGrantForm(forms.ModelForm):
+ class Meta:
+ model = models.UserPermissionLink
+ fields = ['granted', 'push_granted']
+
+
+class MetadataFieldForm(forms.ModelForm):
+ class Meta:
+ model = models.MetadataItem
+ fields = ['field', 'value']
+
+
+class LicenceForm(forms.ModelForm):
+ class Meta:
+ model = models.Licence
+ exclude = ['owner']
diff --git a/datasources/management/__init__.py b/datasources/management/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/datasources/management/commands/__init__.py b/datasources/management/commands/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/datasources/management/commands/reset_api_count.py b/datasources/management/commands/reset_api_count.py
new file mode 100644
index 0000000000000000000000000000000000000000..628ddf959de3c449fd022ab599e3f8706e52863a
--- /dev/null
+++ b/datasources/management/commands/reset_api_count.py
@@ -0,0 +1,14 @@
+from django.core.management.base import BaseCommand
+
+from datasources.models import DataSource
+
+
+class Command(BaseCommand):
+ help = 'Resets external API call count on all data sources'
+
+ def handle(self, *args, **options):
+ for datasource in DataSource.objects.all():
+ datasource.external_requests = 0
+ datasource.save()
+
+ self.stdout.write(self.style.SUCCESS('Successfully reset count for data source "%s"' % datasource.pk))
diff --git a/datasources/migrations/0006_one_to_one_group.py b/datasources/migrations/0006_one_to_one_group.py
new file mode 100644
index 0000000000000000000000000000000000000000..e163ea68d7e76f22384c3726876b09fcfed7bc1d
--- /dev/null
+++ b/datasources/migrations/0006_one_to_one_group.py
@@ -0,0 +1,24 @@
+# Generated by Django 2.0.8 on 2018-09-25 12:35
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('datasources', '0005_datasource_plugin_name'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='datasource',
+ name='users_group',
+ field=models.OneToOneField(blank=True, editable=False, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='auth.Group'),
+ ),
+ migrations.AlterField(
+ model_name='datasource',
+ name='users_group_requested',
+ field=models.OneToOneField(blank=True, editable=False, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='auth.Group'),
+ ),
+ ]
diff --git a/datasources/migrations/0007_datasource_api_key.py b/datasources/migrations/0007_datasource_api_key.py
new file mode 100644
index 0000000000000000000000000000000000000000..172c36aa6f0f69d8190eb1a1f9945be053800ee5
--- /dev/null
+++ b/datasources/migrations/0007_datasource_api_key.py
@@ -0,0 +1,18 @@
+# Generated by Django 2.0.8 on 2018-10-10 12:25
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('datasources', '0006_one_to_one_group'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='datasource',
+ name='api_key',
+ field=models.CharField(blank=True, max_length=127),
+ ),
+ ]
diff --git a/datasources/migrations/0008_datasource__connector_string.py b/datasources/migrations/0008_datasource__connector_string.py
new file mode 100644
index 0000000000000000000000000000000000000000..4145862e2da597838922cb2576fd2c3304642eb5
--- /dev/null
+++ b/datasources/migrations/0008_datasource__connector_string.py
@@ -0,0 +1,18 @@
+# Generated by Django 2.0.8 on 2018-10-29 13:20
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('datasources', '0007_datasource_api_key'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='datasource',
+ name='_connector_string',
+ field=models.CharField(blank=True, max_length=255),
+ ),
+ ]
diff --git a/datasources/migrations/0009_unset_url_required.py b/datasources/migrations/0009_unset_url_required.py
new file mode 100644
index 0000000000000000000000000000000000000000..b55e32aece0463922a8bcfc8b2f0b9eb6d1aad4d
--- /dev/null
+++ b/datasources/migrations/0009_unset_url_required.py
@@ -0,0 +1,18 @@
+# Generated by Django 2.0.8 on 2018-10-29 13:34
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('datasources', '0008_datasource__connector_string'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='datasource',
+ name='url',
+ field=models.URLField(blank=True),
+ ),
+ ]
diff --git a/datasources/migrations/0010_datasource_auth_method.py b/datasources/migrations/0010_datasource_auth_method.py
new file mode 100644
index 0000000000000000000000000000000000000000..2d5b88e56c165b9d31359f46001b9f1aadcb2430
--- /dev/null
+++ b/datasources/migrations/0010_datasource_auth_method.py
@@ -0,0 +1,18 @@
+# Generated by Django 2.0.8 on 2018-10-31 14:00
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('datasources', '0009_unset_url_required'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='datasource',
+ name='auth_method',
+ field=models.IntegerField(choices=[('NONE', -1), ('UNKNOWN', 0), ('BASIC', 1), ('HEADER', 2)], default=0, editable=False),
+ ),
+ ]
diff --git a/datasources/migrations/0011_datasource_permissions_table.py b/datasources/migrations/0011_datasource_permissions_table.py
new file mode 100644
index 0000000000000000000000000000000000000000..2e74f808122a0bb8129c717824f9dae6338f33fc
--- /dev/null
+++ b/datasources/migrations/0011_datasource_permissions_table.py
@@ -0,0 +1,75 @@
+# Generated by Django 2.0.8 on 2018-11-06 15:29
+
+import datasources.models
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+def datasource_permissions_forward(apps, schema_editor):
+ """
+ Copy user permissions from old group-based form to new linking-table form.
+ """
+ DataSource = apps.get_model('datasources', 'DataSource')
+ UserPermissionLink = apps.get_model('datasources', 'UserPermissionLink')
+
+ for data_source in DataSource.objects.all():
+ if data_source.access_control:
+ for user in data_source.users_group.user_set.all():
+ UserPermissionLink.objects.get_or_create(
+ user=user,
+ datasource=data_source,
+ granted=datasources.models.UserPermissionLevels.VIEW
+ )
+
+ for user in data_source.users_group_requested.user_set.all():
+ link, created = UserPermissionLink.objects.get_or_create(
+ user=user,
+ datasource=data_source,
+ )
+ if not link.granted >= datasources.models.UserPermissionLevels.VIEW:
+ link.requested = datasources.models.UserPermissionLevels.VIEW
+ link.save()
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+ ('datasources', '0010_datasource_auth_method'),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name='UserPermissionLink',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('granted', models.IntegerField(choices=[('NONE', 0), ('VIEW', 1), ('META', 2), ('DATA', 3), ('PROV', 4)], default=datasources.models.UserPermissionLevels(0))),
+ ('requested', models.IntegerField(choices=[('NONE', 0), ('VIEW', 1), ('META', 2), ('DATA', 3), ('PROV', 4)], default=datasources.models.UserPermissionLevels(0))),
+ ],
+ ),
+ migrations.AddField(
+ model_name='userpermissionlink',
+ name='datasource',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='datasources.DataSource'),
+ ),
+ migrations.AddField(
+ model_name='userpermissionlink',
+ name='user',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
+ ),
+ migrations.AddField(
+ model_name='datasource',
+ name='users',
+ field=models.ManyToManyField(through='datasources.UserPermissionLink', to=settings.AUTH_USER_MODEL),
+ ),
+ migrations.RunPython(datasource_permissions_forward),
+ migrations.RemoveField(
+ model_name='datasource',
+ name='users_group',
+ ),
+ migrations.RemoveField(
+ model_name='datasource',
+ name='users_group_requested',
+ ),
+ ]
diff --git a/datasources/migrations/0012_add_request_reason.py b/datasources/migrations/0012_add_request_reason.py
new file mode 100644
index 0000000000000000000000000000000000000000..b73cf3988311032ec3d31aa6979f8d92e84b05c9
--- /dev/null
+++ b/datasources/migrations/0012_add_request_reason.py
@@ -0,0 +1,29 @@
+# Generated by Django 2.0.8 on 2018-11-08 09:29
+
+import datasources.models
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('datasources', '0011_datasource_permissions_table'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='userpermissionlink',
+ name='reason',
+ field=models.CharField(blank=True, max_length=511),
+ ),
+ migrations.AlterField(
+ model_name='userpermissionlink',
+ name='granted',
+ field=models.IntegerField(choices=[(0, 'NONE'), (1, 'VIEW'), (2, 'META'), (3, 'DATA'), (4, 'PROV')], default=datasources.models.UserPermissionLevels(0)),
+ ),
+ migrations.AlterField(
+ model_name='userpermissionlink',
+ name='requested',
+ field=models.IntegerField(choices=[(0, 'NONE'), (1, 'VIEW'), (2, 'META'), (3, 'DATA'), (4, 'PROV')], default=datasources.models.UserPermissionLevels(0)),
+ ),
+ ]
diff --git a/datasources/migrations/0013_is_encrypted.py b/datasources/migrations/0013_is_encrypted.py
new file mode 100644
index 0000000000000000000000000000000000000000..5a5c6d19a449395e75911a0b246f6f935dee9391
--- /dev/null
+++ b/datasources/migrations/0013_is_encrypted.py
@@ -0,0 +1,23 @@
+# Generated by Django 2.0.8 on 2018-11-12 08:59
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('datasources', '0012_add_request_reason'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='datasource',
+ name='is_encrypted',
+ field=models.BooleanField(default=False),
+ ),
+ migrations.AddField(
+ model_name='datasource',
+ name='encrypted_docs_url',
+ field=models.URLField(blank=True, verbose_name='Documentation URL for managing encrypted data'),
+ ),
+ ]
diff --git a/datasources/migrations/0014_access_control_levels.py b/datasources/migrations/0014_access_control_levels.py
new file mode 100644
index 0000000000000000000000000000000000000000..901fd89755d9dec9c7a6251ceedae22dfa0d5450
--- /dev/null
+++ b/datasources/migrations/0014_access_control_levels.py
@@ -0,0 +1,42 @@
+# Generated by Django 2.0.8 on 2018-11-16 13:43
+
+import datasources.models
+from django.db import migrations, models
+
+
+def access_control_forward(apps, schema_editor):
+ DataSource = apps.get_model('datasources', 'DataSource')
+
+ for data_source in DataSource.objects.all():
+ if data_source.access_control:
+ data_source.public_permission_level = datasources.models.UserPermissionLevels.NONE
+ data_source.save()
+
+
+def access_control_backward(apps, schema_editor):
+ DataSource = apps.get_model('datasources', 'DataSource')
+
+ for data_source in DataSource.objects.all():
+ if data_source.public_permission_level == datasources.models.UserPermissionLevels.NONE:
+ data_source.access_control = True
+ data_source.save()
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('datasources', '0013_is_encrypted'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='datasource',
+ name='public_permission_level',
+ field=models.IntegerField(choices=[(0, 'NONE'), (1, 'VIEW'), (2, 'META'), (3, 'DATA'), (4, 'PROV')], default=datasources.models.UserPermissionLevels(3)),
+ ),
+ migrations.RunPython(access_control_forward, access_control_backward),
+ migrations.RemoveField(
+ model_name='datasource',
+ name='access_control',
+ ),
+ ]
diff --git a/datasources/migrations/0015_add_dynamic_metadata.py b/datasources/migrations/0015_add_dynamic_metadata.py
new file mode 100644
index 0000000000000000000000000000000000000000..f82938fc1c4ecff9885b0df7b5cd58250ef8ae2f
--- /dev/null
+++ b/datasources/migrations/0015_add_dynamic_metadata.py
@@ -0,0 +1,34 @@
+# Generated by Django 2.0.8 on 2018-12-07 11:03
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('datasources', '0014_access_control_levels'),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name='MetadataField',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('name', models.CharField(max_length=63, unique=True)),
+ ],
+ ),
+ migrations.CreateModel(
+ name='MetadataItem',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('value', models.CharField(blank=True, max_length=511)),
+ ('datasource', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='metadata_items', to='datasources.DataSource')),
+ ('field', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='values', to='datasources.MetadataField')),
+ ],
+ ),
+ migrations.AlterUniqueTogether(
+ name='metadataitem',
+ unique_together={('field', 'datasource')},
+ ),
+ ]
diff --git a/datasources/migrations/0015_datasource_prov_exempt.py b/datasources/migrations/0015_datasource_prov_exempt.py
new file mode 100644
index 0000000000000000000000000000000000000000..c5202aca1f24dea8e22296b5b65244b261dc1aa8
--- /dev/null
+++ b/datasources/migrations/0015_datasource_prov_exempt.py
@@ -0,0 +1,18 @@
+# Generated by Django 2.0.8 on 2018-12-07 11:17
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('datasources', '0014_access_control_levels'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='datasource',
+ name='prov_exempt',
+ field=models.BooleanField(default=False, help_text='Should this data source be exempt from PROV tracking? This is useful for utility data sources which expect a large volume of queries, but are not interested in analysing usage patterns.'),
+ ),
+ ]
diff --git a/datasources/migrations/0016_metadatafield_short_name.py b/datasources/migrations/0016_metadatafield_short_name.py
new file mode 100644
index 0000000000000000000000000000000000000000..78132c2f37f5ac1a006105c9a6cd6c2179a69ce2
--- /dev/null
+++ b/datasources/migrations/0016_metadatafield_short_name.py
@@ -0,0 +1,20 @@
+# Generated by Django 2.0.8 on 2018-12-07 13:53
+
+import django.core.validators
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('datasources', '0015_add_dynamic_metadata'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='metadatafield',
+ name='short_name',
+ field=models.CharField(default='', max_length=63, unique=True, validators=[django.core.validators.RegexValidator('^[a-zA-Z][a-zA-Z0-9_]*\\Z', 'Short name must begin with a letter and consist only of letters, numbers and underscores.', 'invalid')]),
+ preserve_default=False,
+ ),
+ ]
diff --git a/datasources/migrations/0016_prov_exempt_help_text.py b/datasources/migrations/0016_prov_exempt_help_text.py
new file mode 100644
index 0000000000000000000000000000000000000000..05e114d89720f5c6d5def026365d1d143a4664b7
--- /dev/null
+++ b/datasources/migrations/0016_prov_exempt_help_text.py
@@ -0,0 +1,18 @@
+# Generated by Django 2.0.8 on 2018-12-07 11:23
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('datasources', '0015_datasource_prov_exempt'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='datasource',
+ name='prov_exempt',
+ field=models.BooleanField(default=False, help_text='Should this data source be exempt from PROV tracking? This is useful for utility data sources which expect a large volume of queries, but are not interested in analysing usage patterns. Note that this only disables tracking of data accesses, not of updates to the data source in PEDASI.'),
+ ),
+ ]
diff --git a/datasources/migrations/0017_count_external_requests.py b/datasources/migrations/0017_count_external_requests.py
new file mode 100644
index 0000000000000000000000000000000000000000..4a11b85b2fdb856cad4603c3cbde022c1c5b8c95
--- /dev/null
+++ b/datasources/migrations/0017_count_external_requests.py
@@ -0,0 +1,23 @@
+# Generated by Django 2.0.8 on 2018-12-13 09:57
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('datasources', '0016_prov_exempt_help_text'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='datasource',
+ name='external_requests',
+ field=models.PositiveIntegerField(default=0, editable=False),
+ ),
+ migrations.AddField(
+ model_name='datasource',
+ name='external_requests_total',
+ field=models.PositiveIntegerField(default=0, editable=False),
+ ),
+ ]
diff --git a/datasources/migrations/0018_editable_auth_method.py b/datasources/migrations/0018_editable_auth_method.py
new file mode 100644
index 0000000000000000000000000000000000000000..f420d7e9b5ed45c9445198a14ee2e85dfd337afa
--- /dev/null
+++ b/datasources/migrations/0018_editable_auth_method.py
@@ -0,0 +1,18 @@
+# Generated by Django 2.0.8 on 2018-12-19 11:48
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('datasources', '0017_count_external_requests'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='datasource',
+ name='auth_method',
+ field=models.IntegerField(choices=[('NONE', -1), ('UNKNOWN', 0), ('BASIC', 1), ('HEADER', 2)], default=0),
+ ),
+ ]
diff --git a/datasources/migrations/0019_auth_method_swap_key_value.py b/datasources/migrations/0019_auth_method_swap_key_value.py
new file mode 100644
index 0000000000000000000000000000000000000000..4d2ec8f15908dc006d2fd170e9218b10bbb425ae
--- /dev/null
+++ b/datasources/migrations/0019_auth_method_swap_key_value.py
@@ -0,0 +1,19 @@
+# Generated by Django 2.0.8 on 2018-12-19 11:58
+
+import datasources.connectors.base
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('datasources', '0018_editable_auth_method'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='datasource',
+ name='auth_method',
+ field=models.IntegerField(choices=[(-1, 'NONE'), (0, 'UNKNOWN'), (1, 'BASIC'), (2, 'HEADER')], default=datasources.connectors.base.AuthMethod(0)),
+ ),
+ ]
diff --git a/datasources/migrations/0020_merge_20190107_1508.py b/datasources/migrations/0020_merge_20190107_1508.py
new file mode 100644
index 0000000000000000000000000000000000000000..74af6a88e922a090de2188319915fc68f7869fbe
--- /dev/null
+++ b/datasources/migrations/0020_merge_20190107_1508.py
@@ -0,0 +1,14 @@
+# Generated by Django 2.0.8 on 2019-01-07 15:08
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('datasources', '0019_auth_method_swap_key_value'),
+ ('datasources', '0016_metadatafield_short_name'),
+ ]
+
+ operations = [
+ ]
diff --git a/datasources/migrations/0021_loosen_unique_metadata.py b/datasources/migrations/0021_loosen_unique_metadata.py
new file mode 100644
index 0000000000000000000000000000000000000000..206b01fde47f9ab4ab9e37863511289f660423bd
--- /dev/null
+++ b/datasources/migrations/0021_loosen_unique_metadata.py
@@ -0,0 +1,17 @@
+# Generated by Django 2.0.8 on 2019-01-10 16:27
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('datasources', '0020_merge_20190107_1508'),
+ ]
+
+ operations = [
+ migrations.AlterUniqueTogether(
+ name='metadataitem',
+ unique_together={('field', 'datasource', 'value')},
+ ),
+ ]
diff --git a/datasources/migrations/0022_datasource_is_deleted.py b/datasources/migrations/0022_datasource_is_deleted.py
new file mode 100644
index 0000000000000000000000000000000000000000..7449f37a4ce798485dd1c342a5fc3c241170f88b
--- /dev/null
+++ b/datasources/migrations/0022_datasource_is_deleted.py
@@ -0,0 +1,18 @@
+# Generated by Django 2.0.8 on 2019-01-11 14:11
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('datasources', '0021_loosen_unique_metadata'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='datasource',
+ name='is_deleted',
+ field=models.BooleanField(default=False, editable=False),
+ ),
+ ]
diff --git a/datasources/migrations/0023_relax_url_to_char.py b/datasources/migrations/0023_relax_url_to_char.py
new file mode 100644
index 0000000000000000000000000000000000000000..c8ee9956bb34e967553192f939f4d00ebde84d90
--- /dev/null
+++ b/datasources/migrations/0023_relax_url_to_char.py
@@ -0,0 +1,18 @@
+# Generated by Django 2.0.8 on 2019-01-16 11:02
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('datasources', '0022_datasource_is_deleted'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='datasource',
+ name='url',
+ field=models.CharField(blank=True, max_length=255),
+ ),
+ ]
diff --git a/datasources/migrations/0024_metadatafield_operational.py b/datasources/migrations/0024_metadatafield_operational.py
new file mode 100644
index 0000000000000000000000000000000000000000..ab4950fc1c248640487adca251425914dbbddc6e
--- /dev/null
+++ b/datasources/migrations/0024_metadatafield_operational.py
@@ -0,0 +1,18 @@
+# Generated by Django 2.0.8 on 2019-01-18 08:58
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('datasources', '0023_relax_url_to_char'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='metadatafield',
+ name='operational',
+ field=models.BooleanField(default=False),
+ ),
+ ]
diff --git a/datasources/migrations/0025_unique_together_permission_link.py b/datasources/migrations/0025_unique_together_permission_link.py
new file mode 100644
index 0000000000000000000000000000000000000000..29ef5f1d35e5adad23a5deee68632e3f3d9ccacd
--- /dev/null
+++ b/datasources/migrations/0025_unique_together_permission_link.py
@@ -0,0 +1,19 @@
+# Generated by Django 2.0.8 on 2019-01-25 11:27
+
+from django.conf import settings
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+ ('datasources', '0024_metadatafield_operational'),
+ ]
+
+ operations = [
+ migrations.AlterUniqueTogether(
+ name='userpermissionlink',
+ unique_together={('user', 'datasource')},
+ ),
+ ]
diff --git a/datasources/migrations/0026_userpermissionlink_push_data.py b/datasources/migrations/0026_userpermissionlink_push_data.py
new file mode 100644
index 0000000000000000000000000000000000000000..0f3021bc4e0b5ccb430ab403fb8c26eb21b4bd6d
--- /dev/null
+++ b/datasources/migrations/0026_userpermissionlink_push_data.py
@@ -0,0 +1,18 @@
+# Generated by Django 2.0.8 on 2019-01-25 14:40
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('datasources', '0025_unique_together_permission_link'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='userpermissionlink',
+ name='push_data',
+ field=models.BooleanField(default=False),
+ ),
+ ]
diff --git a/datasources/migrations/0027_add_push_granted.py b/datasources/migrations/0027_add_push_granted.py
new file mode 100644
index 0000000000000000000000000000000000000000..74411a94290d0d8e1ea0b975f24824534607440c
--- /dev/null
+++ b/datasources/migrations/0027_add_push_granted.py
@@ -0,0 +1,23 @@
+# Generated by Django 2.0.8 on 2019-01-25 14:50
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('datasources', '0026_userpermissionlink_push_data'),
+ ]
+
+ operations = [
+ migrations.RenameField(
+ model_name='userpermissionlink',
+ old_name='push_data',
+ new_name='push_requested',
+ ),
+ migrations.AddField(
+ model_name='userpermissionlink',
+ name='push_granted',
+ field=models.BooleanField(default=False),
+ ),
+ ]
diff --git a/datasources/migrations/0028_add_licence_model.py b/datasources/migrations/0028_add_licence_model.py
new file mode 100644
index 0000000000000000000000000000000000000000..42c40f54479a635834af824bf7f18aa9402fb617
--- /dev/null
+++ b/datasources/migrations/0028_add_licence_model.py
@@ -0,0 +1,33 @@
+# Generated by Django 2.0.8 on 2019-01-30 12:49
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('datasources', '0027_add_push_granted'),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name='License',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('name', models.CharField(max_length=63)),
+ ('short_name', models.CharField(blank=True, max_length=63)),
+ ('version', models.CharField(max_length=63)),
+ ('url', models.CharField(blank=True, max_length=255)),
+ ],
+ ),
+ migrations.AlterUniqueTogether(
+ name='license',
+ unique_together={('name', 'version')},
+ ),
+ migrations.AddField(
+ model_name='datasource',
+ name='license',
+ field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='datasources', to='datasources.License'),
+ ),
+ ]
diff --git a/datasources/migrations/0029_license_owner.py b/datasources/migrations/0029_license_owner.py
new file mode 100644
index 0000000000000000000000000000000000000000..08ad7fe7701936df62b1ce44a17a3551653041f3
--- /dev/null
+++ b/datasources/migrations/0029_license_owner.py
@@ -0,0 +1,22 @@
+# Generated by Django 2.0.8 on 2019-01-30 13:53
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+ ('datasources', '0028_add_licence_model'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='license',
+ name='owner',
+ field=models.ForeignKey(default=1, limit_choices_to={'groups__name': 'Data providers'}, on_delete=django.db.models.deletion.PROTECT, related_name='licences', to=settings.AUTH_USER_MODEL),
+ preserve_default=False,
+ ),
+ ]
diff --git a/datasources/migrations/0030_rename_licence.py b/datasources/migrations/0030_rename_licence.py
new file mode 100644
index 0000000000000000000000000000000000000000..3429e6e62ad12e532a55b530d98275dadf05a499
--- /dev/null
+++ b/datasources/migrations/0030_rename_licence.py
@@ -0,0 +1,24 @@
+# Generated by Django 2.0.8 on 2019-01-30 14:29
+
+from django.conf import settings
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+ ('datasources', '0029_license_owner'),
+ ]
+
+ operations = [
+ migrations.RenameModel(
+ old_name='License',
+ new_name='Licence',
+ ),
+ migrations.RenameField(
+ model_name='datasource',
+ old_name='license',
+ new_name='licence',
+ ),
+ ]
diff --git a/datasources/models.py b/datasources/models.py
index 7681ccea7c2f52b6d5a7bcbaef56e7db495505b0..c95627b513885a0ce2149e2e135bac3c5d871c4e 100644
--- a/datasources/models.py
+++ b/datasources/models.py
@@ -1,10 +1,186 @@
+import contextlib
+import enum
+import json
+import typing
+
from django.conf import settings
from django.contrib.auth.models import Group
+from django.core import validators
from django.db import models
from django.urls import reverse
+import requests
+import requests.exceptions
+
+from core.models import BaseAppDataModel, MAX_LENGTH_API_KEY, MAX_LENGTH_NAME, MAX_LENGTH_PATH, SoftDeletionManager
+from datasources.connectors.base import AuthMethod, BaseDataConnector, REQUEST_AUTH_FUNCTIONS
+
+#: Length of request reason field - must include brief description of project
+MAX_LENGTH_REASON = 511
+
+
+class Licence(models.Model):
+ """
+ Model representing a licence under which a data source is published e.g. Open Government Licence.
+ """
+
+ #: User who has responsibility for this licence
+ owner = models.ForeignKey(settings.AUTH_USER_MODEL,
+ limit_choices_to={
+ 'groups__name': 'Data providers'
+ },
+ on_delete=models.PROTECT,
+ related_name='licences',
+ blank=False, null=False)
+
+ #: Name of the licence - e.g. Open Government License
+ name = models.CharField(max_length=MAX_LENGTH_NAME,
+ blank=False, null=False)
+
+ #: Short text identifier - e.g. OGL
+ short_name = models.CharField(max_length=MAX_LENGTH_NAME,
+ blank=True, null=False)
+
+ #: Licence version - e.g. v2.0
+ version = models.CharField(max_length=MAX_LENGTH_NAME,
+ blank=False, null=False)
-from datasources.connectors.base import BaseDataConnector
-from pedasi.common.base_models import BaseAppDataModel, MAX_LENGTH_NAME
+ #: Address at which the licence text may be accessed
+ url = models.CharField(max_length=MAX_LENGTH_PATH,
+ blank=True, null=False)
+
+ class Meta:
+ unique_together = (('name', 'version'),)
+
+ def __str__(self):
+ return self.name
+
+ def get_absolute_url(self):
+ return reverse('datasources:licence.detail', kwargs={'pk': self.pk})
+
+
+class MetadataField(models.Model):
+ """
+ A metadata field that can be dynamically added to a data source.
+
+ Operational MetadataFields are those which have some associated code within PEDASI.
+ They should be present within any deployment of PEDASI.
+
+ Current operational metadata fields are (by short_name):
+ - data_query_param
+ - indexed_field
+ """
+ #: Name of the field
+ name = models.CharField(max_length=MAX_LENGTH_NAME,
+ unique=True,
+ blank=False, null=False)
+
+ #: Short text identifier for the field
+ short_name = models.CharField(max_length=MAX_LENGTH_NAME,
+ validators=[
+ validators.RegexValidator(
+ '^[a-zA-Z][a-zA-Z0-9_]*\Z',
+ 'Short name must begin with a letter and consist only of letters, numbers and underscores.',
+ 'invalid'
+ )
+ ],
+ unique=True,
+ blank=False, null=False)
+
+ # TODO create all operational fields if missing
+ #: Does the field have an operational effect within PEDASI?
+ operational = models.BooleanField(default=False,
+ blank=False, null=False)
+
+ def __str__(self):
+ return self.name
+
+
+class MetadataItem(models.Model):
+ """
+ The value of the metadata field on a given data source.
+ """
+ #: The value of this metadata field
+ value = models.CharField(max_length=MAX_LENGTH_REASON,
+ blank=True, null=False)
+
+ #: To which field does this relate?
+ field = models.ForeignKey(MetadataField,
+ related_name='values',
+ on_delete=models.PROTECT,
+ blank=False, null=False)
+
+ #: To which data source does this relate?
+ datasource = models.ForeignKey('DataSource',
+ related_name='metadata_items',
+ on_delete=models.CASCADE,
+ blank=False, null=False)
+
+ class Meta:
+ unique_together = (('field', 'datasource', 'value'),)
+
+ def __str__(self):
+ return self.value
+
+
+@enum.unique
+class UserPermissionLevels(enum.IntEnum):
+ """
+ User permission levels on data sources.
+ """
+ #: No permissions
+ NONE = 0
+
+ #: Permission to view in PEDASI UI
+ VIEW = 1
+
+ #: Permission to query metadata via API / UI
+ META = 2
+
+ #: Permission to query data via API / UI
+ DATA = 3
+
+ #: Permission to query PROV via API / UI
+ PROV = 4
+
+ @classmethod
+ def choices(cls):
+ return tuple((i.value, i.name) for i in cls)
+
+
+class UserPermissionLink(models.Model):
+ """
+ Model to act as a many to many joining table to handle user permission levels for access to data sources.
+ """
+ #: User being managed
+ user = models.ForeignKey(settings.AUTH_USER_MODEL,
+ on_delete=models.CASCADE)
+
+ #: Data source on which the permissions are being granted
+ datasource = models.ForeignKey('DataSource',
+ on_delete=models.CASCADE)
+
+ #: Granted permission level
+ granted = models.IntegerField(choices=UserPermissionLevels.choices(),
+ default=UserPermissionLevels.NONE,
+ blank=False, null=False)
+
+ #: Requested permission level
+ requested = models.IntegerField(choices=UserPermissionLevels.choices(),
+ default=UserPermissionLevels.NONE,
+ blank=False, null=False)
+
+ #: Have permission to push data?
+ push_granted = models.BooleanField(default=False)
+
+ #: Also require permission to push data?
+ push_requested = models.BooleanField(default=False)
+
+ #: Reason the permission was requested
+ reason = models.CharField(max_length=MAX_LENGTH_REASON,
+ blank=True, null=False)
+
+ class Meta:
+ unique_together = (('user', 'datasource'),)
class DataSource(BaseAppDataModel):
@@ -18,7 +194,12 @@ class DataSource(BaseAppDataModel):
* Track provenance of the data source itself
* Track provenance of data accesses
"""
- # TODO replace this with an admin group
+ objects = SoftDeletionManager()
+
+ #: Address at which the API may be accessed
+ url = models.CharField(max_length=MAX_LENGTH_PATH,
+ blank=True, null=False)
+
#: User who has responsibility for this data source
owner = models.ForeignKey(settings.AUTH_USER_MODEL,
limit_choices_to={
@@ -28,66 +209,238 @@ class DataSource(BaseAppDataModel):
related_name='datasources',
blank=False, null=False)
- #: Group of users who have explicit permission to use (query) this data source
- users_group = models.ForeignKey(Group,
- on_delete=models.SET_NULL,
- related_name='datasource',
- editable=False,
- blank=True, null=True)
-
- #: Groups of users who have requested explicit permission to use this data source
- users_group_requested = models.ForeignKey(Group,
- on_delete=models.SET_NULL,
- related_name='datasource_requested',
- editable=False,
- blank=True, null=True)
-
- #: Do users require explicit permission to use this data source?
- access_control = models.BooleanField(default=False,
- blank=False, null=False)
+ #: Information required to initialise the connector for this data source
+ _connector_string = models.CharField(max_length=MAX_LENGTH_PATH,
+ blank=True, null=False)
#: Name of plugin which allows interaction with this data source
plugin_name = models.CharField(max_length=MAX_LENGTH_NAME,
blank=False, null=False)
+ #: If the data source API requires an API key use this one
+ api_key = models.CharField(max_length=MAX_LENGTH_API_KEY,
+ blank=True, null=False)
+
+ #: Contains encrypted data?
+ is_encrypted = models.BooleanField(default=False,
+ blank=False, null=False)
+
+ #: Where to find information about how to use this encrypted data
+ encrypted_docs_url = models.URLField('Documentation URL for managing encrypted data',
+ blank=True, null=False)
+
+ #: Which authentication method to use - defined in :class:`datasources.connectors.base.AuthMethod` enum
+ auth_method = models.IntegerField(choices=AuthMethod.choices(),
+ default=AuthMethod.UNKNOWN,
+ blank=False, null=False)
+
+ #: Users - linked via a permission table - see :class:`UserPermissionLink`
+ users = models.ManyToManyField(settings.AUTH_USER_MODEL,
+ through=UserPermissionLink)
+
+ #: The level of access that users are assumed to have without gaining explicit permission
+ public_permission_level = models.IntegerField(choices=UserPermissionLevels.choices(),
+ default=UserPermissionLevels.DATA.value,
+ blank=False, null=False)
+
+ #: Is this data source exempt from PROV tracking - e.g. utility data sources - postcode lookup
+ prov_exempt = models.BooleanField(default=False,
+ help_text=(
+ 'Should this data source be exempt from PROV tracking? '
+ 'This is useful for utility data sources which expect a large volume '
+ 'of queries, but are not interested in analysing usage patterns. '
+ 'Note that this only disables tracking of data accesses, '
+ 'not of updates to the data source in PEDASI.'
+ ),
+ blank=False, null=False)
+
+ #: Which licence is this data published under
+ licence = models.ForeignKey(Licence,
+ related_name='datasources',
+ on_delete=models.PROTECT,
+ blank=True, null=True)
+
+ #: Total number of requests sent to the external API
+ external_requests_total = models.PositiveIntegerField(default=0,
+ editable=False, blank=False, null=False)
+
+ #: Number of requests sent to the external API since the last reset - reset at midnight by cron job
+ external_requests = models.PositiveIntegerField(default=0,
+ editable=False, blank=False, null=False)
+
+ #: Has this object been soft deleted?
+ is_deleted = models.BooleanField(default=False,
+ editable=False, blank=False, null=False)
+
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._data_connector = None
- @property
- def data_connector(self):
- if self._data_connector is None:
- BaseDataConnector.load_plugins('datasources/connectors')
- plugin = BaseDataConnector.get_plugin(self.plugin_name)
- self._data_connector = plugin(self.url)
-
- return self._data_connector
+ def delete(self, using=None, keep_parents=False):
+ """
+ Soft delete this object.
+ """
+ self.is_deleted = True
+ self.save()
def has_view_permission(self, user: settings.AUTH_USER_MODEL) -> bool:
"""
- Does a user have permission to use this data source?
+ Does a user have permission to view this data source in the PEDASI UI?
:param user: User to check
:return: User has permission?
"""
- if not self.access_control:
- return True
- if self.owner == user:
+ return self.has_permission_level(user, UserPermissionLevels.VIEW)
+
+ def has_permission_level(self, user: settings.AUTH_USER_MODEL, level: UserPermissionLevels) -> bool:
+ """
+ Does a user have a particular permission level on this data source?
+
+ :param user: User to check
+ :param level: Permission level to check for
+ :return: User has permission?
+ """
+ if self.public_permission_level >= level:
+ # Everyone has access
return True
- return self.users_group.user_set.filter(pk=user.pk).exists()
+ if self.owner == user or user.is_superuser:
+ return True
- def save(self, **kwargs):
- if self.access_control:
- # Create access control groups if they do not exist
- self.users_group, created = Group.objects.get_or_create(
- name=self.name + ' Users'
- )
- self.users_group_requested, created = Group.objects.get_or_create(
- name=self.name + ' Users Requested'
+ try:
+ permission = UserPermissionLink.objects.get(
+ user=user,
+ datasource=self
)
+ except (UserPermissionLink.DoesNotExist, TypeError):
+ # TypeError - user is not logged in
+ return False
+
+ return permission.granted >= level
+
+ def has_edit_permission(self, user: settings.AUTH_USER_MODEL) -> bool:
+ """
+ Does a given user have permission to edit this data source?
+
+ :param user: User to check
+ :return: User has permission to edit?
+ """
+ return user.is_superuser or user == self.owner
+
+ @property
+ def is_catalogue(self) -> bool:
+ return self.data_connector_class.is_catalogue
+
+ @property
+ def connector_string(self):
+ if self._connector_string:
+ return self._connector_string
+ return self.url
+
+ @property
+ def data_connector_class(self) -> typing.Type[BaseDataConnector]:
+ """
+ Get the data connector class for this source.
+
+ :return: Data connector class
+ """
+ BaseDataConnector.load_plugins('datasources/connectors')
+
+ try:
+ plugin = BaseDataConnector.get_plugin(self.plugin_name)
+
+ except KeyError as e:
+ if not self.plugin_name:
+ raise ValueError('Data source plugin is not set') from e
+
+ raise KeyError('Data source plugin not found') from e
+
+ return plugin
+
+ @property
+ @contextlib.contextmanager
+ def data_connector(self) -> BaseDataConnector:
+ """
+ Context manager to construct the data connector for this source.
+
+ :return: Data connector instance
+ """
+ if self._data_connector is None:
+ plugin = self.data_connector_class
+
+ if not self.api_key:
+ self._data_connector = plugin(self.connector_string)
+
+ else:
+ # Is the authentication method set?
+ auth_method = AuthMethod(self.auth_method)
+ if not auth_method:
+ auth_method = self.determine_auth_method(self.url, self.api_key)
+
+ # Inject function to get authenticated request
+ auth_class = REQUEST_AUTH_FUNCTIONS[auth_method]
+
+ self._data_connector = plugin(self.connector_string, self.api_key,
+ auth=auth_class)
+
+ try:
+ # Returns as context manager
+ yield self._data_connector
+
+ finally:
+ # Executed after the context manager is closed
+ self.external_requests += self._data_connector.request_count
+ self.external_requests_total += self._data_connector.request_count
+
+ self.save()
+
+ @property
+ def search_representation(self) -> str:
+ lines = [
+ self.name,
+ self.owner.get_full_name(),
+ self.description,
+ ]
+
+ try:
+ lines.append(json.dumps(
+ self.data_connector.get_metadata(),
+ indent=4
+ ))
+
+ except:
+ # KeyError: Plugin was not found
+ # NotImplementedError: Plugin does not support metadata
+ # ValueError: Plugin was not set
+ pass
+
+ result = '\n'.join(lines)
+ return result
+
+ @staticmethod
+ def determine_auth_method(url: str, api_key: str) -> AuthMethod:
+ # If not using an API key - can't require auth
+ if not api_key:
+ return AuthMethod.NONE
+
+ for auth_method_id, auth_function in REQUEST_AUTH_FUNCTIONS.items():
+ try:
+ # Can we get a response using this auth method?
+ if auth_function is None:
+ response = requests.get(url)
+
+ else:
+ response = requests.get(url,
+ auth=auth_function(api_key, ''))
+
+ response.raise_for_status()
+ return auth_method_id
+
+ except requests.exceptions.HTTPError:
+ pass
- super().save(**kwargs)
+ # None of the attempted authentication methods was successful
+ raise requests.exceptions.ConnectionError('Could not authenticate against external API')
def get_absolute_url(self):
return reverse('datasources:datasource.detail',
diff --git a/datasources/permissions.py b/datasources/permissions.py
new file mode 100644
index 0000000000000000000000000000000000000000..0d5e36966b103f3c703d95d974b8a7707f65927d
--- /dev/null
+++ b/datasources/permissions.py
@@ -0,0 +1,12 @@
+from django.contrib.auth.mixins import UserPassesTestMixin
+
+
+class HasPermissionLevelMixin(UserPassesTestMixin):
+ """
+ Mixin to reject users who do not have permission to view this DataSource.
+ """
+ #: Required permission level from datasources.models.UserPermissionLevels
+ permission_level = None
+
+ def test_func(self) -> bool:
+ return self.get_object().has_permission_level(self.request.user, self.permission_level)
diff --git a/datasources/search_indexes.py b/datasources/search_indexes.py
new file mode 100644
index 0000000000000000000000000000000000000000..bff3740e06661e11ae3006ce3afe160ebf23ebcd
--- /dev/null
+++ b/datasources/search_indexes.py
@@ -0,0 +1,10 @@
+from haystack import indexes
+
+from . import models
+
+
+class DataSourceIndex(indexes.SearchIndex, indexes.Indexable):
+ text = indexes.CharField(document=True, use_template=True)
+
+ def get_model(self):
+ return models.DataSource
diff --git a/datasources/serializers.py b/datasources/serializers.py
new file mode 100644
index 0000000000000000000000000000000000000000..232ecd4725377d62613a487c429cd879db21d5e3
--- /dev/null
+++ b/datasources/serializers.py
@@ -0,0 +1,42 @@
+from rest_framework import serializers
+
+from . import models
+
+
+class LicenceSerializer(serializers.ModelSerializer):
+ class Meta:
+ model = models.Licence
+ fields = ['name', 'short_name', 'version', 'url']
+
+
+class MetadataFieldSerializer(serializers.ModelSerializer):
+ class Meta:
+ model = models.MetadataField
+ fields = ['name', 'short_name']
+
+
+class MetadataItemSerializer(serializers.ModelSerializer):
+ field = MetadataFieldSerializer(read_only=True)
+
+ class Meta:
+ model = models.MetadataItem
+ fields = ['field', 'value']
+
+
+class DataSourceSerializer(serializers.ModelSerializer):
+ metadata_items = MetadataItemSerializer(many=True, read_only=True)
+ licence = LicenceSerializer(many=False, read_only=True)
+
+ class Meta:
+ model = models.DataSource
+ fields = [
+ 'id',
+ 'name',
+ 'description',
+ 'url',
+ 'plugin_name',
+ 'licence',
+ 'is_encrypted',
+ 'encrypted_docs_url',
+ 'metadata_items'
+ ]
diff --git a/datasources/static/js/explorer.js b/datasources/static/js/explorer.js
new file mode 100644
index 0000000000000000000000000000000000000000..030a102b22a147faaf2670cae9dec4fa9531afc0
--- /dev/null
+++ b/datasources/static/js/explorer.js
@@ -0,0 +1,312 @@
+"use strict";
+
+const params = new Map();
+
+let datasourceUrl = null;
+let selectedDataset = null;
+
+
+/**
+ * Set the root URL at which the data source may be accessed in the PEDASI API.
+ *
+ * @param url PEDASI API data source URL
+ */
+function setDatasourceUrl(url) {
+ "use strict";
+ datasourceUrl = url;
+}
+
+
+/**
+ * Append a row of text cells to a table.
+ *
+ * @param table Table to which row should be appended
+ * @param values Iterable of strings to append as new cells
+ */
+function tableAppendRow(table, values) {
+ "use strict";
+ const row = table.insertRow(-1);
+
+ for (const value of values) {
+ const cell = row.insertCell(-1);
+ const text = document.createTextNode(value);
+ cell.appendChild(text);
+ }
+
+ return row;
+}
+
+
+/**
+ * Get the base URL to use for requests to PEDASI API.
+ *
+ * @returns {string} PEDASI API URL
+ */
+function getBaseURL() {
+ "use strict";
+ let url = datasourceUrl;
+
+ if (selectedDataset !== null) {
+ url += "datasets/" + selectedDataset + "/";
+ }
+
+ return url;
+}
+
+
+/**
+ * Render request query parameters into table.
+ */
+function renderParams() {
+ "use strict";
+ const table = document.getElementById("tableParams");
+
+ /* Clear the table */
+ while (table.rows.length > 1) {
+ table.deleteRow(-1);
+ }
+
+ params.forEach(function (value, key, map) {
+ tableAppendRow(table, [key, value]);
+ });
+}
+
+
+/**
+ * Get the query params as a URL query string.
+ *
+ * @returns {string} Query param string
+ */
+function getQueryParamString() {
+ "use strict";
+ const encodedParams = new URLSearchParams();
+ params.forEach(function (value, key, map) {
+ encodedParams.set(key.trim(), value.trim());
+ });
+
+ return encodedParams.toString();
+}
+
+
+/**
+ * Display query param string in #queryParamSpan.
+ */
+function displayParams() {
+ "use strict";
+ const paramString = getQueryParamString();
+ const span = document.getElementById("queryParamSpan");
+
+ span.textContent = paramString;
+}
+
+
+/**
+ * Add a query parameter to the API query.
+ *
+ * @returns {boolean} return false to halt form processing
+ */
+function addParam() {
+ "use strict";
+ const param = document.getElementById("inputParam").value;
+ const value = document.getElementById("inputValue").value;
+
+ params.set(param, value);
+
+ renderParams();
+ displayParams();
+
+ return false;
+}
+
+
+/**
+ * Submit the prepared query to the PEDASI API and render the result into the 'results' panel.
+ */
+function submitQuery() {
+ "use strict";
+ const results = document.getElementById("queryResults");
+ const query = getBaseURL() + "data/?" + getQueryParamString();
+
+ fetch(query).then(function (response) {
+ const contentType = response.headers.get("content-type");
+ if (contentType && contentType.indexOf("application/json") !== -1) {
+ return response.json();
+ } else {
+ return response.text();
+ }
+ }).then(function (data) {
+ if (typeof data === "string") {
+ results.textContent = data;
+ } else {
+ results.textContent = JSON.stringify(data, null, 4);
+ }
+ });
+}
+
+
+/**
+ * Query the PEDASI API for data source internal metadata and render it into the 'metadataInternal' panel.
+ */
+function populateMetadataInternal() {
+ "use strict";
+
+ const table = document.getElementById("metadataInternal");
+
+ const url = getBaseURL();
+
+ fetch(url).then(function (response) {
+ if (response.ok) {
+ return response.json();
+ }
+ throw new Error("Internal metadata request failed.");
+ }).then(function (data) {
+ data.metadata_items.forEach(function (item) {
+ tableAppendRow(table, [item.field.name, item.value]);
+ });
+ }).catch(function (e) {
+ tableAppendRow(table, e.toString());
+ });
+}
+
+
+/**
+ * Query the PEDASI API for data source metadata and render it into the 'metadata' panel.
+ */
+function populateMetadata() {
+ "use strict";
+
+ const table = document.getElementById("metadata");
+
+ /* Clear the table */
+ while (table.rows.length > 0) {
+ table.deleteRow(-1);
+ }
+
+ const url = getBaseURL() + "metadata/";
+
+ fetch(url).then(
+ response => response.json()
+ ).then(function (data) {
+ if (data.status === "success") {
+ try{
+ data.data.forEach(function (item) {
+ tableAppendRow(table, [JSON.stringify(item, null, 4)]);
+ });
+ } catch (e) {
+ for (const key in data.data) {
+ tableAppendRow(table, [
+ key,
+ JSON.stringify(data.data[key], null, 4)
+ ]);
+ }
+ }
+ } else if (data.message) {
+ tableAppendRow(table, [data.message]);
+ }
+ }).catch(function (e) {
+ tableAppendRow(table, [e.toString()]);
+ });
+}
+
+
+/**
+ * Query the PEDASI API for the list of data sets within the data source and render into the 'datasets' panel.
+ */
+function populateDatasets() {
+ "use strict";
+ function rowAppendButton (row, item) {
+ const buttonCell = row.insertCell(-1);
+ const button = document.createElement("button");
+
+ button.id = "btn-" + item;
+ button.classList.add("btn", "btn-secondary", "btn-dataset");
+ button.addEventListener(
+ "click",
+ function () {
+ selectDataset(item);
+ }
+ );
+ button.textContent = "Select";
+ buttonCell.appendChild(button);
+ row.appendChild(buttonCell);
+ }
+
+ const url = getBaseURL() + "datasets/";
+ const table = document.getElementById("datasets");
+
+ fetch(url).then(
+ response => response.json()
+ ).then(function (data) {
+ if (data.status === "success") {
+ const row = tableAppendRow(table, ["Root"]);
+ rowAppendButton(row, null);
+
+ data.data.forEach(function (item) {
+ const row = tableAppendRow(table, [item]);
+ rowAppendButton(row, item)
+ });
+ } else if (data.message) {
+ tableAppendRow(table, [data.message]);
+ }
+ }).catch(function (e) {
+ tableAppendRow(table, [e.toString()]);
+ });
+}
+
+
+/**
+ * Change the active dataset.
+ *
+ * @param datasetId Dataset id to select
+ */
+function selectDataset(datasetId) {
+ "use strict";
+ selectedDataset = datasetId;
+ document.getElementById("selectedDataset").textContent = selectedDataset === null ? "None" : selectedDataset;
+
+ const urlComponentSpan = document.getElementById("datasetUrlSpan");
+ if (selectedDataset == null) {
+ urlComponentSpan.textContent = "";
+ } else {
+ urlComponentSpan.textContent = "datasets/" + selectedDataset + "/";
+ }
+
+ populateMetadata();
+
+ // Have to use for ... of ... loop since collection is live (updates with changes to DOM)
+ for (const button of document.getElementsByClassName("btn-dataset")) {
+ button.removeAttribute("disabled");
+ button.textContent = "Select";
+ }
+
+ const button = document.getElementById("btn-" + selectedDataset);
+ button.textContent = "Selected";
+ button.setAttribute("disabled", "true");
+}
+
+
+function toggleExpandPanel(e) {
+ "use strict";
+ const button = e.target;
+ const panel = document.querySelector(button.dataset.target);
+
+ if (panel === null) {
+ // Happens in MS Edge when a span receives the click event before the button it belongs to
+ return;
+ }
+
+ const iconPlus = button.querySelector(".toggle-icon-plus");
+ const iconMinus = button.querySelector(".toggle-icon-minus");
+
+ if ("expanded" in panel.dataset) {
+ panel.style.height = "30vh";
+ iconPlus.style.display = "inline";
+ iconMinus.style.display = "none";
+ delete panel.dataset.expanded;
+ } else {
+ panel.style.height = "100%";
+ iconPlus.style.display = "none";
+ iconMinus.style.display = "inline";
+ panel.dataset.expanded = "true";
+ }
+}
diff --git a/datasources/static/js/metadata.js b/datasources/static/js/metadata.js
new file mode 100644
index 0000000000000000000000000000000000000000..e05914501082d1d28d7b232df888db58053968c2
--- /dev/null
+++ b/datasources/static/js/metadata.js
@@ -0,0 +1,121 @@
+"use strict";
+
+let metadataUrl = null;
+
+
+/**
+ * Set the root URL at which the data source may be accessed in the PEDASI API.
+ *
+ * @param url PEDASI API data source URL
+ */
+function setMetadataUrl(url) {
+ "use strict";
+ metadataUrl = url;
+}
+
+
+function getCookie(name) {
+ const re = new RegExp("(^| )" + name + "=([^;]+)");
+ const match = document.cookie.match(re);
+ if (match) {
+ return match[2];
+ }
+}
+
+
+function tableAppendRow(table, values, id=null) {
+ const row = table.insertRow(-1);
+ if (id !== null) {
+ row.id = id;
+ }
+
+ for (const value of values) {
+ const cell = row.insertCell(-1);
+ const text = document.createTextNode(value);
+ cell.appendChild(text);
+ }
+
+ return row;
+}
+
+
+function postMetadata() {
+ fetch(
+ metadataUrl,
+ {
+ method: "POST",
+ body: JSON.stringify({
+ field: document.getElementById("id_field").value,
+ value: document.getElementById("id_value").value
+ }),
+ headers: {
+ "Accept": "application/json",
+ "Content-Type": "application/json",
+ "X-CSRFToken": getCookie("csrftoken")
+ }
+ }
+ ).then(
+ response => response.json()
+ ).then(
+ function (response) {
+ if (response.status === "success") {
+ const table = document.getElementById("tableMetadata");
+ const field = response.data.field;
+ const value = response.data.value;
+
+ const row = tableAppendRow(
+ table,
+ [field, value],
+ "metadata-row-" + response.data.field_short + "-" + value
+ );
+
+ const buttonCell = row.insertCell(-1);
+ const button = document.createElement("button");
+
+ button.id = "btn-" + field + "-" + value;
+ button.classList.add("btn", "btn-sm", "btn-danger", "float-right");
+ button.addEventListener(
+ "click",
+ function () {
+ const f = response.data.field_short;
+ const v = value;
+ deleteMetadata(f, v);
+ }
+ );
+ button.textContent = "Delete";
+ buttonCell.appendChild(button);
+ row.appendChild(buttonCell);
+ }
+ }
+ ).catch(
+ function (e) {
+ console.log(e);
+ }
+ )
+}
+
+function deleteMetadata(field, value) {
+ fetch(
+ metadataUrl,
+ {
+ method: "DELETE",
+ body: JSON.stringify({
+ field: field,
+ value: value
+ }),
+ headers: {
+ "Accept": "application/json",
+ "Content-Type": "application/json",
+ "X-CSRFToken": getCookie("csrftoken")
+ }
+ }
+ ).then(
+ response => response.json()
+ ).then(function (response) {
+ if (response.status === "success") {
+ const row = document.getElementById("metadata-row-" + field + "-" + value);
+ row.parentNode.removeChild(row);
+ }
+
+ })
+}
diff --git a/datasources/templates/datasources/datasource/create.html b/datasources/templates/datasources/datasource/create.html
new file mode 100644
index 0000000000000000000000000000000000000000..682d9cb005d954f1d451359871fc6b926626e770
--- /dev/null
+++ b/datasources/templates/datasources/datasource/create.html
@@ -0,0 +1,68 @@
+{% extends "base.html" %}
+{% load bootstrap4 %}
+
+{% block content %}
+ <nav aria-label="breadcrumb">
+ <ol class="breadcrumb">
+ <li class="breadcrumb-item" aria-current="page">
+ <a href="{% url 'index' %}">Home</a>
+ </li>
+ <li class="breadcrumb-item" aria-current="page">
+ <a href="{% url 'datasources:datasource.list' %}">Data Sources</a>
+ </li>
+ <li class="breadcrumb-item active" aria-current="page">
+ New Data Source
+ </li>
+ </ol>
+ </nav>
+
+ <h2 class="pb-3">New Data Source</h2>
+
+ <form class="form" method="post" action="">
+ {% csrf_token %}
+
+ {% bootstrap_field form.name %}
+ {% bootstrap_field form.description %}
+ {% bootstrap_field form.url %}
+ {% bootstrap_field form.api_key %}
+ {% bootstrap_field form.plugin_name %}
+ {% bootstrap_field form.licence %}
+
+ <hr>
+
+ <div class="row">
+ <div class="col-sm-3">
+ {% bootstrap_field form.is_encrypted %}
+ </div>
+
+ <div class="col-sm-9">
+ <div id="encryptedDocsToggle">
+ {% bootstrap_field form.encrypted_docs_url %}
+ </div>
+ </div>
+
+ <script type="application/javascript">
+ const checkboxEncrypted = document.getElementById("id_is_encrypted");
+ const divEncryptedToggle = document.getElementById("encryptedDocsToggle");
+
+ function setupEncryptedToggle() {
+ if (checkboxEncrypted.checked) {
+ divEncryptedToggle.style.display = "block";
+ } else {
+ divEncryptedToggle.style.display = "none";
+ }
+ }
+
+ checkboxEncrypted.addEventListener("change", setupEncryptedToggle);
+
+ setupEncryptedToggle();
+ </script>
+ </div>
+
+ {% bootstrap_field form.public_permission_level %}
+ {% bootstrap_field form.prov_exempt %}
+
+ <input type="submit" class="btn btn-success" value="Create">
+ </form>
+
+{% endblock %}
diff --git a/datasources/templates/datasources/datasource/dataset_search.html b/datasources/templates/datasources/datasource/dataset_search.html
new file mode 100644
index 0000000000000000000000000000000000000000..d7df90065980b6f69fbc3b4f04080d85d4dacc99
--- /dev/null
+++ b/datasources/templates/datasources/datasource/dataset_search.html
@@ -0,0 +1,42 @@
+{% for dataset_uri, dataset in datasets %}
+ <div class="col-md-12 p-2">
+ <a href="#"
+ class="no-hover" role="button">
+
+ <div class="card rounded-0 h-100">
+ <div class="card-body d-flex flex-column">
+ <h5 class="card-title">{{ dataset_uri }}</h5>
+
+ <div class="card-text">
+ <table class="table">
+ <thead>
+ <th scope="col" class="col-md-3 border-0"></th>
+ <th scope="col" class="border-0"></th>
+ </thead>
+
+ <tbody>
+ {% if metadata_type == 'list' %}
+ {% for meta in dataset.get_metadata %}
+ <tr>
+ <td>{{ meta.rel }}</td>
+ <td>{{ meta.val }}</td>
+ </tr>
+ {% endfor %}
+ {% else %}
+ {% for k, v in dataset.get_metadata.items %}
+ <tr>
+ <td>{{ k }}</td>
+ <td>{{ v }}</td>
+ </tr>
+ {% endfor %}
+ {% endif %}
+ </tbody>
+ </table>
+ </div>
+ </div>
+ </div>
+
+ </a>
+ </div>
+{% empty %}
+{% endfor %}
diff --git a/datasources/templates/datasources/datasource/delete.html b/datasources/templates/datasources/datasource/delete.html
new file mode 100644
index 0000000000000000000000000000000000000000..42778a993bd9012488ee977de8a03fddf6037f29
--- /dev/null
+++ b/datasources/templates/datasources/datasource/delete.html
@@ -0,0 +1,64 @@
+{% extends "base.html" %}
+{% load bootstrap4 %}
+
+{% block content %}
+ <nav aria-label="breadcrumb">
+ <ol class="breadcrumb">
+ <li class="breadcrumb-item" aria-current="page">
+ <a href="{% url 'index' %}">Home</a>
+ </li>
+ <li class="breadcrumb-item" aria-current="page">
+ <a href="{% url 'datasources:datasource.list' %}">Data Sources</a>
+ </li>
+ <li class="breadcrumb-item" aria-current="page">
+ <a href="{% url 'datasources:datasource.detail' pk=datasource.pk %}">{{ datasource.name }}</a>
+ </li>
+ <li class="breadcrumb-item active" aria-current="page">
+ Delete
+ </li>
+ </ol>
+ </nav>
+
+ <div class="row">
+ <div class="col-md-10 col-sm-8">
+ <h2>{{ datasource.name }}</h2>
+
+ {% if datasource.description %}
+ {{ datasource.description|linebreaks }}
+ {% endif %}
+ </div>
+ </div>
+
+ <table class="table">
+ <thead>
+ <th scope="col" class="w-25 border-0"></th>
+ <th scope="col" class="border-0"></th>
+ </thead>
+
+ <tbody>
+ <tr>
+ <td>Owner</td>
+ <td>
+ {{ datasource.owner }}
+ </td>
+ </tr>
+ <tr>
+ <td>URL</td>
+ <td>{{ datasource.url }}</td>
+ </tr>
+ </tbody>
+ </table>
+
+ <div class="alert alert-danger">
+ <p><b>Are you sure you want to delete this data source?</b></p>
+
+ <form class="form" method="post">
+ {% csrf_token %}
+
+ <input type="submit" role="button" class="btn btn-danger" value="Delete">
+
+ <a role="button" class="btn btn-info"
+ href="{% url 'datasources:datasource.detail' pk=datasource.pk %}">Cancel</a>
+ </form>
+ </div>
+{% endblock %}
\ No newline at end of file
diff --git a/datasources/templates/datasources/datasource/detail-no-access.html b/datasources/templates/datasources/datasource/detail-no-access.html
index bcd8b6c4737bca90aa161c9401581a4359db878e..cc993c4cc89ebb24939725b6c20e7d0be5298bb9 100644
--- a/datasources/templates/datasources/datasource/detail-no-access.html
+++ b/datasources/templates/datasources/datasource/detail-no-access.html
@@ -1,6 +1,10 @@
{% extends "base.html" %}
{% load bootstrap4 %}
+{% block extra_head %}
+ <script src="https://cdn.jsdelivr.net/npm/js-cookie@2.2.0/src/js.cookie.min.js"></script>
+{% endblock %}
+
{% block content %}
<nav aria-label="breadcrumb">
<ol class="breadcrumb">
@@ -16,17 +20,71 @@
</ol>
</nav>
- <h2>View Data Source - {{ datasource.name }}</h2>
+
+ <div class="row">
+ <div class="col-md-10 col-sm-8">
+ <h2>
+ {{ datasource.name }}
+ {% if datasource.licence %}
+ <small>
+ <a href="{% url 'datasources:licence.detail' pk=datasource.licence.pk %}"
+ class="badge badge-info"
+ data-toggle="tooltip" data-placement="bottom" title="{{ datasource.licence.name }}">
+ {{ datasource.licence.short_name }}
+ </a>
+ </small>
+ {% else %}
+ <small>
+ <span class="badge badge-warning"
+ data-toggle="tooltip" data-placement="bottom" title="No Licence">
+ No Licence
+ </span>
+ </small>
+ {% endif %}
+ </h2>
+
+ {% if datasource.description %}
+ {{ datasource.description|linebreaks }}
+ {% endif %}
+ </div>
+
+ <div class="col-md-2 col-sm-4">
+ {% if request.user != datasource.owner and not request.user.is_superuser %}
+ <a href="{% url 'datasources:datasource.access.request' pk=datasource.id %}"
+ class="btn btn-block btn-secondary" role="button">Manage My Access</a>
+ {% endif %}
+ </div>
+ </div>
<div class="alert alert-warning">
You do not have permission to access this resource.
- <a href="#"
- class="btn btn-primary" role="button">Request Access</a>
+ {% if request.user in datasource.users.all %}
+ Please wait for your request to be approved.
+
+ {% else %}
+ You may request access using the 'Manage My Access' button.
+ {% endif %}
</div>
- {% if datasource.description %}
- <p>{{ datasource.description }}</p>
- {% endif %}
+ <table class="table">
+ <thead>
+ <th scope="col" class="w-25 border-0"></th>
+ <th scope="col" class="border-0"></th>
+ </thead>
+
+ <tbody>
+ <tr>
+ <td>Owner</td>
+ <td>
+ {{ datasource.owner }}
+ </td>
+ </tr>
+ <tr>
+ <td>URL</td>
+ <td>{{ datasource.url }}</td>
+ </tr>
+ </tbody>
+ </table>
{% endblock %}
\ No newline at end of file
diff --git a/datasources/templates/datasources/datasource/detail.html b/datasources/templates/datasources/datasource/detail.html
index 4f258ba87a2717db0ca0231d725bdbc9f8997f57..67152ed7bf0d97367a3d78e7d2d64636e39c01e5 100644
--- a/datasources/templates/datasources/datasource/detail.html
+++ b/datasources/templates/datasources/datasource/detail.html
@@ -1,6 +1,18 @@
{% extends "base.html" %}
{% load bootstrap4 %}
+{% block extra_head %}
+ {% if metadata_field_form %}
+ {% load static %}
+
+ <script type="application/javascript" src="{% static 'js/metadata.js' %}"></script>
+
+ <script type="application/javascript">
+ setMetadataUrl("/datasources/{{ datasource.pk }}/metadata");
+ </script>
+ {% endif %}
+{% endblock %}
+
{% block content %}
<nav aria-label="breadcrumb">
<ol class="breadcrumb">
@@ -16,28 +28,204 @@
</ol>
</nav>
- <h2>View Data Source - {{ datasource.name }}</h2>
+ <div class="row">
+ <div class="col-md-10 col-sm-8">
+ <h2>
+ {{ datasource.name }}
+ {# TODO make this a template tag #}
+ {% if datasource.licence %}
+ <small>
+ <a href="{% url 'datasources:licence.detail' pk=datasource.licence.pk %}"
+ class="badge badge-info"
+ data-toggle="tooltip" data-placement="bottom" title="{{ datasource.licence.name }}">
+ {{ datasource.licence.short_name }}
+ </a>
+ </small>
+ {% else %}
+ <small>
+ <span class="badge badge-warning"
+ data-toggle="tooltip" data-placement="bottom" title="No Licence">
+ No Licence
+ </span>
+ </small>
+ {% endif %}
+ </h2>
+
+ {% if datasource.description %}
+ {{ datasource.description|linebreaks }}
+ {% endif %}
+ </div>
+
+ <div class="col-md-2 col-sm-4">
+ <a href="{% url 'datasources:datasource.explorer' pk=datasource.id %}"
+ class="btn btn-block btn-info" role="button">API Explorer</a>
+
+ {% if request.user != datasource.owner and not request.user.is_superuser %}
+ <a href="{% url 'datasources:datasource.access.request' pk=datasource.id %}"
+ class="btn btn-block btn-secondary" role="button">Manage My Access</a>
+ {% endif %}
- <p>
- Owner: <a href="#" role="link">{{ datasource.owner }}</a>
- </p>
+ {% if has_edit_permission %}
+ <a href="{% url 'datasources:datasource.access.manage' pk=datasource.pk %}"
+ class="btn btn-block btn-primary" role="button">Manage Access</a>
- {% if datasource.description %}
- <p>{{ datasource.description }}</p>
+ <a href="{% url 'datasources:datasource.edit' datasource.id %}"
+ class="btn btn-block btn-success" role="button">Edit</a>
+
+ <a href="{% url 'datasources:datasource.delete' datasource.id %}"
+ class="btn btn-block btn-danger" role="button">Delete</a>
+ {% endif %}
+ </div>
+ </div>
+
+ {% if datasource.is_encrypted %}
+ <div class="alert alert-warning mt-3">
+ <p>
+ This data source contains encrypted data.
+ </p>
+
+ {% if datasource.encrypted_docs_url %}
+ For guidance on how to process this data please see <a href="{{ datasource.encrypted_docs_url }}">{{ datasource.encrypted_docs_url }}</a>.
+ {% endif %}
+ </div>
{% endif %}
- <a href="{% url 'datasources:datasource.query' pk=datasource.id %}"
- class="btn btn-success" role="link">Query</a>
- <a href="{% url 'datasources:datasource.metadata' pk=datasource.id %}"
- class="btn btn-success" role="link">Metadata</a>
-
- <a href="{% url 'admin:datasources_datasource_change' datasource.id %}"
- class="btn btn-success" role="button">Edit</a>
- <a href="{% url 'admin:datasources_datasource_delete' datasource.id %}"
- class="btn btn-danger" role="button">Delete</a>
- {% if datasource.access_control %}
- <a href="{% url 'datasources:datasource.manage-access' pk=datasource.pk %}"
- class="btn btn-primary" role="button">Manage Access</a>
+ <table class="table">
+ <thead>
+ <th scope="col" class="w-25 border-0"></th>
+ <th scope="col" class="border-0"></th>
+ </thead>
+
+ <tbody>
+ <tr>
+ <td>Owner</td>
+ <td>
+ {{ datasource.owner }}
+ </td>
+ </tr>
+ <tr>
+ <td>URL</td>
+ <td>{{ datasource.url }}</td>
+ </tr>
+ <tr>
+ <td>Licence</td>
+ <td>{{ datasource.licence.name }}</td>
+ </tr>
+ </tbody>
+ </table>
+
+ <div class="card">
+ <div class="card-header" data-toggle="collapse" data-target="#collapseMetadata">
+ <h6>Metadata</h6>
+ </div>
+
+ <div id="collapseMetadata" class="card-body collapse show">
+ <table id="tableMetadata" class="table">
+ <thead>
+ <th scope="col" class="w-25 border-0"></th>
+ <th scope="col" class="border-0"></th>
+ {% if metadata_field_form %}
+ <th scope="col" class="w-25 border-0"></th>
+ {% endif %}
+ </thead>
+
+ <tbody>
+ {% for metadata_item in datasource.metadata_items.all %}
+ <tr id="metadata-row-{{ metadata_item.field.short_name }}-{{ metadata_item.value }}">
+ <td>{{ metadata_item.field.name }}</td>
+ <td>{{ metadata_item.value }}</td>
+
+ {% if metadata_field_form %}
+ <td>
+ <button role="button"
+ onclick="deleteMetadata('{{ metadata_item.field.short_name }}','{{ metadata_item.value }}')"
+ class="btn btn-sm btn-danger float-right">Delete</button>
+ </td>
+ {% endif %}
+ </tr>
+ {% endfor %}
+ </tbody>
+ </table>
+
+ {% if metadata_field_form %}
+ <form id="formMetadata" class="form" action="javascript:postMetadata();">
+ {% csrf_token %}
+
+ <div class="row">
+ <div class="col-3">
+ {% bootstrap_field metadata_field_form.field layout='inline' %}
+ </div>
+ <div class="col-6">
+ {% bootstrap_field metadata_field_form.value layout='inline' %}
+ </div>
+ <div class="col-3">
+ <button type="submit" class="btn btn-block btn-success">Add</button>
+ </div>
+ </div>
+ </form>
+ {% endif %}
+ </div>
+ </div>
+
+ <hr>
+
+ {% if is_catalogue %}
+ <h2 class="mt-3">Data Sets</h2>
+
+ <script type="application/javascript">
+ function search() {
+ // Give the user a visual cue that a search is happening
+ $('#btn-search-rest').hide();
+ $('#btn-search-active').show();
+
+ const endpoint = "search?q=" + $("#id_query").val();
+
+ // Load HTML response into element
+ $('#dataset-results').load(endpoint, function(response, status, xhr) {
+ if (status === "error") {
+ const msg = "There was an error: ";
+ console.log(msg + xhr.status + " " + xhr.statusText);
+ }
+
+ // Reset the button
+ $('#btn-search-rest').show();
+ $('#btn-search-active').hide();
+ });
+ }
+
+ function clearResults() {
+ $('#dataset-results').empty();
+ }
+ </script>
+
+ <form class="form-inline" action="javascript:search();">
+ <div class="input-group">
+ <input type="search" class="form-control" id="id_query"
+ aria-label="search" placeholder="Search">
+
+ <div class="input-group-append">
+ <button type="submit" class="btn btn-outline-primary">
+ <i id="btn-search-rest" class="fas fa-search"></i>
+ <i id="btn-search-active" class="fas fa-spinner fa-pulse" style="display: none"></i>
+ </button>
+ </div>
+
+ <div class="input-group-append">
+ <button type="button" class="btn btn-outline-primary" onclick="clearResults();"><i class="fas fa-trash"></i></button>
+ </div>
+ </div>
+ </form>
+
+ <div id="dataset-results" class="row px-2">
+ </div>
{% endif %}
-{% endblock %}
\ No newline at end of file
+{% endblock %}
+
+{% block extra_body %}
+ <script type="application/javascript">
+ $(function () {
+ $("[data-toggle='tooltip']").tooltip()
+ })
+ </script>
+{% endblock %}
diff --git a/datasources/templates/datasources/datasource/explorer.html b/datasources/templates/datasources/datasource/explorer.html
new file mode 100644
index 0000000000000000000000000000000000000000..dff92cde41b694e4db184265fc5f24fffcc0bde9
--- /dev/null
+++ b/datasources/templates/datasources/datasource/explorer.html
@@ -0,0 +1,244 @@
+{% load bootstrap4 %}
+
+<!DOCTYPE html>
+<html lang="en">
+<head>
+ <meta charset="UTF-8">
+ <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
+
+ <title>PEDASI</title>
+
+ {% load static %}
+ <link rel="shortcut icon" href="{% static 'img/favicon.ico' %}" type="image/vnd.microsoft.icon"/>
+
+ {% bootstrap_css %}
+
+ <!-- Font Awesome - https://fontawesome.com -->
+ <link rel="stylesheet"
+ href="https://use.fontawesome.com/releases/v5.2.0/css/solid.css"
+ integrity="sha384-wnAC7ln+XN0UKdcPvJvtqIH3jOjs9pnKnq9qX68ImXvOGz2JuFoEiCjT8jyZQX2z"
+ crossorigin="anonymous">
+
+ <link rel="stylesheet"
+ href="https://use.fontawesome.com/releases/v5.2.0/css/fontawesome.css"
+ integrity="sha384-HbmWTHay9psM8qyzEKPc8odH4DsOuzdejtnr+OFtDmOcIVnhgReQ4GZBH7uwcjf6"
+ crossorigin="anonymous">
+
+ <!-- BootSwatch Journal theme -->
+ <link href="https://stackpath.bootstrapcdn.com/bootswatch/4.1.3/journal/bootstrap.min.css"
+ rel="stylesheet"
+ integrity="sha384-5C8TGNupopdjruopVTTrVJacBbWqxHK9eis5DB+DYE6RfqIJapdLBRUdaZBTq7mE"
+ crossorigin="anonymous">
+
+ <link rel="stylesheet" href="{% static 'css/pedasi.css' %}">
+
+ <script type="application/javascript" src="{% static 'js/explorer.js' %}"></script>
+ <script type="application/javascript">
+ setDatasourceUrl("/api/datasources/{{ datasource.pk }}/");
+ </script>
+</head>
+
+<body>
+
+<nav class="navbar navbar-expand-lg navbar-dark bg-primary">
+ <div class="container-fluid">
+ <a class="navbar-brand" href="/">PEDASI</a>
+ <button class="navbar-toggler" type="button"
+ data-toggle="collapse" data-target="#navbarSupportedContent"
+ aria-controls="navbarSupportedContent" aria-expanded="false" aria-label="Toggle navigation">
+ <span class="navbar-toggler-icon"></span>
+ </button>
+
+ <div class="collapse navbar-collapse" id="navbarSupportedContent">
+ <ul class="navbar-nav mr-auto">
+ <li class="nav-item">
+ <a class="nav-link" href="{% url 'applications:application.list' %}">Applications</a>
+ </li>
+ <li class="nav-item">
+ <a class="nav-link" href="{% url 'datasources:datasource.list' %}">Data Sources</a>
+ </li>
+ </ul>
+
+ <ul class="navbar-nav">
+ {% if request.user.is_authenticated %}
+ <li class="nav-item">
+ <p class="navbar-text">
+ Welcome {{ request.user.username }}
+ </p>
+ </li>
+
+ <li class="nav-item">
+ <a class="nav-link" href="{% url 'profiles:profile' %}">My Profile</a>
+ </li>
+
+ <li class="nav-item">
+ <a class="nav-link" href="{% url 'profiles:logout' %}">Logout</a>
+ </li>
+
+ {% else %}
+ {% if 'google-oauth2' in backends.backends %}
+ <li class="nav-item">
+ <a class="nav-link" href="{% url 'social:begin' 'google-oauth2' %}">Google Login</a>
+ </li>
+ {% endif %}
+
+ <li class="nav-item">
+ <a class="nav-link" href="{% url 'profiles:login' %}">Login</a>
+ </li>
+ {% endif %}
+ </ul>
+ </div>
+ </div>
+</nav>
+
+<div class="container-fluid mt-3">
+ {% bootstrap_messages %}
+
+ <nav aria-label="breadcrumb">
+ <ol class="breadcrumb">
+ <li class="breadcrumb-item" aria-current="page">
+ <a href="{% url 'index' %}">Home</a>
+ </li>
+ <li class="breadcrumb-item" aria-current="page">
+ <a href="{% url 'datasources:datasource.list' %}">Data Sources</a>
+ </li>
+ <li class="breadcrumb-item" aria-current="page">
+ <a href="{% url 'datasources:datasource.detail' pk=datasource.pk %}">{{ datasource.name }}</a>
+ </li>
+ <li class="breadcrumb-item active" aria-current="page">
+ API Explorer
+ </li>
+ </ol>
+ </nav>
+
+ <div class="row">
+ <div class="col-md">
+ <h2>{{ datasource.name }}</h2>
+
+ {% if datasource.description %}
+ <p>{{ datasource.description|linebreaks }}</p>
+ {% endif %}
+ </div>
+ </div>
+
+ <hr>
+
+ {% if datasource.is_catalogue %}
+ <div class="row">
+ <div class="col-md">
+ <div class="alert alert-warning">
+ <h5>Active data set</h5>
+ <span id="selectedDataset">None</span>
+ </div>
+ </div>
+ </div>
+ {% endif %}
+
+ <div class="row">
+ <div class="col-md-6">
+ <h5>Query Builder</h5>
+
+ <form class="form-inline" onsubmit="return addParam();">
+ {% if data_query_params %}
+ <select class="form-control mb-2 mr-sm-2" id="inputParam">
+ {% for param in data_query_params %}
+ <option value="{{ param }}">{{ param }}</option>
+ {% endfor %}
+ </select>
+ {% else %}
+ <input type="text" class="form-control mb-2 mr-sm-2" id="inputParam" placeholder="Parameter">
+ {% endif %}
+
+ <input type="text" class="form-control mb-2 mr-sm-2" id="inputValue" placeholder="Parameter value">
+
+ <button type="submit" class="btn btn-secondary mb-2 mr-sm-2">Add to Query</button>
+ <button type="button" class="btn btn-success mb-2" onclick="submitQuery();">Submit Query</button>
+ </form>
+
+ <div class="alert alert-info w-100">
+ Query URL: /api/datasources/{{ datasource.pk }}/<span id="datasetUrlSpan"></span>data/?<span id="queryParamSpan"></span>
+ </div>
+
+ <table class="table" id="tableParams">
+ <thead>
+ <th>Parameter</th>
+ <th>Value</th>
+ </thead>
+ </table>
+ </div>
+
+ <div class="col-md-6">
+ <h5>Query Results</h5>
+
+ <pre id="queryResults" style="overflow-y: auto; height: 30vh;">
+ </pre>
+ </div>
+ </div>
+
+ <hr>
+
+ <div class="row">
+ <div class="col-md">
+ <h5 class="mb-3">
+ Metadata
+ <button class="btn btn-sm btn-default float-right"
+ onclick="toggleExpandPanel(event)" data-target="#metadataPanel">
+ <span class="toggle-icon-plus fas fa-plus fa-lg" style="pointer-events: none;"></span>
+ <span class="toggle-icon-minus fas fa-minus fa-lg" style="display: none; pointer-events: none;"></span>
+ </button>
+ </h5>
+
+ <div id="metadataPanel" style="overflow-y: auto; height: 30vh;">
+ <h6>Metadata from PEDASI</h6>
+
+ <table id="metadataInternal" class="table">
+ <script type="application/javascript">
+ document.addEventListener("DOMContentLoaded", populateMetadataInternal);
+ </script>
+ </table>
+
+ <h6>Metadata from source</h6>
+
+ <table id="metadata" class="table">
+ <script type="application/javascript">
+ document.addEventListener("DOMContentLoaded", populateMetadata);
+ </script>
+ </table>
+ </div>
+
+ </div>
+
+ <div class="col-md">
+ <h5 class="mb-3">
+ Datasets
+ <button class="btn btn-sm btn-default float-right"
+ onclick="toggleExpandPanel(event);" data-target="#datasetsPanel">
+ <span class="toggle-icon-plus fas fa-plus fa-lg" style="pointer-events: none;"></span>
+ <span class="toggle-icon-minus fas fa-minus fa-lg" style="display: none; pointer-events: none;"></span>
+ </button>
+ </h5>
+
+ <div id="datasetsPanel" style="overflow-y: auto; height: 30vh;">
+ <table id="datasets" class="table">
+ <script type="application/javascript">
+ document.addEventListener("DOMContentLoaded", populateDatasets)
+ </script>
+ </table>
+ </div>
+ </div>
+ </div>
+
+ <hr>
+
+</div>
+
+<footer class="footer bg-primary">
+ <div class="container">
+ <p class="m-0 text-center text-white">PEDASI: IoT Observatory Demonstrator</p>
+ </div>
+</footer>
+
+{% bootstrap_javascript jquery=True %}
+
+</body>
+</html>
diff --git a/datasources/templates/datasources/datasource/list.html b/datasources/templates/datasources/datasource/list.html
index 863c30b829464da6a431cb4121dc3c43dfa502fe..771bf6d7d6315eef819457cb24251cffd2c0dd95 100644
--- a/datasources/templates/datasources/datasource/list.html
+++ b/datasources/templates/datasources/datasource/list.html
@@ -15,26 +15,68 @@
<h2>Data Sources</h2>
- <a href="{% url 'admin:datasources_datasource_add' %}"
- class="btn btn-success" role="button">Create Data Source</a>
+ {% if perms.datasources.add_datasource %}
+ <div class="row">
+ <div class="col-md-6 mx-auto">
+ <a href="{% url 'datasources:datasource.add' %}"
+ class="btn btn-block btn-success" role="button">New Data Source</a>
+ </div>
+ </div>
+ {% endif %}
<div class="mt-3"></div>
- <table class="table">
+ <table class="table table-hover">
<thead class="thead">
<tr>
- <th>Name</th>
- <th></th>
+ <th class="w-75">Name</th>
+ <th class="w-auto">
+ <i class="fas fa-user" aria-hidden="true"></i>
+ </th>
+ <th class="w-auto">Access</th>
+ <th class="w-auto"></th>
</tr>
</thead>
<tbody>
{% for datasource in datasources %}
<tr>
- <td>{{ datasource.name }}</td>
<td>
+ <p>
+ <b>{{ datasource.name }}</b>
+ {% if datasource.licence %}
+ <a href="{% url 'datasources:licence.detail' pk=datasource.licence.pk %}"
+ class="badge badge-info"
+ data-toggle="tooltip" data-placement="bottom" title="{{ datasource.licence.name }}">
+ {{ datasource.licence.short_name }}
+ </a>
+ {% else %}
+ <span class="badge badge-warning"
+ data-toggle="tooltip" data-placement="bottom" title="No Licence">
+ No Licence
+ </span>
+ {% endif %}
+ </p>
+ <p class="pl-5">
+ {{ datasource.description|truncatechars:120 }}
+ </p>
+ </td>
+ <td class="align-middle">
+ {% if datasource.owner == request.user %}
+ <i class="fas fa-user" aria-hidden="true"
+ data-toggle="tooltip" data-placement="top" title="My data source"></i>
+ {% endif %}
+ </td>
+ <td class="align-middle">
+ {# Level 1 is VIEW #}
+ {% if datasource.public_permission_level < 1 %}
+ <i class="fas fa-lock fa-lg"
+ data-toggle="tooltip" data-placement="top" title="Data source has access controls"></i>
+ {% endif %}
+ </td>
+ <td class="align-middle">
<a href="{% url 'datasources:datasource.detail' pk=datasource.pk %}"
- class="btn btn-primary" role="button">Detail</a>
+ class="btn btn-block btn-secondary" role="button">Detail</a>
</td>
</tr>
{% empty %}
@@ -44,4 +86,12 @@
{% endfor %}
</tbody>
</table>
-{% endblock %}
\ No newline at end of file
+{% endblock %}
+
+{% block extra_body %}
+ <script type="application/javascript">
+ $(function () {
+ $("[data-toggle='tooltip']").tooltip()
+ })
+ </script>
+{% endblock %}
diff --git a/datasources/templates/datasources/datasource/manage_access.html b/datasources/templates/datasources/datasource/manage_access.html
index f7263b9988d6ba2b526197e177d75818f90f8ce9..11ce9bdce97f4cf53ac318d7aa07e6c5467234ee 100644
--- a/datasources/templates/datasources/datasource/manage_access.html
+++ b/datasources/templates/datasources/datasource/manage_access.html
@@ -1,6 +1,10 @@
{% extends "base.html" %}
{% load bootstrap4 %}
+{% block extra_head %}
+ <script src="https://cdn.jsdelivr.net/npm/js-cookie@2.2.0/src/js.cookie.min.js"></script>
+{% endblock %}
+
{% block content %}
<nav aria-label="breadcrumb">
<ol class="breadcrumb">
@@ -19,34 +23,60 @@
</ol>
</nav>
- <h2>View Data Source - {{ datasource.name }}</h2>
-
- {% if datasource.description %}
- <p>{{ datasource.description }}</p>
- {% endif %}
+ <h2>{{ datasource.name }}</h2>
- <hr/>
+ <hr>
<h2>Requests for Access</h2>
- <table class="table">
+ <table id="requested-user-table" class="table">
<thead class="thead">
<tr>
<th>Username</th>
+ <th>Requested</th>
+ <th>Current</th>
+ <th>Push?</th>
+ <th></th>
<th></th>
<th></th>
</tr>
</thead>
<tbody>
- {% for user in datasource.users_group_requested.users.all %}
- <tr>
- <td>user.username</td>
+ {% for permission in permissions_requested.all %}
+ <tr id="requested-user-{{ permission.user.pk }}">
+ <td>
+ <p>
+ {{ permission.user.username }}
+ </p>
+ {% if permission.reason %}
+ <div class="alert alert-secondary" role="note">
+ {{ permission.reason }}
+ </div>
+ {% endif %}
+ </td>
+ <td>{{ permission.get_requested_display }}</td>
+ <td>{{ permission.get_granted_display }}</td>
+ <td>
+ {% if permission.push_requested %}
+ <i class="fas fa-upload fa-lg"
+ data-toggle="tooltip" data-placement="top" title="Permission to push data"></i>
+ {% endif %}
+ </td>
+ <td>
+ <button onclick="userGrantAccess({{ permission.user.pk }}, {{ permission.requested }}, {{ permission.push_requested|yesno:'true,false' }})"
+ class="btn btn-success"
+ role="button">Approve</button>
+ </td>
<td>
- <a href="#" class="btn btn-success" role="button">Approve</a>
+ <a href="{% url 'datasources:datasource.access.grant' pk=datasource.pk %}?user={{ permission.user_id }}"
+ class="btn btn-info"
+ role="button">Edit</a>
</td>
<td>
- <a href="#" class="btn btn-danger" role="button">Reject</a>
+ <button onclick="userGrantAccess({{ permission.user.pk }}, {{ permission.granted }}, {{ permission.push_granted|yesno:'true,false' }})"
+ class="btn btn-danger"
+ role="button">Reject</button>
</td>
</tr>
{% empty %}
@@ -59,24 +89,40 @@
<h2>Approved Users</h2>
- <table class="table">
+ <table id="approved-user-table" class="table">
<thead class="thead">
<tr>
<th>Username</th>
+ <th>Requested</th>
+ <th>Current</th>
+ <th>Push?</th>
+ <th></th>
<th></th>
<th></th>
</tr>
</thead>
<tbody>
- {% for user in datasource.users_group.users.all %}
- <tr>
- <td>user.username</td>
+ {% for permission in permissions_granted %}
+ <tr id="approved-user-{{ permission.user.pk }}">
+ <td>{{ permission.user.username }}</td>
+ <td>{{ permission.get_requested_display }}</td>
+ <td>{{ permission.get_granted_display }}</td>
<td>
- <a href="#" class="btn btn-success" role="button">Approve</a>
+ {% if permission.push_granted %}
+ <i class="fas fa-upload fa-lg"
+ data-toggle="tooltip" data-placement="top" title="Permission to push data"></i>
+ {% endif %}
</td>
<td>
- <a href="#" class="btn btn-danger" role="button">Reject</a>
+ <a href="{% url 'datasources:datasource.access.grant' pk=datasource.pk %}?user={{ permission.user_id }}"
+ class="btn btn-info"
+ role="button">Edit</a>
+ </td>
+ <td>
+ <button onclick="userRemoveAccess({{ permission.user.pk }})"
+ class="btn btn-danger"
+ role="button">Remove</button>
</td>
</tr>
{% empty %}
@@ -85,4 +131,52 @@
</tbody>
</table>
+ <script type="application/javascript">
+ function userGrantAccess(userPk, level, push){
+ $.post({
+ url: '{% url 'datasources:datasource.access.grant' pk=datasource.pk %}',
+ data: {
+ 'user': userPk,
+ 'granted': level,
+ 'requested': level,
+ 'push_granted': push,
+ 'push_requested': push
+ },
+ headers: {
+ 'X-CSRFToken': Cookies.get('csrftoken')
+ },
+ success: function(result, status, xhr){
+ document.getElementById('requested-user-' + userPk.toString()).remove();
+
+ // TODO if table is empty add 'table is empty' row
+ }
+ })
+ }
+
+ function userRemoveAccess(userPk){
+ $.post({
+ url: '{% url 'datasources:datasource.access.grant' pk=datasource.pk %}',
+ data: {
+ 'user': userPk,
+ 'granted': 0,
+ 'requested': 0
+ },
+ headers: {
+ 'X-CSRFToken': Cookies.get('csrftoken')
+ },
+ success: function(result, status, xhr){
+ try {
+ document.getElementById('approved-user-' + userPk.toString()).remove();
+ } catch (err) {}
+
+ try {
+ document.getElementById('requested-user-' + userPk.toString()).remove();
+ } catch (err) {}
+
+ // TODO if table is empty add 'table is empty' row
+ }
+ })
+ }
+ </script>
+
{% endblock %}
\ No newline at end of file
diff --git a/datasources/templates/datasources/datasource/update.html b/datasources/templates/datasources/datasource/update.html
new file mode 100644
index 0000000000000000000000000000000000000000..75db48fba5b66ef8e33156e9e7737b6a202cf436
--- /dev/null
+++ b/datasources/templates/datasources/datasource/update.html
@@ -0,0 +1,71 @@
+{% extends "base.html" %}
+{% load bootstrap4 %}
+
+{% block content %}
+ <nav aria-label="breadcrumb">
+ <ol class="breadcrumb">
+ <li class="breadcrumb-item" aria-current="page">
+ <a href="{% url 'index' %}">Home</a>
+ </li>
+ <li class="breadcrumb-item" aria-current="page">
+ <a href="{% url 'datasources:datasource.list' %}">Data Sources</a>
+ </li>
+ <li class="breadcrumb-item" aria-current="page">
+ <a href="{% url 'datasources:datasource.detail' pk=datasource.pk %}">{{ datasource.name }}</a>
+ </li>
+ <li class="breadcrumb-item active" aria-current="page">
+ Edit
+ </li>
+ </ol>
+ </nav>
+
+ <h2 class="pb-3">Edit Data Source</h2>
+
+ <form class="form" method="post" action="">
+ {% csrf_token %}
+
+ {% bootstrap_field form.name %}
+ {% bootstrap_field form.description %}
+ {% bootstrap_field form.url %}
+ {% bootstrap_field form.api_key %}
+ {% bootstrap_field form.plugin_name %}
+ {% bootstrap_field form.licence %}
+
+ <hr>
+
+ <div class="row">
+ <div class="col-sm-3">
+ {% bootstrap_field form.is_encrypted %}
+ </div>
+
+ <div class="col-sm-9">
+ <div id="encryptedDocsToggle">
+ {% bootstrap_field form.encrypted_docs_url %}
+ </div>
+ </div>
+
+ <script type="application/javascript">
+ const checkboxEncrypted = document.getElementById("id_is_encrypted");
+ const divEncryptedToggle = document.getElementById("encryptedDocsToggle");
+
+ function setupEncryptedToggle() {
+ if (checkboxEncrypted.checked) {
+ divEncryptedToggle.style.display = "block";
+ } else {
+ divEncryptedToggle.style.display = "none";
+ }
+ }
+
+ checkboxEncrypted.addEventListener("change", setupEncryptedToggle);
+
+ setupEncryptedToggle();
+ </script>
+ </div>
+
+ {% bootstrap_field form.public_permission_level %}
+ {% bootstrap_field form.prov_exempt %}
+
+ <input type="submit" class="btn btn-success" value="Update">
+ </form>
+
+{% endblock %}
diff --git a/datasources/templates/datasources/licence/create.html b/datasources/templates/datasources/licence/create.html
new file mode 100644
index 0000000000000000000000000000000000000000..9840fe50d99edb8742d0bf7f38e6839d35954f2b
--- /dev/null
+++ b/datasources/templates/datasources/licence/create.html
@@ -0,0 +1,32 @@
+{% extends "base.html" %}
+{% load bootstrap4 %}
+
+{% block content %}
+ <nav aria-label="breadcrumb">
+ <ol class="breadcrumb">
+ <li class="breadcrumb-item" aria-current="page">
+ <a href="{% url 'index' %}">Home</a>
+ </li>
+ <li class="breadcrumb-item" aria-current="page">
+ <a href="{% url 'datasources:datasource.list' %}">Data Sources</a>
+ </li>
+ <li class="breadcrumb-item" aria-current="page">
+ <a href="{% url 'datasources:licence.list' %}">Licences</a>
+ </li>
+ <li class="breadcrumb-item active" aria-current="page">
+ New Licence
+ </li>
+ </ol>
+ </nav>
+
+ <h2 class="pb-3">New Licence</h2>
+
+ <form class="form" method="post" action="">
+ {% csrf_token %}
+
+ {% bootstrap_form form %}
+
+ <input type="submit" class="btn btn-success" value="Create">
+ </form>
+
+{% endblock %}
diff --git a/datasources/templates/datasources/licence/delete.html b/datasources/templates/datasources/licence/delete.html
new file mode 100644
index 0000000000000000000000000000000000000000..b41fbd481393422c1ac57157f6088e7b74db1a1c
--- /dev/null
+++ b/datasources/templates/datasources/licence/delete.html
@@ -0,0 +1,67 @@
+{% extends "base.html" %}
+{% load bootstrap4 %}
+
+{% block content %}
+ <nav aria-label="breadcrumb">
+ <ol class="breadcrumb">
+ <li class="breadcrumb-item" aria-current="page">
+ <a href="{% url 'index' %}">Home</a>
+ </li>
+ <li class="breadcrumb-item" aria-current="page">
+ <a href="{% url 'datasources:datasource.list' %}">Data Sources</a>
+ </li>
+ <li class="breadcrumb-item" aria-current="page">
+ <a href="{% url 'datasources:licence.list' %}">Licences</a>
+ </li>
+ <li class="breadcrumb-item" aria-current="page">
+ <a href="{% url 'datasources:licence.detail' pk=licence.pk %}">{{ licence.name }}</a>
+ </li>
+ <li class="breadcrumb-item active" aria-current="page">
+ Delete
+ </li>
+ </ol>
+ </nav>
+
+ <div class="row">
+ <div class="col-md-10 col-sm-8">
+ <h2>{{ licence.name }}</h2>
+ </div>
+ </div>
+
+ <table class="table">
+ <thead>
+ <th scope="col" class="w-25 border-0"></th>
+ <th scope="col" class="border-0"></th>
+ </thead>
+
+ <tbody>
+ <tr>
+ <td>Short Name</td>
+ <td>{{ licence.short_name }}</td>
+ </tr>
+ <tr>
+ <td>Version</td>
+ <td>{{ licence.version }}</td>
+ </tr>
+ <tr>
+ <td>URL</td>
+ <td>
+ <a href="{{ licence.url }}">{{ licence.url }}</a>
+ </td>
+ </tr>
+ </tbody>
+ </table>
+
+ <div class="alert alert-danger">
+ <p><b>Are you sure you want to delete this licence?</b></p>
+
+ <form class="form" method="post">
+ {% csrf_token %}
+
+ <input type="submit" role="button" class="btn btn-danger" value="Delete">
+
+ <a role="button" class="btn btn-info"
+ href="{% url 'datasources:licence.detail' pk=licence.pk %}">Cancel</a>
+ </form>
+ </div>
+{% endblock %}
\ No newline at end of file
diff --git a/datasources/templates/datasources/licence/detail.html b/datasources/templates/datasources/licence/detail.html
new file mode 100644
index 0000000000000000000000000000000000000000..ffcade1e038d8ee148a3e9b5cd0f0a0b4ce1e6d0
--- /dev/null
+++ b/datasources/templates/datasources/licence/detail.html
@@ -0,0 +1,62 @@
+{% extends "base.html" %}
+{% load bootstrap4 %}
+
+{% block content %}
+ <nav aria-label="breadcrumb">
+ <ol class="breadcrumb">
+ <li class="breadcrumb-item" aria-current="page">
+ <a href="{% url 'index' %}">Home</a>
+ </li>
+ <li class="breadcrumb-item" aria-current="page">
+ <a href="{% url 'datasources:datasource.list' %}">Data Sources</a>
+ </li>
+ <li class="breadcrumb-item" aria-current="page">
+ <a href="{% url 'datasources:licence.list' %}">Licences</a>
+ </li>
+ <li class="breadcrumb-item active" aria-current="page">
+ {{ licence.name }}
+ </li>
+ </ol>
+ </nav>
+
+ <div class="row">
+ <div class="col-md-10 col-sm-8">
+ <h2>{{ licence.name }}</h2>
+ </div>
+
+ <div class="col-md-2 col-sm-4">
+ {% if has_edit_permission %}
+ <a href="{% url 'datasources:licence.edit' pk=licence.id %}"
+ class="btn btn-block btn-success" role="button">Edit</a>
+
+ <a href="{% url 'datasources:licence.delete' pk=licence.id %}"
+ class="btn btn-block btn-danger" role="button">Delete</a>
+ {% endif %}
+ </div>
+ </div>
+
+ <table class="table">
+ <thead>
+ <th scope="col" class="w-25 border-0"></th>
+ <th scope="col" class="border-0"></th>
+ </thead>
+
+ <tbody>
+ <tr>
+ <td>Short Name</td>
+ <td>{{ licence.short_name }}</td>
+ </tr>
+ <tr>
+ <td>Version</td>
+ <td>{{ licence.version }}</td>
+ </tr>
+ <tr>
+ <td>URL</td>
+ <td>
+ <a href="{{ licence.url }}">{{ licence.url }}</a>
+ </td>
+ </tr>
+ </tbody>
+ </table>
+
+{% endblock %}
diff --git a/datasources/templates/datasources/licence/list.html b/datasources/templates/datasources/licence/list.html
new file mode 100644
index 0000000000000000000000000000000000000000..bbd1984107a0fd42dcea98904b8d055dd77a62e7
--- /dev/null
+++ b/datasources/templates/datasources/licence/list.html
@@ -0,0 +1,63 @@
+{% extends "base.html" %}
+{% load bootstrap4 %}
+
+{% block content %}
+ <nav aria-label="breadcrumb">
+ <ol class="breadcrumb">
+ <li class="breadcrumb-item" aria-current="page">
+ <a href="{% url 'index' %}">Home</a>
+ </li>
+ <li class="breadcrumb-item" aria-current="page">
+ <a href="{% url 'datasources:datasource.list' %}">Data Sources</a>
+ </li>
+ <li class="breadcrumb-item active" aria-current="page">
+ Licences
+ </li>
+ </ol>
+ </nav>
+
+ <h2>Licenses</h2>
+
+ {% if perms.datasources.add_datasource %}
+ <div class="row">
+ <div class="col-md-6 mx-auto">
+ <a href="{% url 'datasources:licence.add' %}"
+ class="btn btn-block btn-success" role="button">New License</a>
+ </div>
+ </div>
+ {% endif %}
+
+ <div class="mt-3"></div>
+
+ <table class="table table-hover">
+ <thead class="thead">
+ <tr>
+ <th class="w-75">Name</th>
+ <th class="w-auto"></th>
+ </tr>
+ </thead>
+
+ <tbody>
+ {% for licence in licences %}
+ <tr>
+ <td>
+ <p>
+ <b>{{ licence.name }}</b>
+ </p>
+ <p class="pl-5">
+ {{ datasource.description|truncatechars:120 }}
+ </p>
+ </td>
+ <td class="align-middle">
+ <a href="{% url 'datasources:licence.detail' pk=licence.pk %}"
+ class="btn btn-block btn-secondary" role="button">Detail</a>
+ </td>
+ </tr>
+ {% empty %}
+ <tr>
+ <td>There are no licences currently available.</td>
+ </tr>
+ {% endfor %}
+ </tbody>
+ </table>
+{% endblock %}
diff --git a/datasources/templates/datasources/licence/update.html b/datasources/templates/datasources/licence/update.html
new file mode 100644
index 0000000000000000000000000000000000000000..063b6115d751a0cb040ba93f8e58f54dce2cbe99
--- /dev/null
+++ b/datasources/templates/datasources/licence/update.html
@@ -0,0 +1,32 @@
+{% extends "base.html" %}
+{% load bootstrap4 %}
+
+{% block content %}
+ <nav aria-label="breadcrumb">
+ <ol class="breadcrumb">
+ <li class="breadcrumb-item" aria-current="page">
+ <a href="{% url 'index' %}">Home</a>
+ </li>
+ <li class="breadcrumb-item" aria-current="page">
+ <a href="{% url 'datasources:datasource.list' %}">Data Sources</a>
+ </li>
+ <li class="breadcrumb-item" aria-current="page">
+ <a href="{% url 'datasources:licence.list' %}">Licences</a>
+ </li>
+ <li class="breadcrumb-item active" aria-current="page">
+ Update Licence
+ </li>
+ </ol>
+ </nav>
+
+ <h2 class="pb-3">Update Licence</h2>
+
+ <form class="form" method="post" action="">
+ {% csrf_token %}
+
+ {% bootstrap_form form %}
+
+ <input type="submit" class="btn btn-success" value="Update">
+ </form>
+
+{% endblock %}
diff --git a/datasources/templates/datasources/user_permission_link/permission_grant.html b/datasources/templates/datasources/user_permission_link/permission_grant.html
new file mode 100644
index 0000000000000000000000000000000000000000..272984d6446df890dd7133c4b3d39194e4f8501f
--- /dev/null
+++ b/datasources/templates/datasources/user_permission_link/permission_grant.html
@@ -0,0 +1,72 @@
+{% extends "base.html" %}
+{% load bootstrap4 %}
+
+{% block content %}
+ <nav aria-label="breadcrumb">
+ <ol class="breadcrumb">
+ <li class="breadcrumb-item" aria-current="page">
+ <a href="{% url 'index' %}">Home</a>
+ </li>
+ <li class="breadcrumb-item" aria-current="page">
+ <a href="{% url 'datasources:datasource.list' %}">Data Sources</a>
+ </li>
+ <li class="breadcrumb-item" aria-current="page">
+ <a href="{% url 'datasources:datasource.detail' pk=datasource.pk %}">{{ datasource.name }}</a>
+ </li>
+ <li class="breadcrumb-item active" aria-current="page">
+ Manage Access
+ </li>
+ </ol>
+ </nav>
+
+ <h2>{{ datasource.name }}</h2>
+
+ <hr>
+
+ <table class="table">
+ <thead>
+ <th scope="col" class="col-md-2 border-0"></th>
+ <th scope="col" class="border-0"></th>
+ </thead>
+
+ <tbody>
+ <tr>
+ <td>Owner</td>
+ <td>
+ {{ datasource.owner }}
+ </td>
+ </tr>
+ <tr>
+ <td>URL</td>
+ <td>{{ datasource.url }}</td>
+ </tr>
+ </tbody>
+ </table>
+
+ <h2>Access Request</h2>
+
+ {{ permission.user.username }} requests {{ permission.get_requested_display }} permission on this data source
+
+ {% if permission.push_requested %}
+ and permission to push data
+ {% endif %}
+
+ {% if permission.reason %}
+ <div class="alert alert-secondary" role="note">
+ <p>
+ {{ permission.reason|linebreaks }}
+ </p>
+ </div>
+ {% endif %}
+
+ <form method="post">
+ {% csrf_token %}
+ {% bootstrap_form form %}
+
+ {% buttons %}
+ <input type="submit" class="btn btn-success" value="Grant">
+ <button onclick="window.history.back();" class="btn btn-default" type="button">Cancel</button>
+ {% endbuttons %}
+ </form>
+
+{% endblock %}
\ No newline at end of file
diff --git a/datasources/templates/datasources/user_permission_link/permission_request.html b/datasources/templates/datasources/user_permission_link/permission_request.html
new file mode 100644
index 0000000000000000000000000000000000000000..a4393c6ed1bdfd03e72fce3dae47dd5d84ecfe34
--- /dev/null
+++ b/datasources/templates/datasources/user_permission_link/permission_request.html
@@ -0,0 +1,58 @@
+{% extends "base.html" %}
+{% load bootstrap4 %}
+
+{% block content %}
+ <nav aria-label="breadcrumb">
+ <ol class="breadcrumb">
+ <li class="breadcrumb-item" aria-current="page">
+ <a href="{% url 'index' %}">Home</a>
+ </li>
+ <li class="breadcrumb-item" aria-current="page">
+ <a href="{% url 'datasources:datasource.list' %}">Data Sources</a>
+ </li>
+ <li class="breadcrumb-item" aria-current="page">
+ <a href="{% url 'datasources:datasource.detail' pk=datasource.pk %}">{{ datasource.name }}</a>
+ </li>
+ <li class="breadcrumb-item active" aria-current="page">
+ Request Access
+ </li>
+ </ol>
+ </nav>
+
+ <h2>{{ datasource.name }}</h2>
+
+ <hr>
+
+ <table class="table">
+ <thead>
+ <th scope="col" class="col-md-2 border-0"></th>
+ <th scope="col" class="border-0"></th>
+ </thead>
+
+ <tbody>
+ <tr>
+ <td>Owner</td>
+ <td>
+ {{ datasource.owner }}
+ </td>
+ </tr>
+ <tr>
+ <td>URL</td>
+ <td>{{ datasource.url }}</td>
+ </tr>
+ </tbody>
+ </table>
+
+ <h2>Access Request</h2>
+
+ <form method="post">
+ {% csrf_token %}
+ {% bootstrap_form form %}
+
+ {% buttons %}
+ <input type="submit" class="btn btn-success" value="Request">
+ <button onclick="window.history.back();" class="btn btn-default" type="button">Cancel</button>
+ {% endbuttons %}
+ </form>
+
+{% endblock %}
\ No newline at end of file
diff --git a/datasources/templates/search/indexes/datasources/datasource_text.txt b/datasources/templates/search/indexes/datasources/datasource_text.txt
new file mode 100644
index 0000000000000000000000000000000000000000..16cade4e8d823e5d40337e0992df98839750d095
--- /dev/null
+++ b/datasources/templates/search/indexes/datasources/datasource_text.txt
@@ -0,0 +1 @@
+{{ object.search_representation }}
diff --git a/datasources/tests/test_connectors.py b/datasources/tests/test_connectors.py
index b3e09fb8f042368587b2e84d225cd7aa14a2f562..8527559159265b2990a24ed6ad2120a6b88a9110 100644
--- a/datasources/tests/test_connectors.py
+++ b/datasources/tests/test_connectors.py
@@ -10,12 +10,12 @@ class ConnectorPluginTest(TestCase):
"""
BaseDataConnector.load_plugins('datasources/connectors')
- def test_get_plugin_iotuk(self):
+ def test_get_plugin_simple(self):
"""
- Test that we have the IoTUK plugin and can activate it.
+ Test that we have the plugin for trivial APIs and can activate it.
"""
BaseDataConnector.load_plugins('datasources/connectors')
- plugin = BaseDataConnector.get_plugin('IoTUK')
+ plugin = BaseDataConnector.get_plugin('DataSetConnector')
self.assertIsNotNone(plugin)
@@ -41,19 +41,29 @@ class ConnectorPluginTest(TestCase):
class ConnectorIoTUKTest(TestCase):
url = 'https://api.iotuk.org.uk/iotOrganisation'
+ def _get_connection(self) -> BaseDataConnector:
+ return self.plugin(self.url)
+
def setUp(self):
BaseDataConnector.load_plugins('datasources/connectors')
- self.plugin = BaseDataConnector.get_plugin('IoTUK')
+ self.plugin = BaseDataConnector.get_plugin('DataSetConnector')
def test_get_plugin(self):
self.assertIsNotNone(self.plugin)
def test_plugin_init(self):
- connection = self.plugin(self.url)
+ connection = self._get_connection()
+
self.assertEqual(connection.location, self.url)
+ def test_plugin_type(self):
+ connection = self._get_connection()
+
+ self.assertFalse(connection.is_catalogue)
+
def test_plugin_get_data_fails(self):
- connection = self.plugin(self.url)
+ connection = self._get_connection()
+
result = connection.get_data()
self.assertIn('status', result)
@@ -63,8 +73,9 @@ class ConnectorIoTUKTest(TestCase):
self.assertEqual(result['results'], -1)
def test_plugin_get_data_query(self):
- connection = self.plugin(self.url)
- result = connection.get_data(query_params={'year': 2018})
+ connection = self._get_connection()
+
+ result = connection.get_data(params={'year': 2018})
self.assertIn('status', result)
self.assertEqual(result['status'], 200)
@@ -76,135 +87,39 @@ class ConnectorIoTUKTest(TestCase):
self.assertGreater(len(result['data']), 0)
-class ConnectorHyperCatTest(TestCase):
- url = 'https://portal.bt-hypercat.com/cat'
+class ConnectorRestApiTest(TestCase):
+ url = 'https://api.iotuk.org.uk/'
- # Met Office dataset for weather at Heathrow
- dataset = 'http://api.bt-hypercat.com/sensors/feeds/c7f361c6-7cb7-4ef5-aed9-397a0c0c4088'
+ def _get_connection(self) -> BaseDataConnector:
+ return self.plugin(self.url)
def setUp(self):
BaseDataConnector.load_plugins('datasources/connectors')
- self.plugin = BaseDataConnector.get_plugin('HyperCat')
+ self.plugin = BaseDataConnector.get_plugin('RestApiConnector')
def test_get_plugin(self):
self.assertIsNotNone(self.plugin)
def test_plugin_init(self):
- connection = self.plugin(self.url)
- self.assertEqual(connection.location, self.url)
-
- def test_plugin_get_dataset_metadata(self):
- connection = self.plugin(self.url)
- result = connection.get_metadata(dataset=self.dataset)
-
- for property in [
- 'urn:X-bt:rels:feedTitle',
- 'urn:X-hypercat:rels:hasDescription:en',
- 'urn:X-bt:rels:feedTag',
- 'urn:X-bt:rels:hasSensorStream',
- 'urn:X-hypercat:rels:isContentType',
- ]:
- self.assertIn(property, result)
-
- self.assertIn('Met Office',
- result['urn:X-bt:rels:feedTitle'][0])
-
- self.assertIn('Met Office',
- result['urn:X-hypercat:rels:hasDescription:en'][0])
-
- self.assertEqual(len(result['urn:X-bt:rels:feedTag']), 1)
- self.assertEqual(result['urn:X-bt:rels:feedTag'][0], 'weather')
-
- self.assertGreaterEqual(len(result['urn:X-bt:rels:hasSensorStream']), 1)
-
- self.assertIn('application/json',
- result['urn:X-hypercat:rels:isContentType'])
-
- def test_plugin_get_dataset_data(self):
- """
- Test that we can get data from a single dataset within the catalogue.
- """
- from decouple import config
-
- api_key = config('HYPERCAT_BT_API_KEY')
-
- dataset = self.dataset + '/datastreams/0'
-
- connection = self.plugin(self.url,
- api_key=api_key)
- result = connection.get_data(dataset=dataset)
-
- self.assertIsInstance(result, str)
- self.assertGreaterEqual(len(result), 1)
-
-
-class ConnectorHyperCatCiscoTest(TestCase):
- url = 'https://api.cityverve.org.uk/v1/cat'
- entity_url = 'https://api.cityverve.org.uk/v1/entity'
+ connection = self._get_connection()
- dataset = 'weather-observations-wind'
-
- def setUp(self):
- from decouple import config
-
- BaseDataConnector.load_plugins('datasources/connectors')
- self.plugin = BaseDataConnector.get_plugin('HyperCatCisco')
-
- self.api_key = config('HYPERCAT_CISCO_API_KEY')
-
- def test_get_plugin(self):
- self.assertIsNotNone(self.plugin)
-
- def test_plugin_init(self):
- connection = self.plugin(self.url)
self.assertEqual(connection.location, self.url)
- def test_plugin_get_entities(self):
- connection = self.plugin(self.url,
- api_key=self.api_key,
- entity_url=self.entity_url)
- result = connection.get_entities()
-
- self.assertGreaterEqual(len(result), 1)
-
- for entity in result:
- self.assertIn('id', entity)
- self.assertIn('uri', entity)
+ def test_plugin_type(self):
+ connection = self._get_connection()
- def test_plugin_get_catalogue_metadata(self):
- connection = self.plugin(self.url)
- result = connection.get_metadata()
+ self.assertTrue(connection.is_catalogue)
- self.assertIn('application/vnd.hypercat.catalogue+json',
- result['urn:X-hypercat:rels:isContentType'])
+ def test_plugin_dataset_get_data_query(self):
+ connection = self._get_connection()
- self.assertIn('CityVerve',
- result['urn:X-hypercat:rels:hasDescription:en'][0])
+ result = connection['iotOrganisation'].get_data(params={'year': 2018})
- self.assertEqual('https://developer.cityverve.org.uk',
- result['urn:X-hypercat:rels:hasHomepage'])
-
- def test_plugin_get_dataset_metadata(self):
- connection = self.plugin(self.url)
- result = connection.get_metadata(dataset=self.dataset)
-
- for property in [
- 'urn:X-bt:rels:feedTitle',
- 'urn:X-hypercat:rels:hasDescription:en',
- 'urn:X-bt:rels:feedTag',
- 'urn:X-bt:rels:hasSensorStream',
- 'urn:X-hypercat:rels:isContentType',
- ]:
- self.assertIn(property, result)
-
- self.assertIn('Met Office',
- result['urn:X-bt:rels:feedTitle'][0])
-
- self.assertIn('Met Office',
- result['urn:X-hypercat:rels:hasDescription:en'][0])
-
- self.assertEqual(len(result['urn:X-bt:rels:feedTag']), 1)
- self.assertEqual(result['urn:X-bt:rels:feedTag'][0], 'weather')
+ self.assertIn('status', result)
+ self.assertEqual(result['status'], 200)
- self.assertGreaterEqual(len(result['urn:X-bt:rels:hasSensorStream']), 1)
+ self.assertIn('results', result)
+ self.assertGreater(result['results'], 0)
+ self.assertIn('data', result)
+ self.assertGreater(len(result['data']), 0)
diff --git a/datasources/tests/test_connectors_hypercat.py b/datasources/tests/test_connectors_hypercat.py
new file mode 100644
index 0000000000000000000000000000000000000000..a34bff6a05a8caaa7aeb4464f9f0cea4132332c7
--- /dev/null
+++ b/datasources/tests/test_connectors_hypercat.py
@@ -0,0 +1,367 @@
+import itertools
+import typing
+
+from django.test import TestCase
+from requests.auth import HTTPBasicAuth
+
+from datasources.connectors.base import BaseDataConnector, HttpHeaderAuth
+
+
+def _get_item_by_key_value(collection: typing.Iterable[typing.Mapping],
+ key: str, value) -> typing.Mapping:
+ matches = [item for item in collection if item[key] == value]
+
+ if not matches:
+ raise KeyError
+ elif len(matches) > 1:
+ raise ValueError('Multiple items were found')
+
+ return matches[0]
+
+
+def _count_items_by_key_value(collection: typing.Iterable[typing.Mapping],
+ key: str, value) -> int:
+ matches = [item for item in collection if item[key] == value]
+
+ return len(matches)
+
+
+class ConnectorHyperCatTest(TestCase):
+ url = 'https://portal.bt-hypercat.com/cat'
+
+ # Met Office dataset for weather at Heathrow
+ dataset = 'http://api.bt-hypercat.com/sensors/feeds/c7f361c6-7cb7-4ef5-aed9-397a0c0c4088'
+
+ def _get_connection(self) -> BaseDataConnector:
+ return self.plugin(self.url,
+ api_key=self.api_key,
+ auth=HTTPBasicAuth)
+
+ def setUp(self):
+ from decouple import config
+
+ BaseDataConnector.load_plugins('datasources/connectors')
+ self.plugin = BaseDataConnector.get_plugin('HyperCat')
+
+ self.api_key = config('HYPERCAT_BT_API_KEY')
+
+ def test_get_plugin(self):
+ self.assertIsNotNone(self.plugin)
+
+ def test_plugin_init(self):
+ connection = self._get_connection()
+
+ self.assertEqual(connection.location, self.url)
+
+ def test_plugin_type(self):
+ connection = self._get_connection()
+
+ self.assertTrue(connection.is_catalogue)
+
+ def test_plugin_get_metadata(self):
+ connection = self._get_connection()
+
+ result = connection.get_metadata()
+
+ relations = [relation['rel'] for relation in result]
+ for property in [
+ 'urn:X-hypercat:rels:hasDescription:en',
+ 'urn:X-hypercat:rels:isContentType',
+ ]:
+ self.assertIn(property, relations)
+
+ self.assertEqual('BT Hypercat DataHub Catalog',
+ _get_item_by_key_value(result, 'rel', 'urn:X-hypercat:rels:hasDescription:en')['val'])
+
+ self.assertEqual('application/vnd.hypercat.catalogue+json',
+ _get_item_by_key_value(result, 'rel', 'urn:X-hypercat:rels:isContentType')['val'])
+
+ def test_plugin_get_datasets(self):
+ connection = self._get_connection()
+
+ datasets = connection.get_datasets()
+
+ self.assertEqual(list,
+ type(datasets))
+
+ self.assertLessEqual(1,
+ len(datasets))
+
+ self.assertIn(self.dataset,
+ datasets)
+
+ def test_plugin_iter_datasets(self):
+ connection = self._get_connection()
+
+ for dataset in connection:
+ self.assertEqual(str,
+ type(dataset))
+
+ def test_plugin_len_datasets(self):
+ connection = self._get_connection()
+
+ self.assertLessEqual(1,
+ len(connection))
+
+ def test_plugin_iter_items(self):
+ """
+ Test naive iteration over key-value pairs of datasets within this catalogue.
+
+ This process is SLOW so we only do a couple of iterations.
+ """
+ connection = self._get_connection()
+
+ for k, v in itertools.islice(connection.items(), 5):
+ self.assertEqual(str,
+ type(k))
+
+ self.assertIsInstance(v,
+ BaseDataConnector)
+
+ self.assertEqual(k,
+ v.location)
+
+ def test_plugin_iter_items_context_manager(self):
+ """
+ Test context-managed iteration over key-value pairs of datasets within this catalogue.
+
+ This process is relatively slow so we only do a couple of iterations.
+ """
+ with self.plugin(self.url, api_key=self.api_key, auth=HTTPBasicAuth) as connection:
+ for k, v in itertools.islice(connection.items(), 5):
+ self.assertEqual(str,
+ type(k))
+
+ self.assertIsInstance(v,
+ BaseDataConnector)
+
+ self.assertEqual(k,
+ v.location)
+
+ def test_plugin_get_dataset_metadata(self):
+ connection = self._get_connection()
+
+ result = connection[self.dataset].get_metadata()
+
+ relations = [relation['rel'] for relation in result]
+ for property in [
+ 'urn:X-bt:rels:feedTitle',
+ 'urn:X-hypercat:rels:hasDescription:en',
+ 'urn:X-bt:rels:feedTag',
+ 'urn:X-bt:rels:hasSensorStream',
+ 'urn:X-hypercat:rels:isContentType',
+ ]:
+ self.assertIn(property, relations)
+
+ self.assertIn('Met Office',
+ _get_item_by_key_value(result, 'rel', 'urn:X-bt:rels:feedTitle')['val'])
+
+ self.assertIn('Met Office',
+ _get_item_by_key_value(result, 'rel', 'urn:X-hypercat:rels:hasDescription:en')['val'])
+
+ self.assertEqual(1,
+ _count_items_by_key_value(result, 'rel', 'urn:X-bt:rels:feedTag'))
+
+ self.assertGreaterEqual(_count_items_by_key_value(result, 'rel', 'urn:X-bt:rels:hasSensorStream'),
+ 1)
+
+ def test_plugin_get_dataset_data(self):
+ """
+ Test that we can get data from a single dataset within the catalogue.
+ """
+ connection = self._get_connection()
+
+ dataset = connection[self.dataset]
+ result = dataset.get_data()
+
+ self.assertIsInstance(result, str)
+ self.assertGreaterEqual(len(result), 1)
+ self.assertIn('c7f361c6-7cb7-4ef5-aed9-397a0c0c4088',
+ result)
+
+
+class ConnectorHyperCatCiscoTest(TestCase):
+ url = 'https://api.cityverve.org.uk/v1/cat'
+ subcatalogue = 'https://api.cityverve.org.uk/v1/cat/polling-station'
+ dataset = 'https://api.cityverve.org.uk/v1/entity/polling-station/5'
+
+ def _get_connection(self) -> BaseDataConnector:
+ return self.plugin(self.url,
+ api_key=self.api_key,
+ auth=HttpHeaderAuth)
+
+ def setUp(self):
+ from decouple import config
+
+ BaseDataConnector.load_plugins('datasources/connectors')
+ self.plugin = BaseDataConnector.get_plugin('HyperCat')
+
+ self.api_key = config('HYPERCAT_CISCO_API_KEY')
+ self.auth = None
+
+ def test_get_plugin(self):
+ self.assertIsNotNone(self.plugin)
+
+ def test_plugin_init(self):
+ connection = self._get_connection()
+
+ self.assertEqual(connection.location, self.url)
+
+ def test_plugin_type(self):
+ connection = self._get_connection()
+
+ self.assertTrue(connection.is_catalogue)
+
+ def test_plugin_get_catalogue_metadata(self):
+ connection = self._get_connection()
+
+ result = connection.get_metadata()
+
+ relations = [relation['rel'] for relation in result]
+ for property in [
+ 'urn:X-hypercat:rels:hasDescription:en',
+ 'urn:X-hypercat:rels:isContentType',
+ 'urn:X-hypercat:rels:hasHomepage',
+ ]:
+ self.assertIn(property, relations)
+
+ self.assertEqual('CityVerve Public API - master catalogue',
+ _get_item_by_key_value(result, 'rel', 'urn:X-hypercat:rels:hasDescription:en')['val'])
+
+ self.assertEqual('application/vnd.hypercat.catalogue+json',
+ _get_item_by_key_value(result, 'rel', 'urn:X-hypercat:rels:isContentType')['val'])
+
+ self.assertEqual('https://developer.cityverve.org.uk',
+ _get_item_by_key_value(result, 'rel', 'urn:X-hypercat:rels:hasHomepage')['val'])
+
+ def test_plugin_get_datasets(self):
+ connection = self._get_connection()
+
+ datasets = connection.get_datasets()
+
+ self.assertEqual(list,
+ type(datasets))
+
+ self.assertLessEqual(1,
+ len(datasets))
+
+ # Only check a couple of expected results are present - there's too many to list here
+ expected = {
+ 'https://api.cityverve.org.uk/v1/cat/accident',
+ 'https://api.cityverve.org.uk/v1/cat/advertising-board',
+ 'https://api.cityverve.org.uk/v1/cat/advertising-post',
+ # And because later tests rely on it...
+ 'https://api.cityverve.org.uk/v1/cat/polling-station',
+ }
+ for exp in expected:
+ self.assertIn(exp, datasets)
+
+ def test_plugin_get_subcatalogue_metadata(self):
+ connection = self._get_connection()
+
+ result = connection[self.subcatalogue].get_metadata()
+
+ relations = [relation['rel'] for relation in result]
+ for property in [
+ 'urn:X-hypercat:rels:hasDescription:en',
+ 'urn:X-hypercat:rels:isContentType',
+ 'urn:X-hypercat:rels:hasHomepage',
+ ]:
+ self.assertIn(property, relations)
+
+ self.assertEqual('CityVerve Public API - polling-station catalogue',
+ _get_item_by_key_value(result, 'rel', 'urn:X-hypercat:rels:hasDescription:en')['val'])
+
+ self.assertEqual('application/vnd.hypercat.catalogue+json',
+ _get_item_by_key_value(result, 'rel', 'urn:X-hypercat:rels:isContentType')['val'])
+
+ self.assertEqual('https://developer.cityverve.org.uk',
+ _get_item_by_key_value(result, 'rel', 'urn:X-hypercat:rels:hasHomepage')['val'])
+
+ def test_plugin_get_subcatalogue_datasets(self):
+ connection = self._get_connection()
+
+ subcatalogue = connection[self.subcatalogue]
+ datasets = subcatalogue.get_datasets()
+
+ self.assertEqual(list,
+ type(datasets))
+
+ self.assertLessEqual(1,
+ len(datasets))
+
+ # Only check a couple of expected results are present - there's too many to list here
+ expected = {
+ 'https://api.cityverve.org.uk/v1/entity/polling-station/5',
+ 'https://api.cityverve.org.uk/v1/entity/polling-station/3',
+ 'https://api.cityverve.org.uk/v1/entity/polling-station/4',
+ }
+ for exp in expected:
+ self.assertIn(exp, datasets)
+
+ def test_plugin_get_subcatalogue_dataset_metadata(self):
+ connection = self._get_connection()
+
+ subcatalogue = connection[self.subcatalogue]
+ dataset = subcatalogue[self.dataset]
+ result = dataset.get_metadata()
+
+ relations = [relation['rel'] for relation in result]
+ for property in [
+ 'urn:X-cityverve:rels:id',
+ 'urn:X-cityverve:rels:name',
+ 'urn:X-cityverve:rels:type',
+ 'http://www.w3.org/2003/01/geo/wgs84_pos#long',
+ 'http://www.w3.org/2003/01/geo/wgs84_pos#lat',
+ 'urn:X-cityverve:rels:entity.district',
+ 'urn:X-cityverve:rels:entity.ward',
+ ]:
+ self.assertIn(property, relations)
+
+ self.assertEqual('5',
+ _get_item_by_key_value(result, 'rel', 'urn:X-cityverve:rels:id')['val'])
+
+ self.assertEqual('Cadishead : 5',
+ _get_item_by_key_value(result, 'rel', 'urn:X-cityverve:rels:name')['val'])
+
+ self.assertEqual('polling-station',
+ _get_item_by_key_value(result, 'rel', 'urn:X-cityverve:rels:type')['val'])
+
+ def test_plugin_get_subcatalogue_dataset_data(self):
+ connection = self._get_connection()
+
+ subcatalogue = connection[self.subcatalogue]
+ dataset = subcatalogue[self.dataset]
+ data = dataset.get_data()
+
+ self.assertEqual(list,
+ type(data))
+
+ self.assertEqual(1,
+ len(data))
+
+ data_entry = data[0]
+
+ self.assertEqual(dict,
+ type(data_entry))
+
+ for property in [
+ 'id',
+ 'uri',
+ 'type',
+ 'name',
+ 'loc',
+ 'entity',
+ 'instance',
+ 'legal',
+ ]:
+ self.assertIn(property, data_entry)
+
+ self.assertEqual('5',
+ data_entry['id'])
+
+ self.assertEqual(self.dataset,
+ data_entry['uri'])
+
+ self.assertEqual('polling-station',
+ data_entry['type'])
diff --git a/datasources/urls.py b/datasources/urls.py
index d96f8c95e86741d07d9b006480f1611449da97bc..f8f94d360cdfd75d0baed7a309fe37363be74c2a 100644
--- a/datasources/urls.py
+++ b/datasources/urls.py
@@ -6,22 +6,72 @@ app_name = 'datasources'
urlpatterns = [
path('',
- views.DataSourceListView.as_view(),
+ views.datasource.DataSourceListView.as_view(),
name='datasource.list'),
+ path('add',
+ views.datasource.DataSourceCreateView.as_view(),
+ name='datasource.add'),
+
path('<int:pk>/',
- views.DataSourceDetailView.as_view(),
+ views.datasource.DataSourceDetailView.as_view(),
name='datasource.detail'),
- path('<int:pk>/manage-access',
- views.DataSourceManageAccessView.as_view(),
- name='datasource.manage-access'),
+ path('<int:pk>/edit',
+ views.datasource.DataSourceUpdateView.as_view(),
+ name='datasource.edit'),
- path('<int:pk>/query',
- views.DataSourceQueryView.as_view(),
- name='datasource.query'),
+ path('<int:pk>/delete',
+ views.datasource.DataSourceDeleteView.as_view(),
+ name='datasource.delete'),
path('<int:pk>/metadata',
- views.DataSourceMetadataView.as_view(),
+ views.datasource.DataSourceMetadataAjaxView.as_view(),
name='datasource.metadata'),
+
+ path('<int:pk>/explorer',
+ views.datasource.DataSourceExplorerView.as_view(),
+ name='datasource.explorer'),
+
+ path('<int:pk>/search',
+ views.datasource.DataSourceDataSetSearchView.as_view(),
+ name='datasource.dataset.search'),
+
+ ##########
+ # Licences
+
+ path('licences',
+ views.licence.LicenceListView.as_view(),
+ name='licence.list'),
+
+ path('licences/add',
+ views.licence.LicenceCreateView.as_view(),
+ name='licence.add'),
+
+ path('licences/<int:pk>/',
+ views.licence.LicenceDetailView.as_view(),
+ name='licence.detail'),
+
+ path('licences/<int:pk>/edit',
+ views.licence.LicenceUpdateView.as_view(),
+ name='licence.edit'),
+
+ path('licences/<int:pk>/delete',
+ views.licence.LicenceDeleteView.as_view(),
+ name='licence.delete'),
+
+ #######################
+ # Permission management
+
+ path('<int:pk>/access',
+ views.user_permission_link.DataSourceAccessManageView.as_view(),
+ name='datasource.access.manage'),
+
+ path('<int:pk>/access/request',
+ views.user_permission_link.DataSourceAccessRequestView.as_view(),
+ name='datasource.access.request'),
+
+ path('<int:pk>/access/grant',
+ views.user_permission_link.DataSourceAccessGrantView.as_view(),
+ name='datasource.access.grant'),
]
diff --git a/datasources/views.py b/datasources/views.py
deleted file mode 100644
index 6e49cfc2518a0ab6a4eeecac05b693550bc4b31d..0000000000000000000000000000000000000000
--- a/datasources/views.py
+++ /dev/null
@@ -1,64 +0,0 @@
-from django.views.generic.detail import DetailView
-from django.views.generic.list import ListView
-
-from profiles.permissions import OwnerPermissionRequiredMixin
-from datasources import models
-
-
-class DataSourceListView(ListView):
- model = models.DataSource
- template_name = 'datasources/datasource/list.html'
- context_object_name = 'datasources'
-
-
-class DataSourceDetailView(DetailView):
- model = models.DataSource
- template_name = 'datasources/datasource/detail.html'
- context_object_name = 'datasource'
-
- def get_template_names(self):
- if not self.object.has_view_permission(self.request.user):
- return ['datasources/datasource/detail-no-access.html']
- return super().get_template_names()
-
-
-class DataSourceManageAccessView(OwnerPermissionRequiredMixin, DetailView):
- model = models.DataSource
- template_name = 'datasources/datasource/manage_access.html'
- context_object_name = 'datasource'
-
- permission_required = 'datasources.change_datasource'
-
-
-class DataSourceQueryView(DetailView):
- model = models.DataSource
- template_name = 'datasources/datasource/query.html'
- context_object_name = 'datasource'
-
- def get_context_data(self, **kwargs):
- context = super().get_context_data(**kwargs)
-
- context['results'] = self.object.data_connector.get_data(
- query_params={'year': 2018}
- )
-
- return context
-
-
-class DataSourceMetadataView(DetailView):
- model = models.DataSource
- template_name = 'datasources/datasource/metadata.html'
- context_object_name = 'datasource'
-
- def get_context_data(self, **kwargs):
- context = super().get_context_data(**kwargs)
-
- # Using data connector context manager saves API queries
- with self.object.data_connector as dc:
- context['metadata'] = dc.get_metadata()
- context['datasets'] = {
- dataset: dc.get_metadata(dataset)
- for dataset in dc.get_datasets()
- }
-
- return context
diff --git a/datasources/views/__init__.py b/datasources/views/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..a9638065b7f3d3f840e2ba8905802847088e2825
--- /dev/null
+++ b/datasources/views/__init__.py
@@ -0,0 +1 @@
+from . import datasource, licence, user_permission_link
\ No newline at end of file
diff --git a/datasources/views/datasource.py b/datasources/views/datasource.py
new file mode 100644
index 0000000000000000000000000000000000000000..736d4ea966dcb290b236bf71845668872d1cc1bf
--- /dev/null
+++ b/datasources/views/datasource.py
@@ -0,0 +1,201 @@
+from django.contrib import messages
+from django.contrib.auth.mixins import PermissionRequiredMixin
+from django.http import HttpResponse
+from django.urls import reverse_lazy
+from django.views.generic.detail import DetailView
+from django.views.generic.edit import CreateView, DeleteView, UpdateView
+from django.views.generic.list import ListView
+
+from rest_framework.response import Response
+from rest_framework import serializers
+from rest_framework.views import APIView
+import requests.exceptions
+
+from core.permissions import OwnerPermissionMixin
+from datasources import forms, models
+from datasources.permissions import HasPermissionLevelMixin
+
+
+class DataSourceListView(ListView):
+ model = models.DataSource
+ template_name = 'datasources/datasource/list.html'
+ context_object_name = 'datasources'
+
+
+class DataSourceDetailView(DetailView):
+ model = models.DataSource
+ template_name = 'datasources/datasource/detail.html'
+ context_object_name = 'datasource'
+
+ def get_template_names(self):
+ if not self.object.has_view_permission(self.request.user):
+ return ['datasources/datasource/detail-no-access.html']
+ return super().get_template_names()
+
+ def get_context_data(self, **kwargs):
+ context = super().get_context_data(**kwargs)
+
+ context['has_edit_permission'] = self.request.user.is_superuser or self.request.user == self.object.owner
+ if context['has_edit_permission']:
+ context['metadata_field_form'] = forms.MetadataFieldForm()
+
+ try:
+ context['is_catalogue'] = self.object.is_catalogue
+ except (KeyError, ValueError):
+ messages.error(self.request, 'This data source is not configured correctly. Please notify the owner.')
+
+ return context
+
+
+class DataSourceCreateView(PermissionRequiredMixin, CreateView):
+ model = models.DataSource
+ template_name = 'datasources/datasource/create.html'
+ context_object_name = 'datasource'
+
+ form_class = forms.DataSourceForm
+ permission_required = 'datasources.add_datasource'
+
+ def form_valid(self, form):
+ try:
+ owner = form.instance.owner
+
+ except models.DataSource.owner.RelatedObjectDoesNotExist:
+ form.instance.owner = self.request.user
+
+ return super().form_valid(form)
+
+
+class DataSourceUpdateView(OwnerPermissionMixin, UpdateView):
+ model = models.DataSource
+ template_name = 'datasources/datasource/update.html'
+ context_object_name = 'datasource'
+
+ form_class = forms.DataSourceForm
+
+
+class DataSourceDeleteView(OwnerPermissionMixin, DeleteView):
+ model = models.DataSource
+ template_name = 'datasources/datasource/delete.html'
+ context_object_name = 'datasource'
+
+ success_url = reverse_lazy('datasources:datasource.list')
+
+
+class DataSourceDataSetSearchView(HasPermissionLevelMixin, DetailView):
+ model = models.DataSource
+ template_name = 'datasources/datasource/dataset_search.html'
+ context_object_name = 'datasource'
+
+ permission_level = models.UserPermissionLevels.META
+
+ def get(self, request, *args, **kwargs):
+ try:
+ return super().get(request, *args, **kwargs)
+ except requests.exceptions.HTTPError as e:
+ return HttpResponse(
+ 'API call failed',
+ # Pass status code through unless it was 200 OK
+ status=424 if e.response.status_code == 200 else e.response.status_code
+ )
+
+ def get_context_data(self, **kwargs):
+ context = super().get_context_data(**kwargs)
+
+ connector = self.object.data_connector
+ try:
+ datasets = list(connector.items(
+ params={
+ 'prefix-val': self.request.GET.get('q')
+ }
+ ))
+ context['datasets'] = datasets
+
+ # Check the metadata format of the first dataset
+ # TODO will all metadata formats be the same
+ if isinstance(datasets[0][1].get_metadata(), list):
+ context['metadata_type'] = 'list'
+ else:
+ context['metadata_type'] = 'dict'
+
+ except AttributeError:
+ # DataSource is not a catalogue
+ pass
+
+ return context
+
+
+class DataSourceMetadataAjaxView(OwnerPermissionMixin, APIView):
+ model = models.DataSource
+
+ # Don't redirect to login page if unauthorised
+ raise_exception = True
+
+ class MetadataSerializer(serializers.ModelSerializer):
+ class Meta:
+ model = models.MetadataItem
+ fields = '__all__'
+
+ def get_object(self, pk):
+ return self.model.objects.get(pk=pk)
+
+ def post(self, request, pk, format=None):
+ """
+ Create a new MetadataItem associated with this DataSource.
+ """
+ datasource = self.get_object(pk)
+ if 'datasource' not in request.data:
+ request.data['datasource'] = datasource.pk
+
+ serializer = self.MetadataSerializer(data=request.data)
+
+ if serializer.is_valid():
+ obj = serializer.save()
+
+ return Response({
+ 'status': 'success',
+ 'data': {
+ 'datasource': datasource.pk,
+ 'field': obj.field.name,
+ 'field_short': obj.field.short_name,
+ 'value': obj.value,
+ }
+ })
+
+ return Response({'status': 'failure'}, status=400)
+
+ def delete(self, request, pk, format=None):
+ """
+ Delete a MetadataItem associated with this DataSource.
+ """
+ datasource = self.get_object(pk)
+ if 'datasource' not in request.data:
+ request.data['datasource'] = datasource.pk
+
+ metadata_item = models.MetadataItem.objects.get(
+ datasource=datasource,
+ field__short_name=self.request.data['field'],
+ value=self.request.data['value']
+ )
+
+ metadata_item.delete()
+
+ return Response({
+ 'status': 'success'
+ })
+
+
+class DataSourceExplorerView(HasPermissionLevelMixin, DetailView):
+ model = models.DataSource
+ template_name = 'datasources/datasource/explorer.html'
+ context_object_name = 'datasource'
+
+ permission_level = models.UserPermissionLevels.META
+
+ def get_context_data(self, **kwargs):
+ context = super().get_context_data()
+
+ context['data_query_params'] = self.object.metadata_items.filter(
+ field__short_name='data_query_param'
+ ).values_list('value', flat=True)
+
+ return context
diff --git a/datasources/views/licence.py b/datasources/views/licence.py
new file mode 100644
index 0000000000000000000000000000000000000000..4b90a3694fad9e37d22ee0241c19ee8b0e5448f9
--- /dev/null
+++ b/datasources/views/licence.py
@@ -0,0 +1,68 @@
+from django.contrib.auth.mixins import PermissionRequiredMixin
+from django.urls import reverse_lazy
+from django.views.generic import CreateView, DeleteView, DetailView, ListView, UpdateView
+
+from .. import forms, models
+from core.permissions import OwnerPermissionMixin
+
+
+class LicenceListView(ListView):
+ model = models.Licence
+ template_name = 'datasources/licence/list.html'
+ context_object_name = 'licences'
+
+
+class LicenceCreateView(PermissionRequiredMixin, CreateView):
+ model = models.Licence
+ template_name = 'datasources/licence/create.html'
+ context_object_name = 'licence'
+
+ form_class = forms.LicenceForm
+ permission_required = 'datasources.add_licence'
+
+ def form_valid(self, form):
+ try:
+ owner = form.instance.owner
+
+ except models.Licence.owner.RelatedObjectDoesNotExist:
+ form.instance.owner = self.request.user
+
+ return super().form_valid(form)
+
+
+class LicenceDetailView(DetailView):
+ model = models.Licence
+ template_name = 'datasources/licence/detail.html'
+ context_object_name = 'licence'
+
+ def get_context_data(self, **kwargs):
+ context = super().get_context_data()
+
+ context['has_edit_permission'] = self.request.user.is_superuser or self.request.user == self.object.owner
+
+ return context
+
+
+class LicenceUpdateView(OwnerPermissionMixin, UpdateView):
+ model = models.Licence
+ template_name = 'datasources/licence/update.html'
+ context_object_name = 'licence'
+
+ form_class = forms.LicenceForm
+
+ def form_valid(self, form):
+ try:
+ owner = form.instance.owner
+
+ except models.Licence.owner.RelatedObjectDoesNotExist:
+ form.instance.owner = self.request.user
+
+ return super().form_valid(form)
+
+
+class LicenceDeleteView(OwnerPermissionMixin, DeleteView):
+ model = models.Licence
+ template_name = 'datasources/licence/delete.html'
+ context_object_name = 'licence'
+
+ success_url = reverse_lazy('datasources:licence.list')
diff --git a/datasources/views/user_permission_link.py b/datasources/views/user_permission_link.py
new file mode 100644
index 0000000000000000000000000000000000000000..f9d019c66f3b36f94f30e8a89f06b957e169828e
--- /dev/null
+++ b/datasources/views/user_permission_link.py
@@ -0,0 +1,214 @@
+from django.contrib.auth import get_user_model
+from django.contrib.auth.mixins import LoginRequiredMixin
+from django.db.models import F, ObjectDoesNotExist
+from django.http import HttpResponseRedirect
+from django.shortcuts import reverse
+from django.views.generic.detail import DetailView
+from django.views.generic.edit import UpdateView
+
+from profiles.permissions import OwnerPermissionRequiredMixin
+from datasources import forms, models
+
+
+class DataSourceAccessManageView(OwnerPermissionRequiredMixin, DetailView):
+ model = models.DataSource
+ template_name = 'datasources/datasource/manage_access.html'
+ context_object_name = 'datasource'
+
+ permission_required = 'datasources.change_datasource'
+
+ def get_context_data(self, **kwargs):
+ context = super().get_context_data(**kwargs)
+
+ context['all_users'] = get_user_model().objects
+
+ context['permissions_requested'] = models.UserPermissionLink.objects.filter(
+ datasource=self.object,
+ requested__gt=F('granted')
+ ).union(
+ models.UserPermissionLink.objects.filter(
+ datasource=self.object,
+ ).exclude(
+ push_requested=F('push_granted')
+ )
+ )
+
+ context['permissions_granted'] = models.UserPermissionLink.objects.filter(
+ datasource=self.object,
+ requested__lte=F('granted'),
+ push_requested=F('push_granted')
+ )
+
+ return context
+
+
+# TODO check permissions
+class DataSourceAccessGrantView(LoginRequiredMixin, UpdateView):
+ """
+ Manage a user's access to a DataSource.
+
+ Provides a form view to edit permissions, but permissions may also be set using an AJAX POST request.
+ """
+ model = models.UserPermissionLink
+ form_class = forms.PermissionGrantForm
+ context_object_name = 'permission'
+ template_name = 'datasources/user_permission_link/permission_grant.html'
+
+ def get_context_data(self, **kwargs):
+ """
+ Add data source to the context.
+ """
+ context = super().get_context_data()
+
+ context['datasource'] = models.DataSource.objects.get(pk=self.kwargs['pk'])
+
+ return context
+
+ def get_object(self, queryset=None):
+ """
+ Get or create a permission object for the relevant user.
+ """
+ self.datasource = models.DataSource.objects.get(pk=self.kwargs['pk'])
+
+ try:
+ user = get_user_model().objects.get(id=self.request.POST.get('user'))
+
+ except get_user_model().DoesNotExist:
+ user = get_user_model().objects.get(id=self.request.GET.get('user'))
+
+ obj, created = self.model.objects.get_or_create(
+ user=user,
+ datasource=self.datasource
+ )
+
+ # Set default value to approve request - but do not automatically save this
+ obj.granted = obj.requested
+ obj.push_granted = obj.push_requested
+
+ return obj
+
+ def form_valid(self, form):
+ """
+ Automatically grant requests which are either:
+ - Edited by owner / admin
+ - Requests for a reduction in permission level
+ """
+ form.instance.requested = form.instance.granted
+ form.instance.push_requested = form.instance.push_granted
+
+ if form.instance.requested == models.UserPermissionLevels.NONE and not form.instance.push_requested:
+ form.instance.delete()
+
+ else:
+ form.instance.save()
+
+ return HttpResponseRedirect(self.get_success_url())
+
+ def get_success_url(self):
+ """
+ Return to access management view.
+ """
+ return reverse('datasources:datasource.access.manage', kwargs={'pk': self.datasource.pk})
+
+
+class DataSourceAccessRequestView(LoginRequiredMixin, UpdateView):
+ """
+ Request access to a data source, or request changes to an existing permission.
+
+ Provides a form view to edit permission requests, but permissions may also be requested using an AJAX POST request.
+ """
+ model = models.UserPermissionLink
+ form_class = forms.PermissionRequestForm
+ template_name = 'datasources/user_permission_link/permission_request.html'
+
+ def get_context_data(self, **kwargs):
+ """
+ Add data source to the context.
+ """
+ context = super().get_context_data()
+
+ context['datasource'] = models.DataSource.objects.get(pk=self.kwargs['pk'])
+
+ return context
+
+ def get_object(self, queryset=None):
+ """
+ Get or create a permission object for the relevant user.
+ """
+ self.datasource = models.DataSource.objects.get(pk=self.kwargs['pk'])
+ user = self.request.user
+
+ if self.request.user == self.datasource.owner or self.request.user.is_superuser:
+ try:
+ # Let owner and admins edit other user's requests
+ user = get_user_model().objects.get(id=self.request.GET.get('user'))
+
+ except ObjectDoesNotExist:
+ pass
+
+ try:
+ # Is there an existing record?
+ obj = self.model.objects.get(
+ user=user,
+ datasource=self.datasource
+ )
+
+ except self.model.DoesNotExist:
+ # Don't save to DB - only temporary
+ obj = self.model(
+ user=user,
+ datasource=self.datasource
+ )
+
+ return obj
+
+ def get_form(self, form_class=None):
+ """
+ Get form with choices and default value set for user field.
+ """
+ # Authenticated user and any applications for which they are responsible
+ user_choices = [(self.request.user.pk, self.request.user.username)]
+ user_choices += [(app.proxy_user.pk, 'App: ' + app.name) for app in self.request.user.applications.all()]
+
+ user_field_hidden = len(user_choices) <= 1
+
+ if form_class is None:
+ form_class = self.get_form_class()
+
+ return form_class(
+ user_choices=user_choices,
+ user_initial=self.request.user,
+ user_field_hidden=user_field_hidden,
+ **self.get_form_kwargs()
+ )
+
+ def form_valid(self, form):
+ """
+ Automatically grant requests which are either:
+ - Edited by owner / admin
+ - Requests for a reduction in permission level
+ """
+ if form.instance.requested == models.UserPermissionLevels.NONE:
+ form.instance.delete()
+
+ else:
+ if (
+ (self.request.user == self.datasource.owner or self.request.user.is_superuser) or
+ form.instance.granted > form.instance.requested
+ ):
+ form.instance.granted = form.instance.requested
+
+ form.save()
+
+ return HttpResponseRedirect(self.get_success_url())
+
+ def get_success_url(self):
+ """
+ Return to the data source or access management view depending on user class.
+ """
+ if self.request.user == self.datasource.owner or self.request.user.is_superuser:
+ return reverse('datasources:datasource.access.manage', kwargs={'pk': self.datasource.pk})
+
+ return reverse('datasources:datasource.detail', kwargs={'pk': self.datasource.pk})
+
+
diff --git a/deploy/nginx/sites-available/pedasi b/deploy/nginx/sites-available/pedasi
index 99122e917b0d8dcced6d6055678b594ea9e17989..6d405e53bf3b4e4866a20db25afd339d73de162f 100644
--- a/deploy/nginx/sites-available/pedasi
+++ b/deploy/nginx/sites-available/pedasi
@@ -1,14 +1,23 @@
server {
listen 80;
- server_name localhost;
+ server_name localhost pedasi.* pedasi-dev.* *.iotobservatory.io;
+
+ merge_slashes off;
location /favicon.ico {
alias /var/www/pedasi/static/img/favicon.ico;
}
+
location /static/ {
alias /var/www/pedasi/static/;
}
+ location = /report.html {
+ file /var/www/pedasi/report.html;
+ auth_basic "Restricted Content";
+ auth_basic_user_file /etc/nginx/.htpasswd;
+ }
+
location / {
include uwsgi_params;
uwsgi_pass unix:/run/uwsgi/pedasi.sock;
diff --git a/deploy/nginx/sites-available/pedasi.dev b/deploy/nginx/sites-available/pedasi.dev
new file mode 100644
index 0000000000000000000000000000000000000000..e7a65daa508ee3282fc157c7a3bd830886fecf74
--- /dev/null
+++ b/deploy/nginx/sites-available/pedasi.dev
@@ -0,0 +1,24 @@
+server {
+ listen 80;
+
+ merge_slashes off;
+
+ location = /favicon.ico {
+ alias /var/www/pedasi/static/img/favicon.ico;
+ }
+
+ location /static/ {
+ alias /var/www/pedasi/static/;
+ }
+
+ location = /report.html {
+ alias /var/www/pedasi/report.html;
+ auth_basic "Restricted Content";
+ auth_basic_user_file /etc/nginx/.htpasswd;
+ }
+
+ location / {
+ include uwsgi_params;
+ uwsgi_pass unix:/run/uwsgi/pedasi.sock;
+ }
+}
diff --git a/pedasi/common/base_models.py b/pedasi/common/base_models.py
deleted file mode 100644
index 358d06d2299b1edd9ba21a6b8a0589b537c24d9e..0000000000000000000000000000000000000000
--- a/pedasi/common/base_models.py
+++ /dev/null
@@ -1,23 +0,0 @@
-from django.db import models
-
-
-#: Length of CharFields used to hold the names of objects
-MAX_LENGTH_NAME = 63
-
-
-class BaseAppDataModel(models.Model):
- #: Friendly name of this application
- name = models.CharField(max_length=MAX_LENGTH_NAME,
- blank=False, null=False)
-
- #: A brief description
- description = models.TextField(blank=True, null=False)
-
- #: Address at which the API may be accessed
- url = models.URLField(blank=False, null=False)
-
- def __str__(self):
- return self.name
-
- class Meta:
- abstract = True
diff --git a/pedasi/routers.py b/pedasi/routers.py
deleted file mode 100644
index 6d935dd92e442684a20c144ffffc577181c6dffa..0000000000000000000000000000000000000000
--- a/pedasi/routers.py
+++ /dev/null
@@ -1,32 +0,0 @@
-class DefaultRouter:
- """
- Django database router to route all models to the default database.
- """
-
- db_name = 'default'
-
- def db_for_read(self, model, **hints):
- """
- Read from default database.
- """
- return self.db_name
-
- def db_for_write(self, model, **hints):
- """
- Write to default database.
- """
- return self.db_name
-
- def allow_relation(self, obj1, obj2, **hints):
- """
- Allow relation if both objects are stored in the default database.
- """
- if obj1._state.db == self.db_name and obj2._state.db == self.db_name:
- return True
- return None
-
- def allow_migrate(self, db, app_label, model_name=None, **hints):
- """
- Always allow migrations.
- """
- return True
diff --git a/pedasi/settings.py b/pedasi/settings.py
index 8f4ba12ee20204b5031025ca936792e06843b277..990ea0db8172b2ab1c2ff3b408188fd9b41369cd 100644
--- a/pedasi/settings.py
+++ b/pedasi/settings.py
@@ -43,6 +43,7 @@ from django.urls import reverse_lazy
from decouple import config
import dj_database_url
+import mongoengine
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
@@ -53,7 +54,16 @@ SECRET_KEY = config('SECRET_KEY')
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = config('DEBUG', default=False, cast=bool)
-ALLOWED_HOSTS = []
+if DEBUG:
+ ALLOWED_HOSTS = [
+ '*',
+ ]
+
+else:
+ ALLOWED_HOSTS = [
+ 'localhost',
+ 'pedasi-dev.eastus.cloudapp.azure.com',
+ ]
# Application definition
@@ -68,20 +78,28 @@ DJANGO_APPS = [
]
THIRD_PARTY_APPS = [
+ 'corsheaders',
'bootstrap4',
+ 'haystack',
+ 'rest_framework',
+ 'rest_framework.authtoken',
+ 'social_django',
]
CUSTOM_APPS = [
- 'profiles',
+ 'profiles.apps.ProfilesConfig', # Refer to AppConfig directly since we override the .ready() method
'applications',
'datasources',
- 'prov',
+ 'provenance',
+ 'core',
+ 'api',
]
# Custom apps have to be listed before Django apps so they override default templates
INSTALLED_APPS = CUSTOM_APPS + THIRD_PARTY_APPS + DJANGO_APPS
MIDDLEWARE = [
+ 'corsheaders.middleware.CorsMiddleware',
'django.middleware.security.SecurityMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
'django.middleware.common.CommonMiddleware',
@@ -104,6 +122,7 @@ TEMPLATES = [
'django.template.context_processors.request',
'django.contrib.auth.context_processors.auth',
'django.contrib.messages.context_processors.messages',
+ 'social_django.context_processors.backends',
],
},
},
@@ -121,21 +140,35 @@ DATABASES = {
default='sqlite:///' + os.path.join(BASE_DIR, 'db.sqlite3'),
cast=dj_database_url.parse
),
- # Separate PROV models into their own MongoDB database
- 'prov': {
- 'ENGINE': 'djongo',
- 'NAME': config(
- 'PROV_DATABASE_NAME',
- default='prov'
- )
+}
+
+mongoengine.register_connection(
+ host=config(
+ 'PROV_DATABASE_URL',
+ default='mongodb://localhost/prov',
+ ),
+ alias='default'
+)
+
+mongoengine.register_connection(
+ host=config(
+ 'INTERNAL_DATABASE_URL',
+ default='mongodb://localhost/internal_data'
+ ),
+ alias='internal_data',
+)
+
+
+# Search backend
+
+HAYSTACK_CONNECTIONS = {
+ 'default': {
+ 'ENGINE': 'haystack.backends.whoosh_backend.WhooshEngine',
+ 'PATH': os.path.join(BASE_DIR, 'whoosh_index'),
}
}
-# Database routers direct models into the correct database
-DATABASE_ROUTERS = [
- 'prov.routers.ProvRouter',
- 'pedasi.routers.DefaultRouter',
-]
+HAYSTACK_SIGNAL_PROCESSOR = 'haystack.signals.RealtimeSignalProcessor'
# Password validation
@@ -156,6 +189,72 @@ AUTH_PASSWORD_VALIDATORS = [
},
]
+
+# Social auth app configuration
+
+SOCIAL_AUTH_GOOGLE_OAUTH2_KEY = config('SOCIAL_AUTH_GOOGLE_OAUTH2_KEY', default=None)
+SOCIAL_AUTH_GOOGLE_OAUTH2_SECRET = config('SOCIAL_AUTH_GOOGLE_OAUTH2_SECRET', default=None)
+
+AUTHENTICATION_BACKENDS = [
+ 'django.contrib.auth.backends.ModelBackend',
+]
+
+if SOCIAL_AUTH_GOOGLE_OAUTH2_KEY is not None and SOCIAL_AUTH_GOOGLE_OAUTH2_SECRET is not None:
+ AUTHENTICATION_BACKENDS += [
+ 'social_core.backends.google.GoogleOAuth2',
+ ]
+
+SOCIAL_AUTH_PIPELINE = [
+ # Get the information we can about the user and return it in a simple
+ # format to create the user instance later. On some cases the details are
+ # already part of the auth response from the provider, but sometimes this
+ # could hit a provider API.
+ 'social_core.pipeline.social_auth.social_details',
+
+ # Get the social uid from whichever service we're authing thru. The uid is
+ # the unique identifier of the given user in the provider.
+ 'social_core.pipeline.social_auth.social_uid',
+
+ # Verifies that the current auth process is valid within the current
+ # project, this is where emails and domains whitelists are applied (if
+ # defined).
+ 'social_core.pipeline.social_auth.auth_allowed',
+
+ # Checks if the current social-account is already associated in the site.
+ 'social_core.pipeline.social_auth.social_user',
+
+ # Make up a username for this person, appends a random string at the end if
+ # there's any collision.
+ 'social_core.pipeline.user.get_username',
+
+ # Send a validation email to the user to verify its email address.
+ # Disabled by default.
+ # 'social_core.pipeline.mail.mail_validation',
+
+ # Associates the current social details with another user account with
+ # a similar email address. Disabled by default.
+ # 'social_core.pipeline.social_auth.associate_by_email',
+
+ # Create a user account if we haven't found one yet.
+ 'social_core.pipeline.user.create_user',
+ # 'profiles.social_auth.create_user_disabled',
+
+ # Create the record that associates the social account with the user.
+ 'social_core.pipeline.social_auth.associate_user',
+
+ # Populate the extra_data field in the social record with the values
+ # specified by settings (and the default ones like access_token, etc).
+ 'social_core.pipeline.social_auth.load_extra_data',
+
+ # Update the user record with any changed info from the auth service.
+ 'social_core.pipeline.user.user_details',
+]
+
+SOCIAL_AUTH_LOGIN_REDIRECT_URL = reverse_lazy('index')
+LOGIN_REDIRECT_URL = reverse_lazy('index')
+SOCIAL_AUTH_INACTIVE_USER_URL = reverse_lazy('profiles:inactive')
+
+
# Use Argon2 as hashing algorithm for new passwords
PASSWORD_HASHERS = [
@@ -172,6 +271,26 @@ AUTH_USER_MODEL = 'profiles.User'
LOGIN_URL = reverse_lazy('profiles:login')
+# Application API config
+REST_FRAMEWORK = {
+ 'DEFAULT_AUTHENTICATION_CLASSES': [
+ # Authenticate using tokens for normal API use
+ 'rest_framework.authentication.TokenAuthentication',
+ # Allow logged in users to explore the API through the PEDASI web interface
+ 'rest_framework.authentication.SessionAuthentication',
+ ],
+ 'DEFAULT_PERMISSION_CLASSES': [
+ 'rest_framework.permissions.IsAuthenticated',
+ ]
+}
+
+# Add CORS exemption to API endpoints
+
+CORS_ORIGIN_ALLOW_ALL = True
+
+CORS_URLS_REGEX = r'^/api/.*$'
+
+
# Internationalization
# https://docs.djangoproject.com/en/2.0/topics/i18n/
@@ -193,5 +312,6 @@ STATIC_URL = '/static/'
STATIC_ROOT = os.path.join(BASE_DIR, 'static')
STATICFILES_DIRS = [
- os.path.join(BASE_DIR, 'docs', 'build')
+ os.path.join(BASE_DIR, 'pedasi', 'static'),
+ os.path.join(BASE_DIR, 'docs', 'build'),
]
diff --git a/pedasi/static/css/pedasi.css b/pedasi/static/css/pedasi.css
new file mode 100644
index 0000000000000000000000000000000000000000..52c97bf22ccef3469cef9d38ac9280f9c212c692
--- /dev/null
+++ b/pedasi/static/css/pedasi.css
@@ -0,0 +1,21 @@
+a.no-hover {
+ color: inherit;
+ text-decoration: none;
+}
+
+html {
+ position: relative;
+ min-height: 100%;
+}
+
+body {
+ margin-bottom: 75px;
+}
+
+.footer {
+ position: absolute;
+ bottom: 0;
+ width: 100%;
+ height: 60px;
+ line-height: 60px;
+}
diff --git a/pedasi/static/img/IoT_observatory.png b/pedasi/static/img/IoT_observatory.png
new file mode 100644
index 0000000000000000000000000000000000000000..589ea658fee8b966cb46e032135e2463ce2ff56e
Binary files /dev/null and b/pedasi/static/img/IoT_observatory.png differ
diff --git a/pedasi/static/img/favicon.ico b/pedasi/static/img/favicon.ico
new file mode 100644
index 0000000000000000000000000000000000000000..0cf908313b08eea29cf38d64c98e0681385214aa
Binary files /dev/null and b/pedasi/static/img/favicon.ico differ
diff --git a/pedasi/templates/base.html b/pedasi/templates/base.html
index 0a6bf29dfef7fb145e6bb214c22280aadbcdede5..6fd529e68ab1a120342c7a0ef29b05142f01d3f4 100644
--- a/pedasi/templates/base.html
+++ b/pedasi/templates/base.html
@@ -4,73 +4,119 @@
<html lang="en">
<head>
<meta charset="UTF-8">
+ <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
+
<title>PEDASI</title>
+ {% load static %}
+ <link rel="shortcut icon" href="{% static 'img/favicon.ico' %}" type="image/vnd.microsoft.icon"/>
+
{% bootstrap_css %}
- {% bootstrap_javascript jquery=True %}
<!-- Font Awesome - https://fontawesome.com -->
<link rel="stylesheet"
href="https://use.fontawesome.com/releases/v5.2.0/css/solid.css"
integrity="sha384-wnAC7ln+XN0UKdcPvJvtqIH3jOjs9pnKnq9qX68ImXvOGz2JuFoEiCjT8jyZQX2z"
crossorigin="anonymous">
+
<link rel="stylesheet"
href="https://use.fontawesome.com/releases/v5.2.0/css/fontawesome.css"
integrity="sha384-HbmWTHay9psM8qyzEKPc8odH4DsOuzdejtnr+OFtDmOcIVnhgReQ4GZBH7uwcjf6"
crossorigin="anonymous">
+ <!-- BootSwatch Journal theme -->
+ <link href="https://stackpath.bootstrapcdn.com/bootswatch/4.1.3/journal/bootstrap.min.css"
+ rel="stylesheet"
+ integrity="sha384-5C8TGNupopdjruopVTTrVJacBbWqxHK9eis5DB+DYE6RfqIJapdLBRUdaZBTq7mE"
+ crossorigin="anonymous">
+
+ <link rel="stylesheet" href="{% static 'css/pedasi.css' %}">
+
{% block extra_head %}
{% endblock %}
</head>
<body>
- <nav class="navbar navbar-expand-lg navbar-light bg-light">
- <a class="navbar-brand" href="/">PEDASI</a>
- <button class="navbar-toggler" type="button"
- data-toggle="collapse" data-target="#navbarSupportedContent"
- aria-controls="navbarSupportedContent" aria-expanded="false" aria-label="Toggle navigation">
- <span class="navbar-toggler-icon"></span>
- </button>
-
- <div class="collapse navbar-collapse" id="navbarSupportedContent">
- <ul class="navbar-nav mr-auto">
- <li class="nav-item">
- <a class="nav-link" href="{% url 'applications:application.list' %}">Applications</a>
- </li>
- <li class="nav-item">
- <a class="nav-link" href="{% url 'datasources:datasource.list' %}">Data Sources</a>
- </li>
- </ul>
-
- <ul class="navbar-nav">
- {% if request.user.is_authenticated %}
+ <nav class="navbar navbar-expand-lg navbar-dark bg-primary">
+ <div class="container">
+ <a class="navbar-brand" href="/">PEDASI</a>
+ <button class="navbar-toggler" type="button"
+ data-toggle="collapse" data-target="#navbarSupportedContent"
+ aria-controls="navbarSupportedContent" aria-expanded="false" aria-label="Toggle navigation">
+ <span class="navbar-toggler-icon"></span>
+ </button>
+
+ <div class="collapse navbar-collapse" id="navbarSupportedContent">
+ <ul class="navbar-nav mr-auto">
<li class="nav-item">
- <a class="nav-link" href="{% url 'profiles:profile' %}">My Profile</a>
+ <a class="nav-link" href="{% url 'applications:application.list' %}">Applications</a>
</li>
+
<li class="nav-item">
- <a class="nav-link" href="{% url 'profiles:logout' %}">Logout</a>
+ <a class="nav-link" href="{% url 'datasources:datasource.list' %}">Data Sources</a>
</li>
- {% else %}
+
<li class="nav-item">
- <a class="nav-link" href="{% url 'profiles:login' %}">Login</a>
+ <a class="nav-link" href="{% url 'datasources:licence.list' %}">Licences</a>
</li>
- {% endif %}
- </ul>
+
+ {% if request.user.is_superuser %}
+ <li class="nav-item">
+ <a class="nav-link" href="{% url 'admin:index' %}">Admin Panel</a>
+ </li>
+ {% endif %}
+ </ul>
+
+ <ul class="navbar-nav">
+ {% if request.user.is_authenticated %}
+ <span class="navbar-text">
+ Welcome {{ request.user.username }}
+ </span>
+
+ <li class="nav-item">
+ <a class="nav-link" href="{% url 'profiles:profile' %}">My Profile</a>
+ </li>
+
+ <li class="nav-item">
+ <a class="nav-link" href="{% url 'profiles:logout' %}">Logout</a>
+ </li>
+
+ {% else %}
+ {% if 'google-oauth2' in backends.backends %}
+ <li class="nav-item">
+ <a class="nav-link" href="{% url 'social:begin' 'google-oauth2' %}">Google Login</a>
+ </li>
+ {% endif %}
+
+ <li class="nav-item">
+ <a class="nav-link" href="{% url 'profiles:login' %}">Login</a>
+ </li>
+ {% endif %}
+ </ul>
+ </div>
</div>
</nav>
{% block pre_content %}
{% endblock %}
- <div class="container">
- <div class="mt-3"></div>
-
+ <div class="container mt-3">
{% bootstrap_messages %}
{% block content %}
{% endblock %}
</div>
+ <footer class="footer bg-primary">
+ <div class="container">
+ <p class="m-0 text-center text-white">PEDASI: IoT Observatory Demonstrator</p>
+ </div>
+ </footer>
+
+ {% bootstrap_javascript jquery=True %}
+
+ {% block extra_body %}
+ {% endblock %}
</body>
</html>
\ No newline at end of file
diff --git a/pedasi/templates/search/search.html b/pedasi/templates/search/search.html
new file mode 100644
index 0000000000000000000000000000000000000000..c86673a9bec3fbdac9369b09bd40bd08952a66a4
--- /dev/null
+++ b/pedasi/templates/search/search.html
@@ -0,0 +1,45 @@
+{% extends 'base.html' %}
+
+{% load bootstrap4 %}
+
+{% block content %}
+ <h2>Search</h2>
+
+ <form method="get" action=".">
+ <table>
+ {% bootstrap_form form %}
+ <tr>
+ <td> </td>
+ <td>
+ <input type="submit" value="Search">
+ </td>
+ </tr>
+ </table>
+
+ {% if query %}
+ <h3>Results</h3>
+
+ {% for result in page.object_list %}
+ <p>
+ <a href="{{ result.object.get_absolute_url }}">{{ result.object.name }}</a>
+ </p>
+ <p>
+ {% load highlight %}
+ {% highlight result.text with query html_tag "mark" %}
+ </p>
+ {% empty %}
+ <p>No results found.</p>
+ {% endfor %}
+
+ {% if page.has_previous or page.has_next %}
+ <div>
+ {% if page.has_previous %}<a href="?q={{ query }}&page={{ page.previous_page_number }}">{% endif %}« Previous{% if page.has_previous %}</a>{% endif %}
+ |
+ {% if page.has_next %}<a href="?q={{ query }}&page={{ page.next_page_number }}">{% endif %}Next »{% if page.has_next %}</a>{% endif %}
+ </div>
+ {% endif %}
+ {% else %}
+ {# Show some example queries to run, maybe query syntax, something else? #}
+ {% endif %}
+ </form>
+{% endblock %}
\ No newline at end of file
diff --git a/pedasi/urls.py b/pedasi/urls.py
index 94b1fb13936809779fbb4dbfd63db9c947e2409e..014c0b315b4803f6b15e0639fc5d34b2fb64c605 100644
--- a/pedasi/urls.py
+++ b/pedasi/urls.py
@@ -18,6 +18,7 @@ from django.urls import include, path
from profiles.views import IndexView
+
urlpatterns = [
path('admin/', admin.site.urls),
@@ -39,4 +40,15 @@ urlpatterns = [
include('datasources.urls',
namespace='datasources')
),
+
+ path('api/',
+ include('api.urls',
+ namespace='api')),
+
+ path('search/',
+ include('haystack.urls')),
+
+ path('social/',
+ include('social_django.urls',
+ namespace='social')),
]
diff --git a/playbook.yml b/playbook.yml
index 9c34cc06c621d148831b8e721070bce443405e2c..0d00be0307246ffc30f277e877f343dadb1d4d66 100644
--- a/playbook.yml
+++ b/playbook.yml
@@ -26,32 +26,65 @@
repo: deb [ arch=amd64 ] https://repo.mongodb.org/apt/ubuntu bionic/mongodb-org/4.0 multiverse
filename: mongodb-org-4.0
+ - name: Add Certbot repo
+ apt_repository:
+ repo: ppa:certbot/certbot
+
- name: Update apt
apt:
update_cache: yes
upgrade: yes
- name: Install apt prerequisites
- apt: name={{ item }} state=latest update_cache=yes
- with_items:
- - nginx
- - python3
- - python3-dev
- - python3-pip
- - python3-venv
- - git
- - mysql-server
- - libmysqlclient-dev
- - mongodb-org
- # Required for Ansible to setup DB
- - python3-mysqldb
+ apt:
+ name: '{{ packages }}'
+ state: latest
+ update_cache: yes
+ vars:
+ packages:
+ - nginx
+ - python3
+ - python3-dev
+ - python3-pip
+ - python3-venv
+ - git
+ - mysql-server
+ - libmysqlclient-dev
+ - mongodb-org
+ - goaccess
+ - apache2-utils
+ - python-certbot-nginx
+ # Required for Ansible to setup DB
+ - python3-mysqldb
- name: Setup dev deployment
block:
+ - name: Check if running under Vagrant
+ stat:
+ path: /vagrant
+ register: vagrant_dir
+
- name: Clone / update from local repo
git:
repo: '/vagrant'
dest: '{{ project_dir }}'
+ when: vagrant_dir.stat.exists == True
+
+ - name: Copy deploy key
+ copy:
+ src: 'deploy/.deployment-key'
+ dest: '~/.deployment-key'
+ mode: 0600
+ when: vagrant_dir.stat.exists == False
+
+ - name: Clone / update dev branch from main repo
+ git:
+ repo: 'ssh://git@github.com/Southampton-RSG/PEDASI-IoT.git'
+ dest: '{{ project_dir }}'
+ accept_hostkey: yes
+ key_file: '~/.deployment-key'
+ version: dev
+ when: vagrant_dir.stat.exists == False
- name: Copy dev settings
copy:
@@ -127,18 +160,6 @@
path: '{{ project_dir }}'
mode: 0775
- - name: Compile documentation
- make:
- chdir: '{{ project_dir }}/docs'
- target: '{{ item }}'
- params:
- SPHINXBUILD: '{{ venv_dir }}/bin/sphinx-build'
- SPHINXAPIDOC: '{{ venv_dir }}/bin/sphinx-apidoc'
- loop:
- - clean
- - apidoc
- - html
-
- name: Create static directory
file:
path: '{{ project_dir }}/static'
@@ -147,11 +168,11 @@
group: www-data
mode: 0775
- - name: Perform Django setup
- django_manage: command={{ item }} app_path={{ project_dir }} virtualenv={{ venv_dir }}
- with_items:
- - migrate
- - collectstatic
+ - name: Run Django migrations
+ django_manage:
+ command: migrate
+ app_path: "{{ project_dir }}"
+ virtualenv: "{{ venv_dir }}"
- name: Install uWSGI
pip:
@@ -165,15 +186,32 @@
mode: 0755
- name: Copy web config files
- synchronize: src={{ item.src }} dest={{ item.dest }}
+ file:
+ src: '{{ item.src }}'
+ dest: '{{ item.dest }}'
+ state: link
+ force: yes
with_items:
- - { src: deploy/uwsgi/sites/pedasi.ini, dest: /etc/uwsgi/sites/pedasi.ini }
- - { src: deploy/systemd/system/uwsgi.service, dest: /etc/systemd/system/uwsgi.service }
+ - { src: '{{ project_dir }}/deploy/uwsgi/sites/pedasi.ini', dest: /etc/uwsgi/sites/pedasi.ini }
+ - { src: '{{ project_dir }}/deploy/systemd/system/uwsgi.service', dest: /etc/systemd/system/uwsgi.service }
- - name: Copy web config files
- synchronize:
- src: deploy/nginx/sites-available/pedasi
+ - name: Copy web config files - dev
+ copy:
+ src: '{{ project_dir }}/deploy/nginx/sites-available/pedasi.dev'
dest: /etc/nginx/sites-available/pedasi
+ remote_src: yes
+ # WARNING: this will not update an existing file
+ force: no
+ when: production is not defined
+
+ - name: Copy web config files - production
+ copy:
+ src: '{{ project_dir }}/deploy/nginx/sites-available/pedasi'
+ dest: /etc/nginx/sites-available/pedasi
+ remote_src: yes
+ # WARNING: this will not update an existing file
+ force: no
+ when: production is defined
- name: Activate Nginx site
file:
@@ -188,7 +226,64 @@
- uwsgi
- mongod
+ - name: Set permissions on report.html
+ file:
+ path: "{{ project_dir }}/report.html"
+ state: touch
+ owner: www-data
+ group: www-data
+
+ # Interferes with wildcard server_name in dev deployment
+ - name: Remove default Nginx server config
+ file:
+ path: /etc/nginx/sites-enabled/default
+ state: absent
+
+ - name: Restart Nginx
+ systemd:
+ name: nginx
+ state: restarted
+
- name: Open firewall
ufw:
rule: allow
- name: 'Nginx Full'
+ name: "{{ item }}"
+ state: enabled
+ with_items:
+ - 'Nginx Full'
+ - 'OpenSSH'
+
+ - name: Setup Goaccess Cron job
+ cron:
+ name: "Generate Goaccess report"
+ user: www-data
+ state: present
+ minute: "*/10"
+ job: "{{ project_dir }}/scripts/goaccess.sh {{ project_dir }}/report.html"
+
+ - name: Setup external API access counters reset Cron job
+ cron:
+ name: "Reset external API access counters"
+ user: www-data
+ state: present
+ hour: 0
+ minute: 0
+ job: "{{ venv_dir }}/bin/python {{ project_dir }}/manage.py reset_api_count"
+
+ - name: Compile documentation
+ make:
+ chdir: '{{ project_dir }}/docs'
+ target: '{{ item }}'
+ params:
+ SPHINXBUILD: '{{ venv_dir }}/bin/sphinx-build'
+ SPHINXAPIDOC: '{{ venv_dir }}/bin/sphinx-apidoc'
+ loop:
+ - clean
+ - apidoc
+ - html
+
+ - name: Django collect static files
+ django_manage:
+ command: collectstatic
+ app_path: "{{ project_dir }}"
+ virtualenv: "{{ venv_dir }}"
diff --git a/profiles/apps.py b/profiles/apps.py
index 5501fdad352cf444c6a7fd60ecd52b5d00729aa9..51cb0a78fe63eacbca8d06e30a61b06a8d1bae35 100644
--- a/profiles/apps.py
+++ b/profiles/apps.py
@@ -1,5 +1,54 @@
+import logging
+
from django.apps import AppConfig
+from django.db.utils import ProgrammingError
+
+
+logger = logging.getLogger(__name__)
class ProfilesConfig(AppConfig):
name = 'profiles'
+
+ @staticmethod
+ def create_groups():
+ """
+ Create the necessary groups for Application and Data Source managers.
+ """
+ from django.contrib.auth.models import Group, Permission
+
+ app_providers, created = Group.objects.get_or_create(
+ name='Application Providers',
+ )
+ app_providers.permissions.add(
+ Permission.objects.get(codename='add_application'),
+ Permission.objects.get(codename='change_application'),
+ Permission.objects.get(codename='delete_application')
+ )
+
+ data_providers, created = Group.objects.get_or_create(
+ name='Data Providers',
+ )
+ data_providers.permissions.add(
+ Permission.objects.get(codename='add_datasource'),
+ Permission.objects.get(codename='change_datasource'),
+ Permission.objects.get(codename='delete_datasource'),
+ )
+
+ try:
+ data_providers.permissions.add(
+ Permission.objects.get(codename='add_licence'),
+ Permission.objects.get(codename='change_licence'),
+ Permission.objects.get(codename='delete_licence')
+ )
+
+ except Permission.DoesNotExist:
+ logger.warning('Licence permissions not found - please restart Django server')
+
+ def ready(self):
+ # Runs after app registry is populated - i.e. all models exist and are importable
+ try:
+ self.create_groups()
+
+ except ProgrammingError:
+ logging.warning('Could not create Group fixtures, database has not been initialized')
diff --git a/profiles/fixtures/auth.Group.json b/profiles/fixtures/auth.Group.json
deleted file mode 100644
index 7e4028da3db52c129712684ae1aa624126e7287f..0000000000000000000000000000000000000000
--- a/profiles/fixtures/auth.Group.json
+++ /dev/null
@@ -1,26 +0,0 @@
-[
- {
- "model": "auth.group",
- "pk": 1,
- "fields": {
- "name": "Application Providers",
- "permissions": [
- 19,
- 20,
- 21
- ]
- }
- },
- {
- "model": "auth.group",
- "pk": 2,
- "fields": {
- "name": "Data Providers",
- "permissions": [
- 22,
- 23,
- 24
- ]
- }
- }
-]
\ No newline at end of file
diff --git a/profiles/models.py b/profiles/models.py
index ee4f2a2c254dec7622019818076909fe70119938..9e97bb33adacd1128297d29bbe086de53769bce8 100644
--- a/profiles/models.py
+++ b/profiles/models.py
@@ -1,4 +1,5 @@
from django.contrib.auth.models import AbstractUser
+from django.urls import reverse
class User(AbstractUser):
@@ -6,4 +7,10 @@ class User(AbstractUser):
Custom Django user model to allow for additional functionality to be
added more easily in the future.
"""
- pass
+ def get_uri(self):
+ """
+ Get a URI for this user.
+
+ Used in PROV records.
+ """
+ return reverse('profiles:uri', kwargs={'pk': self.pk})
diff --git a/profiles/permissions.py b/profiles/permissions.py
index 3c32a3638b97f9b124119e69b15bf2a5550330f0..dd58254727d8a73532135ff5277ff4ef25ae7224 100644
--- a/profiles/permissions.py
+++ b/profiles/permissions.py
@@ -1,4 +1,4 @@
-from django.contrib.auth.mixins import PermissionRequiredMixin
+from django.contrib.auth.mixins import UserPassesTestMixin, PermissionRequiredMixin
class OwnerPermissionRequiredMixin(PermissionRequiredMixin):
@@ -15,4 +15,15 @@ class OwnerPermissionRequiredMixin(PermissionRequiredMixin):
:return: Does the user have permission to perform this action?
"""
- return super().has_permission() and self.request.user == getattr(self.get_object(), self.owner_attribute)
+ return self.request.user.is_superuser or (
+ super().has_permission() and
+ self.request.user == getattr(self.get_object(), self.owner_attribute)
+ )
+
+
+class HasViewPermissionMixin(UserPassesTestMixin):
+ """
+ Mixin to reject users who do not have permission to view this DataSource.
+ """
+ def test_func(self) -> bool:
+ return self.get_object().has_view_permission(self.request.user)
diff --git a/profiles/social_auth.py b/profiles/social_auth.py
new file mode 100644
index 0000000000000000000000000000000000000000..b4d320fc414d66500d14aa5320443f2a3174c39c
--- /dev/null
+++ b/profiles/social_auth.py
@@ -0,0 +1,13 @@
+from social_core.pipeline.user import create_user
+
+
+def create_user_disabled(strategy, details, backend, user=None, *args, **kwargs):
+ # Returns dict containing 'is_new' and 'user'
+ result = create_user(strategy, details, backend, user, *args, **kwargs)
+
+ if result['is_new']:
+ django_user = result['user']
+ django_user.is_active = False
+ django_user.save()
+
+ return result
diff --git a/profiles/static/css/masthead.css b/profiles/static/css/masthead.css
index d24332afd4e3ddbf83306c4240f73f26d0464e39..56fb41ee26eaff0c75341a8dd745092b1e83915b 100644
--- a/profiles/static/css/masthead.css
+++ b/profiles/static/css/masthead.css
@@ -1,12 +1,15 @@
header.masthead {
position: relative;
- background: #343a40 url("https://via.placeholder.com/350x150") no-repeat top;
+ background: #343a40 no-repeat center;
-webkit-background-size: cover;
-moz-background-size: cover;
-o-background-size: cover;
background-size: cover;
padding-top: 8rem;
padding-bottom: 8rem;
+ min-height: 200px;
+ height: 60vh;
+ z-index: -2;
}
header.masthead .overlay {
@@ -16,7 +19,8 @@ header.masthead .overlay {
width: 100%;
top: 0;
left: 0;
- opacity: 0.1;
+ opacity: 0.4;
+ z-index: -1;
}
header.masthead .textbox-container {
diff --git a/profiles/static/img/pedasi-hero.jpg b/profiles/static/img/pedasi-hero.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..e8e22c5f4263a24e90fad132534da9f1876dde34
Binary files /dev/null and b/profiles/static/img/pedasi-hero.jpg differ
diff --git a/profiles/templates/index.html b/profiles/templates/index.html
index c6f7e5bdffa385cc2fc7028622471c2da804f7fe..732e933b68e7d3cb73856451f423433a64d5ed08 100644
--- a/profiles/templates/index.html
+++ b/profiles/templates/index.html
@@ -7,49 +7,166 @@
{% endblock %}
{% block pre_content %}
- <header class="masthead text-white text-left">
+ <header class="container-fluid masthead text-white text-left"
+ style="background-image: url('{% static 'img/pedasi-hero.jpg' %}')">
<div class="overlay"></div>
+
+ <div class="row">
+ <div class="ml-5 px-4 mt-3 pt-3 textbox-container">
+ <h2 class="display-1">PEDASI</h2>
+ <p class="lead">IoT Observatory Demonstrator</p>
+ </div>
+ </div>
+ </header>
+
+ <div class="bg-secondary py-3">
+ <div class="container text-white">
+ <h2>
+ A platform for research in data sharing
+ </h2>
+ </div>
+ </div>
+
+ <div class="bg-light py-2">
<div class="container">
<div class="row">
- <div class="px-4 pt-3 textbox-container">
- <h2 class="display-1">PEDASI</h2>
- <p class="lead">IoT Observatory Demonstrator</p>
+
+ <div class="col-md-4">
+ <div class="card text-center">
+ <div class="card-body">
+ <h2>Find Data Sources</h2>
+
+ <span class="fas fa-5x fa-atlas"></span>
+ </div>
+ </div>
</div>
+
+ <div class="col-md-4">
+ <div class="card text-center">
+ <div class="card-body">
+ <h2>Access Data</h2>
+
+ <span class="fas fa-5x fa-cloud-download-alt"></span>
+ </div>
+ </div>
+ </div>
+
+ <div class="col-md-4">
+ <div class="card text-center">
+ <div class="card-body">
+ <h2>Share Results</h2>
+
+ <span class="fas fa-5x fa-tablet-alt"></span>
+ </div>
+ </div>
+ </div>
+
</div>
</div>
- </header>
+ </div>
{% endblock %}
{% block content %}
- <h2>Recent Data Sources</h2>
+ <div class="container pb-3">
+ <div class="row align-items-center">
+ <div class="col-sm-8">
+ <h2 class="pb-2">About PEDASI</h2>
+
+ <p>
+ This project will address issues related to sharing IoT datasets on a large distributed scale to support
+ innovation in a way that will not compromise privacy and security. The project will identify and address
+ infrastructural, technological and legal issues to that end, and will initiate the deployment of an
+ infrastructure that will enable individuals or organisations to share IoT datasets.
+ </p>
- <div class="row">
- {% for datasource in datasources %}
- <div class="card col-lg-4 col-md-6 border-0">
- <div class="card-body d-flex flex-column">
- <h5 class="card-title">{{ datasource.name }}</h5>
- <p class="card-text">{{ datasource.description }}</p>
+ <p>
+ This activity works in synergy with existing initiatives within the Web Science and Internet Science communities.
+ It will provide for accessing IoT datasets already available on the Web and it will aim to host datasets from the
+ experiments of the PETRAS project providing a vehicle for community engagement for the development of analytics
+ and visualisations on those datasets across the PETRAS community and beyond.
+ </p>
+ </div>
- <a href="{% url 'datasources:datasource.detail' pk=datasource.pk %}"
- class="btn btn-primary btn-block mt-auto" role="button">Detail</a>
+ <div class="col-sm-4">
+ <img class="img-fluid py-3" src="{% static 'img/IoT_observatory.png' %}">
</div>
</div>
- {% endfor %}
</div>
- <h2 class="mt-3">Recent Applications</h2>
+ <div class="jumbotron jumbotron-fluid pt-3 pb-2 mb-3">
+ <div class="container">
+ <h2>Featured Applications</h2>
- <div class="row">
- {% for application in applications %}
- <div class="card col-lg-4 col-md-6 border-0">
- <div class="card-body d-flex flex-column">
- <h5 class="card-title">{{ application.name }}</h5>
- <p class="card-text">{{ application.description }}</p>
+ <div class="row px-2">
+
+ {% for application in applications %}
+ <div class="col-md-6 col-lg-4 p-2">
+ <a href="{% url 'applications:application.detail' pk=application.pk %}"
+ class="no-hover" role="button">
+
+ <div class="card rounded-0 h-100">
+ <img src="{% static 'img/IoT_observatory.png' %}"
+ class="card-img-top rounded-0 mt-4 mx-auto d-block"
+ style="width: 150px;">
+
+ <div class="card-body d-flex flex-column pb-2">
+ <h5 class="card-title">{{ application.name }}</h5>
+ <p class="card-text">{{ application.description|truncatechars:120 }}</p>
+
+ <p class="card-text text-right mt-auto">
+ <small class="text-muted">
+ Last updated X ago
+ </small>
+ </p>
+
+ </div>
+ </div>
+
+ </a>
+ </div>
+ {% endfor %}
- <a href="{% url 'applications:application.detail' pk=application.pk %}"
- class="btn btn-primary btn-block mt-auto" role="button">Detail</a>
- </div>
</div>
- {% endfor %}
+
+ </div>
</div>
+
+ <div class="jumbotron jumbotron-fluid pt-3 pb-2">
+ <div class="container">
+ <h2>Featured Data Sources</h2>
+
+ <div class="row px-2">
+
+ {% for datasource in datasources %}
+ <div class="col-md-6 col-lg-4 p-2">
+ <a href="{% url 'datasources:datasource.detail' pk=datasource.pk %}"
+ class="no-hover" role="button">
+
+ <div class="card rounded-0 h-100">
+ <img src="{% static 'img/IoT_observatory.png' %}"
+ class="card-img-top rounded-0 mt-4 mx-auto d-block"
+ style="width: 150px;">
+
+ <div class="card-body d-flex flex-column pb-2">
+ <h5 class="card-title">{{ datasource.name }}</h5>
+ <p class="card-text">{{ datasource.description|truncatechars:120 }}</p>
+
+ <p class="card-text text-right mt-auto">
+ <small class="text-muted">
+ {{ datasource.external_requests }} recent connection{{ datasource.external_requests|pluralize }}
+ </small>
+ </p>
+
+ </div>
+ </div>
+
+ </a>
+ </div>
+ {% endfor %}
+
+ </div>
+
+ </div>
+ </div>
+
{% endblock %}
\ No newline at end of file
diff --git a/profiles/templates/profiles/user/inactive.html b/profiles/templates/profiles/user/inactive.html
new file mode 100644
index 0000000000000000000000000000000000000000..8a4681c915a4d157ac50fd6cce42eb44d844a85f
--- /dev/null
+++ b/profiles/templates/profiles/user/inactive.html
@@ -0,0 +1,21 @@
+{% extends "base.html" %}
+{% load bootstrap4 %}
+
+{% block content %}
+ <nav aria-label="breadcrumb">
+ <ol class="breadcrumb">
+ <li class="breadcrumb-item" aria-current="page">
+ <a href="{% url 'index' %}">Home</a>
+ </li>
+ <li class="breadcrumb-item active" aria-current="page">
+ My Profile
+ </li>
+ </ol>
+ </nav>
+
+ <div class="alert alert-warning">
+ <b>Inactive Account:</b>
+ Your account has not yet been activated - please wait for an admin to approve your request.
+ </div>
+
+{% endblock %}
\ No newline at end of file
diff --git a/profiles/templates/profiles/user/profile.html b/profiles/templates/profiles/user/profile.html
index 0871fc9a389c85543c3c51e5b96910ac795fbac3..52af26156d30256b035cd272c4b05353c51917e5 100644
--- a/profiles/templates/profiles/user/profile.html
+++ b/profiles/templates/profiles/user/profile.html
@@ -15,4 +15,40 @@
<h2>My Profile</h2>
+ <table class="table">
+ <thead>
+ <th scope="col" class="w-25 border-0"></th>
+ <th scope="col" class="border-0"></th>
+ </thead>
+
+ <tbody>
+ <tr>
+ <td>API Token</td>
+ <td>
+ <span id="spanApiToken">
+ {% if user == request.user and user.auth_token %}
+ {{ user.auth_token }}
+
+ {% else %}
+ <script type="application/javascript">
+ function getToken() {
+ $.ajax({
+ dataType: "json",
+ url: "{% url 'profiles:token' %}",
+ data: null,
+ success: function (data) {
+ $('#spanApiToken').text(data.data.token.key);
+ }
+ });
+ }
+ </script>
+
+ <button onclick="getToken();" class="btn btn-default" role="button">Generate an API Token</button>
+ {% endif %}
+ </span>
+ </td>
+ </tr>
+ </tbody>
+ </table>
+
{% endblock %}
\ No newline at end of file
diff --git a/profiles/templates/registration/login.html b/profiles/templates/registration/login.html
index 92c590526b51ee39fce2a7f9247985ddbf483735..51b8b156359c72bf61ddf3b1dac1b1322c544fa6 100644
--- a/profiles/templates/registration/login.html
+++ b/profiles/templates/registration/login.html
@@ -4,11 +4,27 @@
{% block content %}
<h2>Login</h2>
- {% if form.errors %}
- <div> </div>
- {% else %}
- <div class="alert alert-warning">Please log in</div>
- {% endif %}
+ {% if 'google-oauth2' in backends.backends %}
+ <div class="alert alert-info mt-3">
+ <p>
+ The preferred method to log in to PEDASI is using your Google account.
+ </p>
+ <p>
+ If you do not have a Google account and wish to use PEDASI you should contact a
+ system administrator to create a PEDASI account for you.
+
+ Once your account has been created you may log in using the form below.
+ </p>
+ </div>
+
+ <div class="row justify-content-center">
+ <a class="btn btn-lg btn-success col-sm-4"
+ role="button" href="{% url 'social:begin' 'google-oauth2' %}">Google Login</a>
+
+ </div>
+ {% endif %}
+
+ <hr>
<div>
<form action="" method="post">
diff --git a/profiles/tests.py b/profiles/tests.py
new file mode 100644
index 0000000000000000000000000000000000000000..75e3d2c60ce4d5b15446147a96409ce94083fc6a
--- /dev/null
+++ b/profiles/tests.py
@@ -0,0 +1,64 @@
+from django.test import Client, TestCase
+from django.urls import reverse
+
+from rest_framework.authtoken.models import Token
+
+from . import models
+
+
+class UserTest(TestCase):
+ def test_user_uri(self):
+ """
+ Test that we can get a URI for a user.
+ """
+ user = models.User.objects.create_user('Test User')
+ uri = user.get_uri()
+
+ self.assertEqual(str, type(uri))
+ self.assertLessEqual(1, len(uri))
+
+ client = Client()
+ response = client.get(uri)
+
+ self.assertEqual(200, response.status_code)
+
+ content = response.json()
+ self.assertIn('status', content)
+ self.assertIn('data', content)
+
+ data = content['data']
+ self.assertIn('user', data)
+ self.assertIn('pk', data['user'])
+ self.assertEqual(user.pk, data['user']['pk'])
+
+ def test_user_get_token(self):
+ """
+ Check that we can get an API Token for a given user.
+ """
+ user = models.User.objects.create_user('Test User')
+
+ client = Client()
+ client.force_login(user)
+
+ with self.assertRaises(Token.DoesNotExist):
+ # User should not already have token
+ token = Token.objects.get(user=user)
+
+ response = client.get(reverse('profiles:token'))
+
+ self.assertEqual(200, response.status_code)
+
+ content = response.json()
+ self.assertIn('status', content)
+ self.assertIn('data', content)
+ self.assertEqual('success', content['status'])
+
+ data = content['data']
+ self.assertIn('token', data)
+ self.assertIn('key', data['token'])
+
+ key = data['token']['key']
+
+ token = Token.objects.get(user=user)
+
+ self.assertEqual(key, token.key)
diff --git a/profiles/urls.py b/profiles/urls.py
index ba5a39b9b30c189dea5eb5c5d382bb844db42eb0..1c71dc1268c0de04e2ea8bc075da22834d6a701f 100644
--- a/profiles/urls.py
+++ b/profiles/urls.py
@@ -1,6 +1,6 @@
from django.urls import include, path
-from .views.views import UserProfileView
+from . import views
app_name = 'profiles'
@@ -8,6 +8,18 @@ urlpatterns = [
path('', include('django.contrib.auth.urls')),
path('profile',
- UserProfileView.as_view(),
+ views.UserProfileView.as_view(),
name='profile'),
+
+ path('inactive',
+ views.UserInactiveView.as_view(),
+ name='inactive'),
+
+ path('uri/<int:pk>',
+ views.UserUriView.as_view(),
+ name='uri'),
+
+ path('token',
+ views.UserGetTokenView.as_view(),
+ name='token'),
]
diff --git a/profiles/views.py b/profiles/views.py
new file mode 100644
index 0000000000000000000000000000000000000000..d6cca5d14ec6f69bedf3d8bdf925e267c71e425f
--- /dev/null
+++ b/profiles/views.py
@@ -0,0 +1,86 @@
+from django.contrib.auth import get_user_model
+from django.contrib.auth.mixins import LoginRequiredMixin
+from django.http import JsonResponse
+from django.views.generic import TemplateView
+from django.views.generic.detail import DetailView
+
+from rest_framework.authtoken.models import Token
+
+from applications.models import Application
+from datasources.models import DataSource
+
+
+class IndexView(TemplateView):
+ template_name = 'index.html'
+
+ def get_context_data(self, **kwargs):
+ """
+ Add recent Applications and DataSources to index page.
+
+ :return: Django context dictionary
+ """
+ context = super().get_context_data(**kwargs)
+
+ context['datasources'] = DataSource.objects.order_by('-external_requests')[:6]
+ context['applications'] = Application.objects.order_by('-id')[:3]
+
+ return context
+
+
+class UserProfileView(DetailView):
+ template_name = 'profiles/user/profile.html'
+ context_object_name = 'user'
+
+ def get_object(self, queryset=None):
+ return self.request.user
+
+ def get_context_data(self, **kwargs):
+ context = super().get_context_data(**kwargs)
+
+ return context
+
+
+class UserUriView(DetailView):
+ """
+ View providing verification that a PEDASI User URI exists.
+ """
+ model = get_user_model()
+
+ def render_to_response(self, context, **response_kwargs):
+ return JsonResponse({
+ 'status': 'success',
+ 'data': {
+ 'user': {
+ 'pk': self.object.pk,
+ }
+ }
+ })
+
+
+class UserInactiveView(TemplateView):
+ template_name = 'profiles/user/inactive.html'
+
+
+class UserGetTokenView(LoginRequiredMixin, DetailView):
+ """
+ Get an API Token for the currently authenticated user.
+ """
+ def get_object(self, queryset=None):
+ return self.request.user
+
+ def render_to_response(self, context, **response_kwargs):
+ """
+ Get an existing API Token or create a new one for the currently authenticated user.
+
+ :return: JSON containing Token key
+ """
+ api_token, created = Token.objects.get_or_create(user=self.object)
+
+ return JsonResponse({
+ 'status': 'success',
+ 'data': {
+ 'token': {
+ 'key': api_token.key
+ }
+ }
+ })
diff --git a/profiles/views/__init__.py b/profiles/views/__init__.py
deleted file mode 100644
index 6330c39c81ceef4998d8f53c8e92305046882675..0000000000000000000000000000000000000000
--- a/profiles/views/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-from .views import IndexView
diff --git a/profiles/views/views.py b/profiles/views/views.py
deleted file mode 100644
index 9d1afdcf38cf0f39e7b6631bb5041181cf9b5be8..0000000000000000000000000000000000000000
--- a/profiles/views/views.py
+++ /dev/null
@@ -1,30 +0,0 @@
-from django.views.generic import TemplateView
-from django.views.generic.detail import DetailView
-
-from applications.models import Application
-from datasources.models import DataSource
-
-
-class IndexView(TemplateView):
- template_name = 'index.html'
-
- def get_context_data(self, **kwargs):
- """
- Add recent Applications and DataSources to index page.
-
- :return: Django context dictionary
- """
- context = super().get_context_data(**kwargs)
-
- context['datasources'] = DataSource.objects.order_by('-id')[:3]
- context['applications'] = Application.objects.order_by('-id')[:3]
-
- return context
-
-
-class UserProfileView(DetailView):
- template_name = 'profiles/user/profile.html'
- context_object_name = 'user'
-
- def get_object(self, queryset=None):
- return self.request.user
diff --git a/prov/migrations/0001_initial.py b/prov/migrations/0001_initial.py
deleted file mode 100644
index e3412fe56312e3a66bb915f2de92254fd92a1852..0000000000000000000000000000000000000000
--- a/prov/migrations/0001_initial.py
+++ /dev/null
@@ -1,26 +0,0 @@
-# Generated by Django 2.0.8 on 2018-08-29 12:26
-
-from django.db import migrations, models
-import djongo.models.fields
-import prov.models
-
-
-class Migration(migrations.Migration):
-
- initial = True
-
- dependencies = [
- ]
-
- operations = [
- migrations.CreateModel(
- name='ProvCollection',
- fields=[
- ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
- ('app_label', models.CharField(max_length=100)),
- ('model_name', models.CharField(max_length=100)),
- ('related_pk', models.PositiveIntegerField()),
- ('entries', djongo.models.fields.ArrayModelField(blank=True, default=[], model_container=prov.models.ProvEntry, null=False)),
- ],
- ),
- ]
diff --git a/prov/migrations/0002_editable_false.py b/prov/migrations/0002_editable_false.py
deleted file mode 100644
index f6fbf163386b6c07040d43250d640ccfcb19acf3..0000000000000000000000000000000000000000
--- a/prov/migrations/0002_editable_false.py
+++ /dev/null
@@ -1,23 +0,0 @@
-# Generated by Django 2.0.8 on 2018-08-30 09:40
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('prov', '0001_initial'),
- ]
-
- operations = [
- migrations.AlterField(
- model_name='provcollection',
- name='app_label',
- field=models.CharField(editable=False, max_length=100),
- ),
- migrations.AlterField(
- model_name='provcollection',
- name='model_name',
- field=models.CharField(editable=False, max_length=100),
- ),
- ]
diff --git a/prov/models.py b/prov/models.py
deleted file mode 100644
index d43c6a29531ec91253f45c289b4932b5f82bde1b..0000000000000000000000000000000000000000
--- a/prov/models.py
+++ /dev/null
@@ -1,82 +0,0 @@
-from django.db.models import signals
-from django.dispatch import receiver
-
-from djongo import models
-
-from applications.models import Application
-from datasources.models import DataSource
-
-MAX_LENGTH_NAME_FIELD = 100
-
-
-class ProvEntry(models.Model):
- """
- Stored PROV record for a single action.
-
- e.g. Update a model's metadata, use a model.
- """
- #: Time at which the action occurred
- time = models.DateTimeField(auto_now_add=True,
- editable=False,
- blank=False, null=False)
-
- class Meta:
- # Make this model abstract to avoid creating a table
- # since it will only be used inside a ProvCollection model
- abstract = True
-
-
-class ProvCollection(models.Model):
- """
- The complete set of PROV records storing all actions performed on a single model instance.
- """
- #: App from which the model comes
- app_label = models.CharField(max_length=MAX_LENGTH_NAME_FIELD,
- editable=False,
- blank=False, null=False)
-
- #: Name of the model
- model_name = models.CharField(max_length=MAX_LENGTH_NAME_FIELD,
- editable=False,
- blank=False, null=False)
-
- # TODO should this be editable=False? Can a model PK change?
- #: Primary key of the model instance
- related_pk = models.PositiveIntegerField(blank=False, null=False)
-
- #: List of ProvEntry actions
- entries = models.ArrayModelField(
- model_container=ProvEntry,
- default=[],
- blank=True, null=False
- )
-
- @classmethod
- def for_model_instance(cls, instance: models.Model) -> 'ProvCollection':
- """
- Get the :class:`ProvCollection` instance related to a particular model instance.
-
- Create a :class:`ProvCollection` instance if there is not one already.
-
- :param instance: Model instance for which to get :class:`ProvCollection`
- :return: :class:`ProvCollection` instance
- """
- obj, created = cls.objects.get_or_create(
- app_label=instance._meta.app_label,
- model_name=instance._meta.model_name,
- related_pk=instance.pk
- )
-
- return obj
-
-
-@receiver(signals.post_save, sender=Application)
-@receiver(signals.post_save, sender=DataSource)
-def save_prov(sender, instance, **kwargs):
- """
- Signal receiver to update a ProvCollection when a PROV tracked model is saved.
- """
- obj = ProvCollection.for_model_instance(instance)
-
- obj.entries.append(ProvEntry())
- obj.save()
diff --git a/prov/routers.py b/prov/routers.py
deleted file mode 100644
index 3fa878a2aaa3d59719ac6856ea58a64359236cd7..0000000000000000000000000000000000000000
--- a/prov/routers.py
+++ /dev/null
@@ -1,42 +0,0 @@
-class ProvRouter:
- """
- Django database router to direct models within the PROV app to the correct database.
-
- This is required to separate the PROV models into MongoDB.
-
- This router should be listed before the default router in the Django settings file.
- """
-
- db_name = 'prov'
- app_label = 'prov'
-
- def db_for_read(self, model, **hints):
- """
- Read from default database.
- """
- if model._meta.app_label == self.app_label:
- return self.db_name
- return None
-
- def db_for_write(self, model, **hints):
- """
- Write to default database.
- """
- if model._meta.app_label == self.app_label:
- return self.db_name
- return None
-
- def allow_relation(self, obj1, obj2, **hints):
- """
- Allow relation if a model in this app is involved.
- """
- if obj1._meta.app_label == self.app_label or obj2._meta.app_label == self.app_label:
- return True
- return None
-
- def allow_migrate(self, db, app_label, model_name=None, **hints):
- """
- Always allow migrations.
- """
- if app_label == self.app_label:
- return db == self.db_name
diff --git a/prov/tests.py b/prov/tests.py
deleted file mode 100644
index 59f1ccc17fb75091105a0b863e8a6891e54bc795..0000000000000000000000000000000000000000
--- a/prov/tests.py
+++ /dev/null
@@ -1,77 +0,0 @@
-import unittest
-
-from django.contrib.auth import get_user_model
-from django.test import TestCase
-
-from datasources.models import DataSource
-from prov.models import ProvCollection
-
-
-class ProvCollectionTest(TestCase):
- @classmethod
- def setUpTestData(cls):
- cls.user_model = get_user_model()
- cls.user = cls.user_model.objects.create_user('test')
-
- def setUp(self):
- self.datasource = DataSource.objects.create(
- name='Test Data Source',
- url='http://www.example.com',
- owner=self.user,
- plugin_name='TEST'
- )
-
- def test_prov_datasource_create(self):
- """
- Test that a PROV collection / entry is created when a model is created.
- """
- # PROV record should be created when model is created
- prov_collection = ProvCollection.for_model_instance(self.datasource)
- self.assertEqual(len(prov_collection.entries), 1)
-
- def test_prov_datasource_update(self):
- """
- Test that a new PROV entry is created when a model is updated.
- """
- prov_collection = ProvCollection.for_model_instance(self.datasource)
- n_provs = len(prov_collection.entries)
-
- self.datasource.plugin_name = 'CHANGED'
- self.datasource.save()
-
- # Another PROV record should be created when model is changed and saved
- prov_collection = ProvCollection.for_model_instance(self.datasource)
- self.assertEqual(len(prov_collection.entries), n_provs + 1)
-
- @unittest.expectedFailure
- def test_prov_datasource_null_update(self):
- """
- Test that no new PROV entry is created when a model is saved without changes.
- """
- prov_collection = ProvCollection.for_model_instance(self.datasource)
- n_provs = len(prov_collection.entries)
-
- self.datasource.save()
-
- # No PROV record should be created when saving a model that has not changed
- prov_collection = ProvCollection.for_model_instance(self.datasource)
- self.assertEqual(len(prov_collection.entries), n_provs)
-
- def test_prov_records_distinct(self):
- """
- Test that a distinct PROV collection is created for each model instance.
- """
- prov_collection = ProvCollection.for_model_instance(self.datasource)
-
- new_datasource = DataSource.objects.create(
- name='Another Test Data Source',
- url='http://www.example.com',
- owner=self.user,
- plugin_name='TEST'
- )
- new_prov_collection = ProvCollection.for_model_instance(new_datasource)
-
- self.assertIsNot(prov_collection, new_prov_collection)
-
- self.assertEqual(prov_collection.related_pk, self.datasource.pk)
- self.assertEqual(new_prov_collection.related_pk, new_datasource.pk)
diff --git a/provenance/__init__.py b/provenance/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/prov/apps.py b/provenance/apps.py
similarity index 100%
rename from prov/apps.py
rename to provenance/apps.py
diff --git a/provenance/data/prov-json.schema.json b/provenance/data/prov-json.schema.json
new file mode 100644
index 0000000000000000000000000000000000000000..056a62d2974212e7706d7fe35b7c147b574e1673
--- /dev/null
+++ b/provenance/data/prov-json.schema.json
@@ -0,0 +1,347 @@
+{
+ "id": "http://provenance.ecs.soton.ac.uk/prov-json/schema#",
+ "$schema": "http://json-schema.org/draft-04/schema#",
+ "description": "Schema for a PROV-JSON document",
+ "type": "object",
+ "additionalProperties": false,
+ "properties": {
+ "prefix": {
+ "type": "object",
+ "patternProperties": {
+ "^[a-zA-Z0-9_\\-]+$": { "type" : "string", "format": "uri" }
+ }
+ },
+ "entity": {
+ "type": "object",
+ "additionalProperties": { "$ref":"#/definitions/entity" }
+ },
+ "activity": {
+ "type": "object",
+ "additionalProperties": { "$ref":"#/definitions/activity" }
+ },
+ "agent": {
+ "type": "object",
+ "additionalProperties": { "$ref":"#/definitions/agent" }
+ },
+ "wasGeneratedBy": {
+ "type": "object",
+ "additionalProperties": { "$ref":"#/definitions/generation" }
+ },
+ "used": {
+ "type": "object",
+ "additionalProperties": { "$ref":"#/definitions/usage" }
+ },
+ "wasInformedBy": {
+ "type": "object",
+ "additionalProperties": { "$ref":"#/definitions/communication" }
+ },
+ "wasStartedBy": {
+ "type": "object",
+ "additionalProperties": { "$ref":"#/definitions/start" }
+ },
+ "wasEndedby": {
+ "type": "object",
+ "additionalProperties": { "$ref":"#/definitions/end" }
+ },
+ "wasInvalidatedBy": {
+ "type": "object",
+ "additionalProperties": { "$ref":"#/definitions/invalidation" }
+ },
+ "wasDerivedFrom": {
+ "type": "object",
+ "additionalProperties": { "$ref":"#/definitions/derivation" }
+ },
+ "wasAttributedTo": {
+ "type": "object",
+ "additionalProperties": { "$ref":"#/definitions/attribution" }
+ },
+ "wasAssociatedWith": {
+ "type": "object",
+ "additionalProperties": { "$ref":"#/definitions/association" }
+ },
+ "actedOnBehalfOf": {
+ "type": "object",
+ "additionalProperties": { "$ref":"#/definitions/delegation" }
+ },
+ "wasInfluencedBy": {
+ "type": "object",
+ "additionalProperties": { "$ref":"#/definitions/influence" }
+ },
+ "specializationOf": {
+ "type": "object",
+ "additionalProperties": { "$ref":"#/definitions/specialization" }
+ },
+ "alternateOf": {
+ "type": "object",
+ "additionalProperties": { "$ref":"#/definitions/alternate" }
+ },
+ "hadMember": {
+ "type": "object",
+ "additionalProperties": { "$ref":"#/definitions/membership" }
+ },
+ "bundle": {
+ "type": "object",
+ "additionalProperties": { "$ref":"#/definitions/bundle" }
+ }
+ },
+ "definitions": {
+ "typedLiteral": {
+ "title": "PROV-JSON Typed Literal",
+ "type": "object",
+ "properties": {
+ "$": { "type": "string" },
+ "type": { "type": "string", "format": "uri" },
+ "lang": { "type": "string" }
+ },
+ "required": ["$"],
+ "additionalProperties": false
+ },
+ "stringLiteral": {"type": "string"},
+ "numberLiteral": {"type": "number"},
+ "booleanLiteral": {"type": "boolean"},
+ "literalArray": {
+ "type": "array",
+ "minItems": 1,
+ "items": {
+ "anyOf": [
+ { "$ref": "#/definitions/stringLiteral" },
+ { "$ref": "#/definitions/numberLiteral" },
+ { "$ref": "#/definitions/booleanLiteral" },
+ { "$ref": "#/definitions/typedLiteral" }
+ ]
+ }
+ },
+ "attributeValues": {
+ "anyOf": [
+ { "$ref": "#/definitions/stringLiteral" },
+ { "$ref": "#/definitions/numberLiteral" },
+ { "$ref": "#/definitions/booleanLiteral" },
+ { "$ref": "#/definitions/typedLiteral" },
+ { "$ref": "#/definitions/literalArray" }
+ ]
+ },
+ "entity": {
+ "type": "object",
+ "title": "entity",
+ "additionalProperties": { "$ref": "#/definitions/attributeValues" }
+ },
+ "agent": { "$ref": "#/definitions/entity" },
+ "activity": {
+ "type": "object",
+ "title": "activity",
+ "prov:startTime": { "type": "string", "format": "date-time" },
+ "prov:endTime": { "type": "string", "format": "date-time" },
+ "additionalProperties": { "$ref": "#/definitions/attributeValues" }
+ },
+ "generation": {
+ "type": "object",
+ "title": "generation/usage",
+ "properties": {
+ "prov:entity": { "type": "string", "format": "uri" },
+ "prov:activity": { "type": "string", "format": "uri" },
+ "prov:time": { "type": "string", "format": "date-time" }
+ },
+ "required": ["prov:entity"],
+ "additionalProperties": { "$ref": "#/definitions/attributeValues" }
+ },
+ "usage": {"$ref":"#/definitions/generation"},
+ "communication":{
+ "type": "object",
+ "title": "communication",
+ "properties": {
+ "prov:informant": {"type": "string", "format": "uri"},
+ "prov:informed": {"type": "string", "format": "uri"}
+ },
+ "required": ["prov:informant", "prov:informed"],
+ "additionalProperties": { "$ref": "#/definitions/attributeValues" }
+ },
+ "start":{
+ "type": "object",
+ "title": "start/end",
+ "properties": {
+ "prov:activity": {"type": "string", "format": "uri"},
+ "prov:time": {"type": "string", "format": "date-time"},
+ "prov:trigger": {"type": "string", "format": "uri"}
+ },
+ "required": ["prov:activity"],
+ "additionalProperties": { "$ref": "#/definitions/attributeValues" }
+ },
+ "end": {"$ref":"#/definitions/start"},
+ "invalidation":{
+ "type": "object",
+ "title": "invalidation",
+ "properties": {
+ "prov:entity": { "type": "string", "format": "uri" },
+ "prov:time": { "type": "string", "format": "date-time" },
+ "prov:activity": { "type": "string", "format": "uri" }
+ },
+ "required": ["prov:entity"],
+ "additionalProperties": { "$ref": "#/definitions/attributeValues" }
+ },
+ "derivation":{
+ "type": "object",
+ "title": "derivation",
+ "properties": {
+ "prov:generatedEntity": { "type": "string", "format": "uri" },
+ "prov:usedEntity": { "type": "string", "format": "uri" },
+ "prov:activity": { "type": "string", "format": "uri" },
+ "prov:generation": { "type": "string", "format": "uri" },
+ "prov:usage": { "type": "string", "format": "uri" }
+ },
+ "required": ["prov:generatedEntity", "prov:usedEntity"],
+ "additionalProperties": { "$ref": "#/definitions/attributeValues" }
+ },
+ "attribution":{
+ "type": "object",
+ "title": "attribution",
+ "properties": {
+ "prov:entity": { "type": "string", "format": "uri" },
+ "prov:agent": { "type": "string", "format": "uri" }
+ },
+ "required": ["prov:entity", "prov:agent"],
+ "additionalProperties": { "$ref": "#/definitions/attributeValues" }
+ },
+ "association": {
+ "type": "object",
+ "title": "association",
+ "properties": {
+ "prov:activity": { "type": "string", "format": "uri" },
+ "prov:agent": { "type": "string", "format": "uri" },
+ "prov:plan": { "type": "string", "format": "uri" }
+ },
+ "required": ["prov:activity"],
+ "additionalProperties": { "$ref": "#/definitions/attributeValues" }
+ },
+ "delegation": {
+ "type": "object",
+ "title": "delegation",
+ "properties": {
+ "prov:delegate": { "type": "string", "format": "uri" },
+ "prov:responsible": { "type": "string", "format": "uri" },
+ "prov:activity": { "type": "string", "format": "uri" }
+ },
+ "required": ["prov:delegate", "prov:responsible"],
+ "additionalProperties": { "$ref": "#/definitions/attributeValues" }
+ },
+ "influence": {
+ "type": "object",
+ "title": "",
+ "properties": {
+ "prov:influencer": { "type": "string", "format": "uri" },
+ "prov:influencee": { "type": "string", "format": "uri" }
+ },
+ "required": ["prov:influencer", "prov:influencee"],
+ "additionalProperties": { "$ref": "#/definitions/attributeValues" }
+ },
+ "specialization": {
+ "type": "object",
+ "title": "specialization",
+ "properties": {
+ "prov:generalEntity": { "type": "string", "format": "uri" },
+ "prov:specificEntity": { "type": "string", "format": "uri" }
+ },
+ "required": ["prov:generalEntity", "prov:specificEntity"],
+ "additionalProperties": { "$ref": "#/definitions/attributeValues" }
+ },
+ "alternate": {
+ "type": "object",
+ "title": "alternate",
+ "properties": {
+ "prov:alternate1": { "type": "string", "format": "uri" },
+ "prov:alternate2": { "type": "string", "format": "uri" }
+ },
+ "required": ["prov:alternate1", "prov:alternate2"],
+ "additionalProperties": { "$ref": "#/definitions/attributeValues" }
+ },
+ "membership": {
+ "type": "object",
+ "title": "membership",
+ "properties": {
+ "prov:collection": { "type": "string", "format": "uri" },
+ "prov:entity": { "type": "string", "format": "uri" }
+ },
+ "required": ["prov:collection", "prov:entity"],
+ "additionalProperties": { "$ref": "#/definitions/attributeValues" }
+ },
+ "bundle": {
+ "type": "object",
+ "title": "bundle",
+ "properties":{
+ "prefix": {
+ "type": "object",
+ "patternProperties": {
+ "^[a-zA-Z0-9_\\-]+$": { "type" : "string", "format": "uri" }
+ }
+ },
+ "entity": {
+ "type": "object",
+ "additionalProperties": { "$ref":"#/definitions/entity" }
+ },
+ "activity": {
+ "type": "object",
+ "additionalProperties": { "$ref":"#/definitions/activity" }
+ },
+ "agent": {
+ "type": "object",
+ "additionalProperties": { "$ref":"#/definitions/agent" }
+ },
+ "wasGeneratedBy": {
+ "type": "object",
+ "additionalProperties": { "$ref":"#/definitions/generation" }
+ },
+ "used": {
+ "type": "object",
+ "additionalProperties": { "$ref":"#/definitions/usage" }
+ },
+ "wasInformedBy": {
+ "type": "object",
+ "additionalProperties": { "$ref":"#/definitions/communication" }
+ },
+ "wasStartedBy": {
+ "type": "object",
+ "additionalProperties": { "$ref":"#/definitions/start" }
+ },
+ "wasEndedby": {
+ "type": "object",
+ "additionalProperties": { "$ref":"#/definitions/end" }
+ },
+ "wasInvalidatedBy": {
+ "type": "object",
+ "additionalProperties": { "$ref":"#/definitions/invalidation" }
+ },
+ "wasDerivedFrom": {
+ "type": "object",
+ "additionalProperties": { "$ref":"#/definitions/derivation" }
+ },
+ "wasAttributedTo": {
+ "type": "object",
+ "additionalProperties": { "$ref":"#/definitions/attribution" }
+ },
+ "wasAssociatedWith": {
+ "type": "object",
+ "additionalProperties": { "$ref":"#/definitions/association" }
+ },
+ "actedOnBehalfOf": {
+ "type": "object",
+ "additionalProperties": { "$ref":"#/definitions/delegation" }
+ },
+ "wasInfluencedBy": {
+ "type": "object",
+ "additionalProperties": { "$ref":"#/definitions/influence" }
+ },
+ "specializationOf": {
+ "type": "object",
+ "additionalProperties": { "$ref":"#/definitions/specialization" }
+ },
+ "alternateOf": {
+ "type": "object",
+ "additionalProperties": { "$ref":"#/definitions/alternate" }
+ },
+ "hadMember": {
+ "type": "object",
+ "additionalProperties": { "$ref":"#/definitions/membership" }
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/provenance/migrations/__init__.py b/provenance/migrations/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/provenance/models.py b/provenance/models.py
new file mode 100644
index 0000000000000000000000000000000000000000..0716eac6023c74260a0aa2dbf288f9a98bb37722
--- /dev/null
+++ b/provenance/models.py
@@ -0,0 +1,264 @@
+"""
+This module provides models required for the creation, storage and manipulation of PROV documents.
+
+These PROV documents describe actions made by users and applications, allowing usage patterns to be tracked.
+
+For details on PROV see https://www.w3.org/TR/2013/NOTE-prov-overview-20130430/
+"""
+
+import enum
+import json
+import typing
+import uuid
+
+from django import apps
+from django.contrib.contenttypes.models import ContentType
+from django.db.models import QuerySet, signals
+from django.dispatch import receiver
+from django.utils import timezone
+from django.utils.text import slugify
+
+import mongoengine
+from mongoengine.queryset.visitor import Q
+import prov.model
+
+from applications.models import Application
+from core.models import BaseAppDataModel
+from datasources.models import DataSource
+
+MAX_LENGTH_NAME_FIELD = 100
+
+
+class PedasiDummyApplication:
+ """
+ Dummy application model to fall back to when an action was performed via the PEDASI web interface.
+ """
+ name = 'PEDASI'
+ pk = 'pedasi' # We convert pk to string anyway - so this is fine
+
+ @staticmethod
+ def get_absolute_url():
+ """
+ Return the URL at which PEDASI is hosted.
+ """
+ # TODO don't hardcode URL
+ return 'http://www.pedasi-iot.org/'
+
+
+@enum.unique
+class ProvActivity(enum.Enum):
+ """
+ Enum representing the types of activity to be tracked by PROV.
+ """
+ UPDATE = 'piot:update'
+ ACCESS = 'piot:access'
+
+
+class ProvEntry(mongoengine.DynamicDocument):
+ """
+ Stored PROV record for a single action.
+
+ e.g. Update a model's metadata, use a model.
+
+ These will be referred to by a :class:`ProvWrapper` document.
+ """
+ @classmethod
+ def create_prov(cls,
+ instance: BaseAppDataModel,
+ user_uri: str,
+ application: typing.Optional[Application] = None,
+ activity_type: typing.Optional[ProvActivity] = ProvActivity.UPDATE) -> 'ProvEntry':
+ """
+ Build a PROV document representing a particular activity within PEDASI.
+
+ :param instance: Application or DataSource which is the object of the activity
+ :param user_uri: URI of user who performed the activity
+ :param application: Application which the user used to perform the activity
+ :param activity_type: Type of the activity - from :class:`ProvActivity`
+ :return: PROV document in PROV-JSON form
+ """
+ instance_type = ContentType.objects.get_for_model(instance)
+
+ document = prov.model.ProvDocument(namespaces={
+ 'piot': 'http://www.pedasi-iot.org/',
+ 'foaf': 'http://xmlns.com/foaf/0.1/',
+ 'xsd': 'http://www.w3.org/2001/XMLSchema#',
+ })
+
+ entity = document.entity(
+ # TODO unique identifier for instance
+ 'piot:e-' + slugify(instance_type.model) + str(instance.pk),
+ other_attributes={
+ prov.model.PROV_TYPE: 'piot:' + slugify(instance_type.model),
+ 'xsd:anyURI': instance.get_absolute_url(),
+ }
+ )
+
+ activity = document.activity(
+ 'piot:a-' + str(uuid.uuid4()),
+ timezone.now(),
+ None,
+ other_attributes={
+ prov.model.PROV_TYPE: activity_type.value
+ }
+ )
+
+ agent_user = document.agent(
+ # Generate a UUID so we can lookup records belonging to a user
+ # But not identify the user from a given record
+ # TODO how strongly do we want to prevent user identification?
+ # See https://github.com/Southampton-RSG/PEDASI-IoT/issues/10
+ 'piot:u-' + str(uuid.uuid5(uuid.NAMESPACE_URL, user_uri)),
+ other_attributes={
+ prov.model.PROV_TYPE: 'prov:Person',
+ }
+ )
+
+ if application is None:
+ application = PedasiDummyApplication
+
+ agent_application = document.agent(
+ 'piot:app-' + str(application.pk),
+ other_attributes={
+ prov.model.PROV_TYPE: 'prov:SoftwareAgent',
+ 'xsd:anyURI': application.get_absolute_url(),
+ }
+ )
+
+ document.actedOnBehalfOf(
+ agent_application, # User who performs the action, on behalf of...
+ agent_user, # User who is responsible
+ activity, # NB: The prov library documentation suggests these are the other way round
+ other_attributes={
+ prov.model.PROV_TYPE: 'piot:ApplicationAction',
+ }
+ )
+
+ # PROV library does not appear to be able to give a Python dictionary directly - have to go via string
+ return cls.deserialize(content=document.serialize())
+
+ @classmethod
+ def deserialize(cls, source=None, content: str = None, format: str = 'json', **kwargs):
+ """
+ Create an instance of :class:`ProvEntry` from another instance or a string document.
+
+ Used to create a :class:`ProvEntry` from a serialized PROV document.
+
+ Provide one of 'source' or 'content'.
+
+ :param source: Source from which to copy object
+ :param content: Text from which to create object
+ :param format: Format of text - e.g. JSON
+ :return: New instance of :class:`ProvEntry`
+ """
+ if source is None and content is not None and format == 'json':
+ json_string = content
+
+ else:
+ document = prov.model.ProvDocument.deserialize(source, content, format, **kwargs)
+ json_string = document.serialize(format='json')
+
+ # PROV library does not appear to be able to give a Python dictionary directly - have to go via string
+ prov_json = json.loads(json_string)
+ return cls(**prov_json)
+
+
+class ProvWrapper(mongoengine.Document):
+ """
+ Wrapper around a single PROV record (:class:`ProvEntry`) which allows it to be easily linked to an instance
+ of a Django model.
+
+ This is managed using MongoEngine rather than as a Django model.
+ """
+ #: App from which the model comes
+ app_label = mongoengine.fields.StringField(max_length=MAX_LENGTH_NAME_FIELD,
+ required=True, null=False)
+
+ #: Name of the model
+ model_name = mongoengine.fields.StringField(max_length=MAX_LENGTH_NAME_FIELD,
+ required=True, null=False)
+
+ # TODO should this be editable=False? Can a model PK change?
+ # TODO can we make this behave like a primary key?
+ #: Primary key of the model instance
+ related_pk = mongoengine.fields.IntField(required=True, null=False)
+
+ #: The actual PROV entry
+ entry = mongoengine.fields.ReferenceField(
+ document_type=ProvEntry
+ )
+
+ @property
+ def instance(self):
+ """
+ Return the Django model instance to which this :class:`ProvWrapper` refers.
+ """
+ model = apps.apps.get_model(self.app_label, self.model_name)
+ return model.objects.get(pk=self.related_pk)
+
+ @classmethod
+ def filter_model_instance(cls, instance: BaseAppDataModel) -> QuerySet:
+ """
+ Get all :class:`ProvEntry` documents related to a particular Django model instance.
+
+ :param instance: Model instance for which to get all :class:`ProvEntry`s
+ :return: List of :class:`ProvEntry`s
+ """
+ instance_type = ContentType.objects.get_for_model(instance)
+
+ return ProvWrapper.objects(
+ Q(app_label=instance_type.app_label) &
+ Q(model_name=instance_type.model) &
+ Q(related_pk=instance.pk)
+ ).values_list('entry')
+
+ @classmethod
+ def create_prov(cls,
+ instance: BaseAppDataModel,
+ user_uri: str,
+ application: typing.Optional[Application] = None,
+ activity_type: typing.Optional[ProvActivity] = ProvActivity.UPDATE) -> ProvEntry:
+ """
+ Create a PROV record for a single action.
+
+ e.g. Update a model's metadata, use a model.
+
+ These will create and return a :class:`ProvEntry` document.
+ """
+ prov_entry = ProvEntry.create_prov(instance, user_uri,
+ application=application,
+ activity_type=activity_type)
+ prov_entry.save()
+
+ instance_type = ContentType.objects.get_for_model(instance)
+
+ wrapper = cls(
+ app_label=instance_type.app_label,
+ model_name=instance_type.model,
+ related_pk=instance.pk,
+ entry=prov_entry
+ )
+ wrapper.save()
+
+ return prov_entry
+
+ def delete(self, signal_kwargs=None, **write_concern):
+ """
+ Delete this document and the :class:`ProvEntry` to which it refers.
+ """
+ self.entry.delete(signal_kwargs, **write_concern)
+ super().delete(signal_kwargs, **write_concern)
+
+
+@receiver(signals.post_save, sender=Application)
+@receiver(signals.post_save, sender=DataSource)
+def save_prov(sender, instance, **kwargs):
+ """
+ Signal receiver to create a :class:`ProvEntry` when a PROV tracked model is saved.
+ """
+ ProvWrapper.create_prov(
+ instance,
+ # TODO what if an admin edits a model?
+ instance.owner.get_uri(),
+ activity_type=ProvActivity.UPDATE
+ )
diff --git a/provenance/tests.py b/provenance/tests.py
new file mode 100644
index 0000000000000000000000000000000000000000..21029c68d10595a4d0e53e999cfdc41aa6629a75
--- /dev/null
+++ b/provenance/tests.py
@@ -0,0 +1,166 @@
+"""
+Tests for PROV tracking functionality and the models required to support it.
+"""
+
+import json
+import pathlib
+import unittest
+
+from django.conf import settings
+from django.contrib.auth import get_user_model
+from django.contrib.contenttypes.models import ContentType
+from django.test import TestCase
+
+import jsonschema
+import mongoengine
+from mongoengine.queryset.visitor import Q
+
+from datasources.models import DataSource
+from provenance import models
+
+
+# Create connection to test DB
+TEST_DB = mongoengine.connect('test_prov')
+TEST_DB.drop_database('test_prov')
+
+
+class ProvEntryTest(TestCase):
+ """
+ Test the :class:`ProvEntry` model in isolation.
+ """
+
+ @classmethod
+ def setUpTestData(cls):
+ cls.user_model = get_user_model()
+ cls.user = cls.user_model.objects.create_user('Test Prov User')
+
+ def setUp(self):
+ self.datasource = DataSource.objects.create(
+ name='Test Data Source',
+ url='http://www.example.com',
+ owner=self.user,
+ plugin_name='TEST'
+ )
+
+ def tearDown(self):
+ # Have to delete instance manually since we're not using Django's database manager
+ datasource_type = ContentType.objects.get_for_model(DataSource)
+
+ models.ProvWrapper.objects(
+ Q(app_label=datasource_type.app_label) &
+ Q(model_name=datasource_type.model) &
+ Q(related_pk=self.datasource.pk)
+ ).delete()
+
+ def test_prov_created(self):
+ """
+ Test the creation of a :class:`ProvEntry` in isolation.
+ """
+ entry = models.ProvEntry.create_prov(self.datasource,
+ self.user.get_uri())
+
+ self.assertIsNotNone(entry)
+
+ # TODO test content of PROV document - not just compliance to spec
+ def test_prov_schema(self):
+ """
+ Validate :class:`ProvEntry` against PROV-JSON schema.
+ """
+ entry = models.ProvEntry.create_prov(self.datasource,
+ self.user.get_uri())
+ entry_json = json.loads(entry.to_json())
+
+ schema_path = pathlib.Path(settings.BASE_DIR).joinpath('provenance', 'data', 'prov-json.schema.json')
+ with open(schema_path) as schema_file:
+ schema = json.load(schema_file)
+
+ validator = jsonschema.Draft4Validator(schema)
+
+ try:
+ validator.validate(entry_json)
+ except jsonschema.exceptions.ValidationError:
+ for error in validator.iter_errors(entry_json):
+ print(error.message)
+ raise
+
+
+class ProvWrapperTest(TestCase):
+ """
+ Test the wrapper that allows us to look up :class:`ProvEntry`s for a given object.
+ """
+
+ @classmethod
+ def setUpTestData(cls):
+ cls.user_model = get_user_model()
+ cls.user = cls.user_model.objects.create_user('Test Prov User')
+
+ def setUp(self):
+ self.datasource = DataSource.objects.create(
+ name='Test Data Source',
+ url='http://www.example.com',
+ owner=self.user,
+ plugin_name='TEST'
+ )
+
+ def tearDown(self):
+ # Have to delete instance manually since we're not using Django's database manager
+ datasource_type = ContentType.objects.get_for_model(DataSource)
+
+ models.ProvWrapper.objects(
+ Q(app_label=datasource_type.app_label) &
+ Q(model_name=datasource_type.model) &
+ Q(related_pk=self.datasource.pk)
+ ).delete()
+
+ @staticmethod
+ def _count_prov(datasource: DataSource) -> int:
+ prov_entries = models.ProvWrapper.filter_model_instance(datasource)
+ return prov_entries.count()
+
+ def test_prov_datasource_create(self):
+ """
+ Test that a :class:`ProvEntry` is created when a model is created.
+ """
+ # PROV record should be created when model is created
+ self.assertEqual(self._count_prov(self.datasource), 1)
+
+ def test_prov_datasource_update(self):
+ """
+ Test that a new :class:`ProvEntry` is created when a model is updated.
+ """
+ n_provs = self._count_prov(self.datasource)
+
+ self.datasource.plugin_name = 'CHANGED'
+ self.datasource.save()
+
+ # Another PROV record should be created when model is changed and saved
+ self.assertEqual(self._count_prov(self.datasource), n_provs + 1)
+
+ @unittest.expectedFailure
+ def test_prov_datasource_null_update(self):
+ """
+ Test that no new :class:`ProvEntry` is created when a model is saved without changes.
+ """
+ n_provs = self._count_prov(self.datasource)
+
+ self.datasource.save()
+
+ # No PROV record should be created when saving a model that has not changed
+ self.assertEqual(self._count_prov(self.datasource), n_provs)
+
+ def test_prov_records_distinct(self):
+ """
+ Test that :class:`ProvEntry`s are not reused.
+ """
+ prov_entries = models.ProvWrapper.filter_model_instance(self.datasource)
+
+ new_datasource = DataSource.objects.create(
+ name='Another Test Data Source',
+ url='http://www.example.com',
+ owner=self.user,
+ plugin_name='TEST'
+ )
+ new_prov_entries = models.ProvWrapper.filter_model_instance(new_datasource)
+
+ intersection = set(prov_entries).intersection(new_prov_entries)
+ self.assertFalse(intersection)
diff --git a/requirements.txt b/requirements.txt
index 49e5f7f8fa1e5fcae6e7fae5502ad40d33c2eda1..5ec9ca35cae6d7137f67d446713fd0a74e470810 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -5,37 +5,54 @@ Babel==2.6.0
certifi==2018.8.13
cffi==1.11.5
chardet==3.0.4
-dataclasses==0.6
+decorator==4.3.0
+defusedxml==0.5.0
dj-database-url==0.5.0
Django==2.0.8
django-bootstrap4==0.0.6
-djongo==1.2.29
+django-cors-headers==2.4.0
+django-haystack==2.8.1
+djangorestframework==3.9.1
docutils==0.14
idna==2.7
imagesize==1.0.0
+isodate==0.6.0
isort==4.3.4
Jinja2==2.10
+jsonschema==2.6.0
lazy-object-proxy==1.3.1
+lxml==4.2.5
MarkupSafe==1.0
mccabe==0.6.1
mongoengine==0.15.3
mysqlclient==1.3.13
+networkx==2.2
+oauthlib==2.1.0
packaging==17.1
+prov==1.5.2
pycparser==2.18
Pygments==2.2.0
+PyJWT==1.6.4
pylint==2.1.1
pylint-django==2.0
pylint-plugin-utils==0.4
pymongo==3.7.1
pyparsing==2.2.0
+python-dateutil==2.7.3
python-decouple==3.1
+python3-openid==3.1.0
pytz==2018.5
+rdflib==4.2.2
requests==2.19.1
+requests-oauthlib==1.0.0
six==1.11.0
snowballstemmer==1.2.1
+social-auth-app-django==3.0.0
+social-auth-core==2.0.0
Sphinx==1.7.6
sphinxcontrib-websupport==1.1.0
-sqlparse==0.2.4
+SQLAlchemy==1.2.14
typed-ast==1.1.0
urllib3==1.23
+Whoosh==2.7.4
wrapt==1.10.11
diff --git a/scripts/goaccess.sh b/scripts/goaccess.sh
new file mode 100755
index 0000000000000000000000000000000000000000..bf0eb4494dc12f4bd3e9d7375aa0d273cdb3f90f
--- /dev/null
+++ b/scripts/goaccess.sh
@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+if [[ "$#" -ne 1 ]]; then
+ echo "Requires output file name"
+ exit 1
+fi
+
+if [[ -f /var/log/nginx/access.log.2.gz ]]; then
+ zcat /var/log/nginx/access.log.*.gz | goaccess -q --log-format=COMBINED -o $1 /var/log/nginx/access.log /var/log/nginx/access.log.1 -
+elif [[ -f /var/log/nginx/access.log.1 ]]; then
+ goaccess -q --log-format=COMBINED -o $1 /var/log/nginx/access.log /var/log/nginx/access.log.1 -
+else
+ goaccess -q --log-format=COMBINED -o $1 /var/log/nginx/access.log
+fi