Skip to content
Snippets Groups Projects
Commit 3c537aa0 authored by James Graham's avatar James Graham
Browse files

Add initial data source connector plugin API and implementation for

IoTUK plugin

See issue #5
parent 64759642
No related branches found
No related tags found
No related merge requests found
...@@ -27,4 +27,3 @@ ubuntu-bionic-18.04-cloudimg-console.log ...@@ -27,4 +27,3 @@ ubuntu-bionic-18.04-cloudimg-console.log
# Experimental # Experimental
/applications/connectors/ /applications/connectors/
/datasources/connectors/
import abc
import typing
class BaseDataConnector(abc.ABC):
"""
Base class of data connectors which provide access to data / metadata via an external API.
DataConnectors may be defined for sources which provide:
* A single dataset
* A data catalogue - a collection of datasets
TODO:
* Should this connector interface be able to handle data catalogues and datasets
or should we create a new connector for datasets within a catalogue?
* What other operations do we need?
"""
def __init__(self, location):
self.location = location
@property
@abc.abstractmethod
def name(self) -> str:
"""
Friendly name of the connector class.
"""
raise NotImplementedError
@abc.abstractmethod
def get_data(self,
dataset: typing.Optional[str] = None,
query_params: typing.Optional[typing.Mapping[str, str]] = None):
"""
Get data from this source using the appropriate API.
:param dataset: Optional dataset for which to get data
:param query_params: Optional query parameter filters
:return: Requested data
"""
raise NotImplementedError
@classmethod
def __enter__(cls, location):
return cls(location)
@classmethod
def __exit__(cls, exc_type, exc_val, exc_tb):
pass
@classmethod
def _recurse_subclasses(cls):
from . import iotuk
for subclass in cls.__subclasses__():
yield subclass
yield from subclass._recurse_subclasses()
@classmethod
def get_plugin(cls, name: str):
"""
Find the plugin matching the requested name.
:param name: Name of plugin to find
:return: Plugin class
"""
subclasses = list(cls._recurse_subclasses())
for subclass in subclasses:
if subclass.name == name:
return subclass
class DataConnectorHasMetaData:
@abc.abstractmethod
def get_metadata(self,
dataset: typing.Optional[str] = None,
query_params: typing.Optional[typing.Mapping[str, str]] = None):
"""
Get metadata from this source using the appropriate API.
:param dataset: Optional dataset for which to get metadata
:param query_params: Optional query parameter filters
:return: Requested metadata
"""
raise NotImplementedError
class InternalDataConnector(BaseDataConnector):
"""
Base class of data connectors which provide access to data / metadata stored internally.
"""
import typing
import requests
from .base import BaseDataConnector
class IoTUK(BaseDataConnector):
name = 'IoTUK'
def __init__(self, url='https://api.iotuk.org.uk/iotOrganisation'):
super().__init__(location=url)
def get_data(self,
dataset: typing.Optional[str] = None,
query_params: typing.Optional[typing.Mapping[str, str]] = None):
"""
Get data from a source using the IoTUK Nation API.
:param dataset: Optional dataset for which to get data
:param query_params: Optional query parameter filters
:return: Requested data
"""
r = requests.get(self.location, params=query_params)
return r.json()
# Generated by Django 2.0.8 on 2018-08-21 14:45
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('datasources', '0003_allow_blank_access_group'),
]
operations = [
migrations.AlterField(
model_name='datasource',
name='users_group',
field=models.ForeignKey(blank=True, editable=False, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='datasource', to='auth.Group'),
),
migrations.AlterField(
model_name='datasource',
name='users_group_requested',
field=models.ForeignKey(blank=True, editable=False, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='datasource_requested', to='auth.Group'),
),
]
# Generated by Django 2.0.8 on 2018-08-22 10:00
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('datasources', '0004_remove_groups_from_form'),
]
operations = [
migrations.AddField(
model_name='datasource',
name='plugin_name',
field=models.CharField(default='IoTUK', max_length=63),
preserve_default=False,
),
]
...@@ -57,6 +57,10 @@ class DataSource(models.Model): ...@@ -57,6 +57,10 @@ class DataSource(models.Model):
access_control = models.BooleanField(default=False, access_control = models.BooleanField(default=False,
blank=False, null=False) blank=False, null=False)
#: Name of plugin which allows interaction with this data source
plugin_name = models.CharField(max_length=MAX_LENGTH_NAME,
blank=False, null=False)
def has_view_permission(self, user: settings.AUTH_USER_MODEL) -> bool: def has_view_permission(self, user: settings.AUTH_USER_MODEL) -> bool:
""" """
Does a user have permission to use this data source? Does a user have permission to use this data source?
......
...@@ -23,6 +23,9 @@ ...@@ -23,6 +23,9 @@
<p>{{ datasource.description }}</p> <p>{{ datasource.description }}</p>
{% endif %} {% endif %}
<a href="{% url 'datasources:datasource.query' pk=datasource.id %}"
class="btn btn-success" role="link">Query</a>
<a href="{% url 'admin:datasources_datasource_change' datasource.id %}" <a href="{% url 'admin:datasources_datasource_change' datasource.id %}"
class="btn btn-success" role="button">Edit</a> class="btn btn-success" role="button">Edit</a>
<a href="{% url 'admin:datasources_datasource_delete' datasource.id %}" <a href="{% url 'admin:datasources_datasource_delete' datasource.id %}"
......
{% extends "base.html" %}
{% load bootstrap4 %}
{% block content %}
<nav aria-label="breadcrumb">
<ol class="breadcrumb">
<li class="breadcrumb-item" aria-current="page">
<a href="{% url 'datasources:datasource.list' %}">Data Sources</a>
</li>
<li class="breadcrumb-item active" aria-current="page">
{{ datasource.name }}
</li>
</ol>
</nav>
<h2>View Data Source - {{ datasource.name }}</h2>
<p>
Owner: <a href="#" role="link">{{ datasource.owner }}</a>
</p>
{% if datasource.description %}
<p>{{ datasource.description }}</p>
{% endif %}
<hr>
<table class="table">
<thead class="thead">
<tr>
<th>Name</th>
</tr>
</thead>
<tbody>
{% for data_row in results.data.values %}
<tr>
<td>{{ data_row.organisation_name }}</td>
</tr>
{% endfor %}
</tbody>
</table>
{% endblock %}
\ No newline at end of file
...@@ -16,4 +16,8 @@ urlpatterns = [ ...@@ -16,4 +16,8 @@ urlpatterns = [
path('<int:pk>/manage-access', path('<int:pk>/manage-access',
views.DataSourceManageAccessView.as_view(), views.DataSourceManageAccessView.as_view(),
name='datasource.manage-access'), name='datasource.manage-access'),
path('<int:pk>/query',
views.DataSourceQueryView.as_view(),
name='datasource.query'),
] ]
...@@ -2,6 +2,7 @@ from django.views.generic.detail import DetailView ...@@ -2,6 +2,7 @@ from django.views.generic.detail import DetailView
from django.views.generic.list import ListView from django.views.generic.list import ListView
from profiles.permissions import OwnerPermissionRequiredMixin from profiles.permissions import OwnerPermissionRequiredMixin
from .connectors.base import BaseDataConnector
from . import models from . import models
...@@ -28,3 +29,17 @@ class DataSourceManageAccessView(OwnerPermissionRequiredMixin, DetailView): ...@@ -28,3 +29,17 @@ class DataSourceManageAccessView(OwnerPermissionRequiredMixin, DetailView):
context_object_name = 'datasource' context_object_name = 'datasource'
permission_required = 'datasources.change_datasource' permission_required = 'datasources.change_datasource'
class DataSourceQueryView(DetailView):
model = models.DataSource
template_name = 'datasources/datasource/query.html'
context_object_name = 'datasource'
def get_context_data(self, **kwargs):
context = super().get_context_data(**kwargs)
plugin = BaseDataConnector.get_plugin(self.object.plugin_name)
context['results'] = plugin().get_data(query_params={'year': 2018})
return context
-r ../requirements.txt
alabaster==0.7.11
Babel==2.6.0
certifi==2018.8.13
chardet==3.0.4
docutils==0.14
idna==2.7
imagesize==1.0.0
Jinja2==2.10
MarkupSafe==1.0
packaging==17.1
Pygments==2.2.0
pyparsing==2.2.0
requests==2.19.1
snowballstemmer==1.2.1
Sphinx==1.7.6
sphinxcontrib-websupport==1.1.0
urllib3==1.23
alabaster==0.7.11
argon2-cffi==18.1.0 argon2-cffi==18.1.0
astroid==2.0.4
Babel==2.6.0
certifi==2018.8.13
cffi==1.11.5 cffi==1.11.5
chardet==3.0.4
dj-database-url==0.5.0 dj-database-url==0.5.0
Django==2.0.8 Django==2.0.8
django-bootstrap4==0.0.6 django-bootstrap4==0.0.6
docutils==0.14
idna==2.7
imagesize==1.0.0
isort==4.3.4
Jinja2==2.10
lazy-object-proxy==1.3.1
MarkupSafe==1.0
mccabe==0.6.1
mysqlclient==1.3.13 mysqlclient==1.3.13
packaging==17.1
pycparser==2.18 pycparser==2.18
Pygments==2.2.0
pylint==2.1.1
pylint-django==2.0
pylint-plugin-utils==0.4
pyparsing==2.2.0
python-decouple==3.1 python-decouple==3.1
pytz==2018.5 pytz==2018.5
requests==2.19.1
six==1.11.0 six==1.11.0
snowballstemmer==1.2.1
Sphinx==1.7.6
sphinxcontrib-websupport==1.1.0
typed-ast==1.1.0
urllib3==1.23
wrapt==1.10.11
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment