diff --git a/api/permissions.py b/api/permissions.py
index e296bf823cb0b21b2f05572b3cc9d04939f9df55..2153980d9d6bb4202438b406d6bc07aa97dbfbfb 100644
--- a/api/permissions.py
+++ b/api/permissions.py
@@ -76,3 +76,14 @@ class DataPushPermission(permissions.BasePermission):
except models.UserPermissionLink.DoesNotExist:
# Permission must have been granted explicitly
return False
+
+
+class IsAdminOrReadOnly(permissions.BasePermission):
+ """
+ Grant admins write access - all others get read-only.
+ """
+ def has_permission(self, request, view):
+ return bool(
+ request.method in permissions.SAFE_METHODS or
+ request.user.is_superuser
+ )
diff --git a/api/urls.py b/api/urls.py
index 9f39481a759622de22ae9645d632b145cae4ba29..7adf2a1dc491d88f2a58f71435974eb1c6ea4054 100644
--- a/api/urls.py
+++ b/api/urls.py
@@ -1,16 +1,43 @@
from django.urls import include, path
-from rest_framework import routers
+from rest_framework_nested import routers
-from .views import datasources as datasource_views
+from .views.datasources import (
+ DataSourceApiViewset,
+ MetadataItemApiViewset
+)
+from .views.quality import (
+ QualityCriterionApiViewset,
+ QualityLevelApiViewset,
+ QualityRulesetApiViewset
+)
app_name = 'api'
# Register ViewSets
router = routers.DefaultRouter()
-router.register('datasources', datasource_views.DataSourceApiViewset)
+router.register('datasources', DataSourceApiViewset, base_name='datasource')
+router.register('rulesets', QualityRulesetApiViewset, base_name='rulesets')
+
+datasource_router = routers.NestedSimpleRouter(router, 'datasources', lookup='datasource')
+datasource_router.register('metadata_items', MetadataItemApiViewset, base_name='metadata-item')
+
+ruleset_router = routers.NestedSimpleRouter(router, 'rulesets', lookup='ruleset')
+ruleset_router.register('levels', QualityLevelApiViewset, base_name='levels')
+
+level_router = routers.NestedSimpleRouter(ruleset_router, 'levels', lookup='level')
+level_router.register('criteria', QualityCriterionApiViewset, base_name='criteria')
urlpatterns = [
path('',
include(router.urls)),
+
+ path('',
+ include(datasource_router.urls)),
+
+ path('',
+ include(ruleset_router.urls)),
+
+ path('',
+ include(level_router.urls)),
]
diff --git a/api/views/__init__.py b/api/views/__init__.py
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..4231549fb1eab823cefba2c086e854ce663eedf7 100644
--- a/api/views/__init__.py
+++ b/api/views/__init__.py
@@ -0,0 +1,7 @@
+from .datasources import DataSourceApiViewset
+
+from .quality import (
+ QualityCriterionApiViewset,
+ QualityLevelApiViewset,
+ QualityRulesetApiViewset
+)
\ No newline at end of file
diff --git a/api/views/datasources.py b/api/views/datasources.py
index 793acbb21ba9fa9a3a3f41ec939a12b416f96c6e..a5d2f9c45c9705cfa57c390889600e4934c802dc 100644
--- a/api/views/datasources.py
+++ b/api/views/datasources.py
@@ -6,8 +6,10 @@ import csv
import json
import typing
+from django.contrib.auth import get_user_model
from django.db.models import ObjectDoesNotExist
from django.http import HttpResponse, JsonResponse
+from django.shortcuts import get_object_or_404
from rest_framework import decorators, request, response, viewsets
from requests.exceptions import HTTPError
@@ -18,6 +20,18 @@ from datasources.connectors.base import DatasetNotFoundError
from provenance import models as prov_models
+class MetadataItemApiViewset(viewsets.ModelViewSet):
+ serializer_class = serializers.MetadataItemSerializer
+ permission_classes = [permissions.IsAdminOrReadOnly]
+
+ def get_queryset(self):
+ return models.MetadataItem.objects.filter(datasource=self.kwargs['datasource_pk'])
+
+ def perform_create(self, serializer):
+ datasource = get_object_or_404(models.DataSource, pk=self.kwargs['datasource_pk'])
+ serializer.save(datasource=datasource)
+
+
class DataSourceApiViewset(viewsets.ReadOnlyModelViewSet):
"""
Provides views for:
@@ -28,6 +42,9 @@ class DataSourceApiViewset(viewsets.ReadOnlyModelViewSet):
/api/datasources/<int>/
Retrieve a single :class:`datasources.models.DataSource`.
+ /api/datasources/<int>/quality/
+ Get the quality level of a :class:`datasources.models.DataSource` using the current ruleset.
+
/api/datasources/<int>/prov/
Retrieve PROV records related to a :class:`datasources.models.DataSource`.
@@ -164,7 +181,21 @@ class DataSourceApiViewset(viewsets.ReadOnlyModelViewSet):
serializer = self.get_serializer(queryset, many=True)
return response.Response(serializer.data)
- @decorators.action(detail=True, permission_classes=[permissions.ProvPermission])
+ @decorators.action(detail=True, permission_classes=[permissions.ViewPermission])
+ def quality(self, request, pk=None):
+ """
+ View for /api/datasources/<int>/quality/
+
+ Get the quality level of a data source using the current ruleset.
+ """
+ ruleset = get_user_model().get_quality_ruleset()
+ instance = self.get_object()
+
+ return response.Response({
+ 'quality': ruleset(instance),
+ }, status=200)
+
+ @decorators.action(detail=True, permission_classes=[permissions.ProvPermission], name='datasource-quality')
def prov(self, request, pk=None):
"""
View for /api/datasources/<int>/prov/
diff --git a/api/views/quality.py b/api/views/quality.py
new file mode 100644
index 0000000000000000000000000000000000000000..f55fe5b4ad66271ed5de47109f809dce1b1dff94
--- /dev/null
+++ b/api/views/quality.py
@@ -0,0 +1,52 @@
+"""
+This module contains the API for data quality rulesets.
+"""
+
+from django.shortcuts import get_object_or_404
+
+from rest_framework import viewsets
+
+
+from datasources import models, serializers
+from .. import permissions
+
+
+class QualityRulesetApiViewset(viewsets.ModelViewSet):
+ serializer_class = serializers.QualityRulesetSerializer
+ permission_classes = [permissions.IsAdminOrReadOnly]
+
+ def get_queryset(self):
+ return models.QualityRuleset.objects.all()
+
+
+class QualityLevelApiViewset(viewsets.ModelViewSet):
+ serializer_class = serializers.QualityLevelSerializer
+ permission_classes = [permissions.IsAdminOrReadOnly]
+
+ def get_queryset(self):
+ return models.QualityLevel.objects.filter(ruleset=self.kwargs['ruleset_pk'])
+
+ def get_object(self):
+ return self.get_queryset().get(level=self.kwargs['pk'])
+
+ def perform_create(self, serializer):
+ ruleset = get_object_or_404(models.QualityRuleset, pk=self.kwargs['ruleset_pk'])
+ serializer.save(ruleset=ruleset)
+
+
+class QualityCriterionApiViewset(viewsets.ModelViewSet):
+ queryset = models.QualityCriterion.objects.all()
+ serializer_class = serializers.QualityCriterionSerializer
+ permission_classes = [permissions.IsAdminOrReadOnly]
+
+ def get_queryset(self):
+ return models.QualityCriterion.objects.filter(
+ quality_level__ruleset=self.kwargs['ruleset_pk'],
+ quality_level__level=self.kwargs['level_pk']
+ )
+
+ def perform_create(self, serializer):
+ level = get_object_or_404(models.QualityLevel,
+ ruleset=self.kwargs['ruleset_pk'],
+ level=self.kwargs['level_pk'])
+ serializer.save(quality_level=level)
diff --git a/datasources/admin.py b/datasources/admin.py
index c2419c0629b2ba98784c3ecfee0fcd8309da7548..b65be07898221fa3ab2d85e98f6fd0452d7b5d0a 100644
--- a/datasources/admin.py
+++ b/datasources/admin.py
@@ -46,3 +46,18 @@ class DataSourceAdmin(admin.ModelAdmin):
form.instance.owner = request.user
super().save_model(request, obj, form, change)
+
+
+@admin.register(models.QualityRuleset)
+class QualityRulesetAdmin(admin.ModelAdmin):
+ pass
+
+
+@admin.register(models.QualityLevel)
+class QualityLevelAdmin(admin.ModelAdmin):
+ pass
+
+
+@admin.register(models.QualityCriterion)
+class QualityCriterionAdmin(admin.ModelAdmin):
+ pass
diff --git a/datasources/forms.py b/datasources/forms.py
index e3678de99976f8f6f5c6ad0184d4db9b95e8058d..774ae7895e679d2293205a9ba23efc2361ce8bb0 100644
--- a/datasources/forms.py
+++ b/datasources/forms.py
@@ -97,12 +97,6 @@ class PermissionGrantForm(forms.ModelForm):
fields = ['granted', 'push_granted']
-class MetadataFieldForm(forms.ModelForm):
- class Meta:
- model = models.MetadataItem
- fields = ['field', 'value']
-
-
class LicenceForm(forms.ModelForm):
class Meta:
model = models.Licence
diff --git a/datasources/migrations/0032_add_quality_rules.py b/datasources/migrations/0032_add_quality_rules.py
new file mode 100644
index 0000000000000000000000000000000000000000..65198fd8dd0fe6d21198cacc8bae5c07504338fc
--- /dev/null
+++ b/datasources/migrations/0032_add_quality_rules.py
@@ -0,0 +1,61 @@
+# Generated by Django 2.0.13 on 2019-04-10 12:47
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('datasources', '0031_default_connector_name'),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name='QualityCriterion',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('weight', models.FloatField(default=1)),
+ ('metadata_field', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='+', to='datasources.MetadataField')),
+ ],
+ ),
+ migrations.CreateModel(
+ name='QualityLevel',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('level', models.PositiveSmallIntegerField()),
+ ('threshold', models.FloatField(blank=True, null=True)),
+ ],
+ ),
+ migrations.CreateModel(
+ name='QualityRuleset',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('name', models.CharField(max_length=63)),
+ ('short_name', models.CharField(blank=True, max_length=63, unique=True)),
+ ('version', models.CharField(max_length=63)),
+ ],
+ ),
+ migrations.AlterUniqueTogether(
+ name='qualityruleset',
+ unique_together={('name', 'version')},
+ ),
+ migrations.AddField(
+ model_name='qualitylevel',
+ name='ruleset',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='levels', to='datasources.QualityRuleset'),
+ ),
+ migrations.AddField(
+ model_name='qualitycriterion',
+ name='quality_level',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='criteria', to='datasources.QualityLevel'),
+ ),
+ migrations.AlterUniqueTogether(
+ name='qualitylevel',
+ unique_together={('ruleset', 'level')},
+ ),
+ migrations.AlterUniqueTogether(
+ name='qualitycriterion',
+ unique_together={('quality_level', 'metadata_field')},
+ ),
+ ]
diff --git a/datasources/migrations/0033_quality_levels_ordered.py b/datasources/migrations/0033_quality_levels_ordered.py
new file mode 100644
index 0000000000000000000000000000000000000000..25fe97975b7d833c0589d51d9d1974e61a11a0f0
--- /dev/null
+++ b/datasources/migrations/0033_quality_levels_ordered.py
@@ -0,0 +1,21 @@
+# Generated by Django 2.0.13 on 2019-04-10 14:07
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('datasources', '0032_add_quality_rules'),
+ ]
+
+ operations = [
+ migrations.AlterModelOptions(
+ name='qualitycriterion',
+ options={'verbose_name_plural': 'quality criteria'},
+ ),
+ migrations.AlterModelOptions(
+ name='qualitylevel',
+ options={'ordering': ['level']},
+ ),
+ ]
diff --git a/datasources/models/__init__.py b/datasources/models/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..2e459ef74f8149af14bcd1ee0d76219b19a48d59
--- /dev/null
+++ b/datasources/models/__init__.py
@@ -0,0 +1,23 @@
+"""
+This package contains the :class:`DataSource` model representing an internal or
+external data source and the models necessary to support it.
+"""
+
+from .datasource import (
+ Licence,
+ UserPermissionLevels,
+ UserPermissionLink,
+ DataSource
+)
+
+from .metadata import (
+ MetadataField,
+ MetadataItem
+)
+
+
+from .quality import (
+ QualityRuleset,
+ QualityLevel,
+ QualityCriterion
+)
diff --git a/datasources/models.py b/datasources/models/datasource.py
similarity index 83%
rename from datasources/models.py
rename to datasources/models/datasource.py
index e7c1b70fcfbba5e81ebb70ab6a16b56fd0097c4d..aad5cdeafe3d47f007bfdf7b14d6782e5fd8d7a4 100644
--- a/datasources/models.py
+++ b/datasources/models/datasource.py
@@ -8,7 +8,6 @@ import json
import typing
from django.conf import settings
-from django.core import validators
from django.db import models
from django.urls import reverse
@@ -60,91 +59,6 @@ class Licence(models.Model):
return reverse('datasources:licence.detail', kwargs={'pk': self.pk})
-class MetadataField(models.Model):
- """
- A metadata field that can be dynamically added to a data source.
-
- Operational MetadataFields are those which have some associated code within PEDASI.
- They should be present within any deployment of PEDASI.
-
- Current operational metadata fields are (by short_name):
- - data_query_param
- - indexed_field
- """
- #: Name of the field
- name = models.CharField(max_length=MAX_LENGTH_NAME,
- unique=True,
- blank=False, null=False)
-
- #: Short text identifier for the field
- short_name = models.CharField(
- max_length=MAX_LENGTH_NAME,
- validators=[
- validators.RegexValidator(
- r'^[a-zA-Z][a-zA-Z0-9_]*\Z',
- 'Short name must begin with a letter and consist only of letters, numbers and underscores.',
- 'invalid'
- )
- ],
- unique=True,
- blank=False, null=False
- )
-
- #: Does the field have an operational effect within PEDASI?
- operational = models.BooleanField(default=False,
- blank=False, null=False)
-
- def __str__(self):
- return self.name
-
- @classmethod
- def load_inline_fixtures(cls):
- """
- Create any instances required for the functioning of PEDASI.
-
- This is called from within the AppConfig.
- """
- fixtures = (
- ('data_query_param', 'data_query_param', True),
- ('indexed_field', 'indexed_field', True),
- )
-
- for name, short_name, operational in fixtures:
- obj, created = cls.objects.get_or_create(
- name=name,
- short_name=short_name
- )
- obj.operational = operational
- obj.save()
-
-
-class MetadataItem(models.Model):
- """
- The value of the metadata field on a given data source.
- """
- #: The value of this metadata field
- value = models.CharField(max_length=MAX_LENGTH_REASON,
- blank=True, null=False)
-
- #: To which field does this relate?
- field = models.ForeignKey(MetadataField,
- related_name='values',
- on_delete=models.PROTECT,
- blank=False, null=False)
-
- #: To which data source does this relate?
- datasource = models.ForeignKey('DataSource',
- related_name='metadata_items',
- on_delete=models.CASCADE,
- blank=False, null=False)
-
- class Meta:
- unique_together = (('field', 'datasource', 'value'),)
-
- def __str__(self):
- return self.value
-
-
@enum.unique
class UserPermissionLevels(enum.IntEnum):
"""
diff --git a/datasources/models/metadata.py b/datasources/models/metadata.py
new file mode 100644
index 0000000000000000000000000000000000000000..b018132ea274e67ba1457dd18a47bbdd5f1fb3ba
--- /dev/null
+++ b/datasources/models/metadata.py
@@ -0,0 +1,94 @@
+"""
+This module contains models for dynamic assignment of metadata to data sources.
+"""
+
+from django.core import validators
+from django.db import models
+
+from core.models import MAX_LENGTH_NAME
+from .datasource import DataSource, MAX_LENGTH_REASON
+
+
+class MetadataField(models.Model):
+ """
+ A metadata field that can be dynamically added to a data source.
+
+ Operational MetadataFields are those which have some associated code within PEDASI.
+ They should be present within any deployment of PEDASI.
+
+ Current operational metadata fields are (by short_name):
+ - data_query_param
+ - indexed_field
+ """
+ #: Name of the field
+ name = models.CharField(max_length=MAX_LENGTH_NAME,
+ unique=True,
+ blank=False, null=False)
+
+ #: Short text identifier for the field
+ short_name = models.CharField(
+ max_length=MAX_LENGTH_NAME,
+ validators=[
+ validators.RegexValidator(
+ r'^[a-zA-Z][a-zA-Z0-9_]*\Z',
+ 'Short name must begin with a letter and consist only of letters, numbers and underscores.',
+ 'invalid'
+ )
+ ],
+ unique=True,
+ blank=False, null=False
+ )
+
+ #: Does the field have an operational effect within PEDASI?
+ operational = models.BooleanField(default=False,
+ blank=False, null=False)
+
+ def __str__(self):
+ return self.name
+
+ @classmethod
+ def load_inline_fixtures(cls):
+ """
+ Create any instances required for the functioning of PEDASI.
+
+ This is called from within the AppConfig.
+ """
+ fixtures = (
+ ('data_query_param', 'data_query_param', True),
+ ('indexed_field', 'indexed_field', True),
+ )
+
+ for name, short_name, operational in fixtures:
+ obj, created = cls.objects.get_or_create(
+ name=name,
+ short_name=short_name
+ )
+ obj.operational = operational
+ obj.save()
+
+
+class MetadataItem(models.Model):
+ """
+ The value of the metadata field on a given data source.
+ """
+ #: The value of this metadata field
+ value = models.CharField(max_length=MAX_LENGTH_REASON,
+ blank=True, null=False)
+
+ #: To which field does this relate?
+ field = models.ForeignKey(MetadataField,
+ related_name='values',
+ on_delete=models.PROTECT,
+ blank=False, null=False)
+
+ #: To which data source does this relate?
+ datasource = models.ForeignKey(DataSource,
+ related_name='metadata_items',
+ on_delete=models.CASCADE,
+ blank=False, null=False)
+
+ class Meta:
+ unique_together = (('field', 'datasource', 'value'),)
+
+ def __str__(self):
+ return self.value
diff --git a/datasources/models/quality.py b/datasources/models/quality.py
new file mode 100644
index 0000000000000000000000000000000000000000..b5bc6bd207badb85cb8ce8ed3d4340e9a5155755
--- /dev/null
+++ b/datasources/models/quality.py
@@ -0,0 +1,143 @@
+"""
+This module contains models related to the quality assessment of data sources.
+"""
+import itertools
+
+from django.db import models
+
+from core.models import MAX_LENGTH_NAME
+from .datasource import DataSource
+from .metadata import MetadataField
+
+
+class QualityRuleset(models.Model):
+ """
+ A ruleset for assessing the quality of a data source.
+ """
+ class Meta:
+ unique_together = (('name', 'version',),)
+
+ # Prevent template engine from trying to call the model
+ do_not_call_in_templates = True
+
+ #: Name of the ruleset
+ name = models.CharField(max_length=MAX_LENGTH_NAME,
+ blank=False, null=False)
+
+ #: Short text identifier
+ short_name = models.CharField(max_length=MAX_LENGTH_NAME,
+ unique=True, blank=True, null=False)
+
+ #: Ruleset version - distinguishes successive versions of the same set
+ version = models.CharField(max_length=MAX_LENGTH_NAME,
+ blank=False, null=False)
+
+ def __call__(self, datasource: DataSource) -> int:
+ """
+ Evaluate a data source to get its quality level under this ruleset.
+
+ :param datasource: Data source to assess
+ :return: Quality level of data source
+ """
+ # Get list of all levels that the data source passes - stop when a fail is encountered
+ passes = list(
+ itertools.takewhile(
+ lambda x: x(datasource),
+ self.levels.all()
+ )
+ )
+
+ try:
+ # Highest passed level will be the last in the list
+ return passes[-1].level
+
+ except IndexError:
+ return 0
+
+ def __str__(self):
+ return '{0} - {1}'.format(self.name, self.version)
+
+
+class QualityLevel(models.Model):
+ """
+ A set of criteria that is required to grant a particular quality level.
+ """
+ class Meta:
+ unique_together = (('ruleset', 'level'),)
+ ordering = ['level']
+
+ # Prevent template engine from trying to call the model
+ do_not_call_in_templates = True
+
+ #: Which ruleset does this level belong to?
+ ruleset = models.ForeignKey(QualityRuleset, related_name='levels',
+ on_delete=models.CASCADE,
+ blank=False, null=False)
+
+ #: What level is this?
+ level = models.PositiveSmallIntegerField(blank=False, null=False)
+
+ #: Threshold level that must be exceeded by criteria weights to pass this level
+ threshold = models.FloatField(blank=True, null=True)
+
+ def __call__(self, datasource: DataSource, rtol: float = 1e-3) -> bool:
+ """
+ Does a data source pass the criteria for this quality level?
+
+ :param datasource: Data source to assess
+ :param rtol: Relative tolerance to compare floating point threshold
+ :return: Passes this quality level?
+ """
+ threshold = self.threshold
+ if threshold is None:
+ threshold = self.criteria.aggregate(models.Sum('weight'))['weight__sum']
+
+ total = sum(criterion(datasource) for criterion in self.criteria.all())
+
+ # Compare using relative tolerance to account for floating point error
+ return total >= (threshold * (1 - rtol))
+
+ def __str__(self):
+ return '{0} - level {1}'.format(self.ruleset, self.level)
+
+
+class QualityCriterion(models.Model):
+ """
+ A weighted criterion to determine whether a data source meets a certain quality level.
+ """
+ class Meta:
+ unique_together = (('quality_level', 'metadata_field'),)
+ verbose_name_plural = 'quality criteria'
+
+ # Prevent template engine from trying to call the model
+ do_not_call_in_templates = True
+
+ #: Which quality level does this criterion belong to?
+ quality_level = models.ForeignKey(QualityLevel, related_name='criteria',
+ on_delete=models.CASCADE,
+ blank=False, null=False)
+
+ #: Which metadata field represents this criterion?
+ metadata_field = models.ForeignKey(MetadataField, related_name='+',
+ on_delete=models.PROTECT,
+ blank=False, null=False)
+
+ #: What proportion of the quality level does this criterion provide?
+ weight = models.FloatField(default=1,
+ blank=False, null=False)
+
+ def __call__(self, datasource: DataSource) -> float:
+ """
+ Weight provided to the quality level based on passing or failing this criterion.
+
+ :param datasource: Data source to assess
+ :return: Weight provided to the quality level from passing or failing this criterion
+ """
+ return self.weight if datasource.metadata_items.filter(field=self.metadata_field).exists() else 0
+
+ def __str__(self):
+ return '{0} - weight {1}'.format(self.metadata_field, self.weight)
+
+
+
+
diff --git a/datasources/serializers.py b/datasources/serializers.py
index 232ecd4725377d62613a487c429cd879db21d5e3..2d73fc3d5c613a749396bc03a4295fb4ef0ac3ec 100644
--- a/datasources/serializers.py
+++ b/datasources/serializers.py
@@ -16,8 +16,6 @@ class MetadataFieldSerializer(serializers.ModelSerializer):
class MetadataItemSerializer(serializers.ModelSerializer):
- field = MetadataFieldSerializer(read_only=True)
-
class Meta:
model = models.MetadataItem
fields = ['field', 'value']
@@ -40,3 +38,25 @@ class DataSourceSerializer(serializers.ModelSerializer):
'encrypted_docs_url',
'metadata_items'
]
+
+
+class QualityCriterionSerializer(serializers.ModelSerializer):
+ class Meta:
+ model = models.QualityCriterion
+ fields = ['weight', 'metadata_field']
+
+
+class QualityLevelSerializer(serializers.ModelSerializer):
+ criteria = QualityCriterionSerializer(many=True, read_only=True)
+
+ class Meta:
+ model = models.QualityLevel
+ fields = ['level', 'threshold', 'criteria']
+
+
+class QualityRulesetSerializer(serializers.ModelSerializer):
+ levels = QualityLevelSerializer(many=True, read_only=True)
+
+ class Meta:
+ model = models.QualityRuleset
+ fields = '__all__'
diff --git a/datasources/static/js/metadata.js b/datasources/static/js/metadata.js
index e05914501082d1d28d7b232df888db58053968c2..89a3ec747c6789c720752d19f31f9e7154396ffa 100644
--- a/datasources/static/js/metadata.js
+++ b/datasources/static/js/metadata.js
@@ -1,6 +1,7 @@
"use strict";
let metadataUrl = null;
+let ratingUrl = null;
/**
@@ -13,8 +14,14 @@ function setMetadataUrl(url) {
metadataUrl = url;
}
+function setRatingUrl(url) {
+ "use strict";
+ ratingUrl = url;
+}
+
function getCookie(name) {
+ "use strict";
const re = new RegExp("(^| )" + name + "=([^;]+)");
const match = document.cookie.match(re);
if (match) {
@@ -23,30 +30,33 @@ function getCookie(name) {
}
-function tableAppendRow(table, values, id=null) {
- const row = table.insertRow(-1);
- if (id !== null) {
- row.id = id;
- }
+/**
+ * Create a new metadata item or update an existing one.
+ *
+ * @param event Form event
+ */
+function submitMetadataForm(event) {
+ "use strict";
- for (const value of values) {
- const cell = row.insertCell(-1);
- const text = document.createTextNode(value);
- cell.appendChild(text);
- }
+ const metadataIdField = event.target.getElementsByClassName("fieldMetadataId")[0];
+ const metadataValueIdField = event.target.getElementsByClassName("fieldMetadataValueId")[0];
+ const valueField = event.target.getElementsByClassName("fieldMetadataValue")[0];
- return row;
-}
+ let url = metadataUrl;
+ let method = "POST";
+ if (metadataValueIdField && metadataValueIdField.value) {
+ url = metadataUrl + metadataValueIdField.value.toString() + "/";
+ method = "PUT";
+ }
-function postMetadata() {
fetch(
- metadataUrl,
+ url,
{
- method: "POST",
+ method: method,
body: JSON.stringify({
- field: document.getElementById("id_field").value,
- value: document.getElementById("id_value").value
+ field: metadataIdField.value,
+ value: valueField.value
}),
headers: {
"Accept": "application/json",
@@ -54,68 +64,90 @@ function postMetadata() {
"X-CSRFToken": getCookie("csrftoken")
}
}
- ).then(
- response => response.json()
- ).then(
- function (response) {
- if (response.status === "success") {
- const table = document.getElementById("tableMetadata");
- const field = response.data.field;
- const value = response.data.value;
-
- const row = tableAppendRow(
- table,
- [field, value],
- "metadata-row-" + response.data.field_short + "-" + value
- );
-
- const buttonCell = row.insertCell(-1);
- const button = document.createElement("button");
-
- button.id = "btn-" + field + "-" + value;
- button.classList.add("btn", "btn-sm", "btn-danger", "float-right");
- button.addEventListener(
- "click",
- function () {
- const f = response.data.field_short;
- const v = value;
- deleteMetadata(f, v);
- }
- );
- button.textContent = "Delete";
- buttonCell.appendChild(button);
- row.appendChild(buttonCell);
- }
+
+ ).then(function (response) {
+ if (!response.ok) {
+ throw new Error("Request failed: " + response.statusText);
}
- ).catch(
- function (e) {
- console.log(e);
+
+ valueField.dataset.currentValue = valueField.value;
+ window.location.reload();
+
+ }).catch(function (error) {
+ console.log(error);
+
+ if (valueField.dataset.currentValue !== null) {
+ valueField.value = valueField.dataset.lastValue;
}
- )
+
+ });
}
-function deleteMetadata(field, value) {
+/**
+ * Delete an existing metadata item.
+ *
+ * @param event Form event
+ */
+function deleteMetadataForm(event) {
+ "use strict";
+
+ const metadataIdField = event.target.getElementsByClassName("fieldMetadataValueId")[0];
+
fetch(
- metadataUrl,
+ metadataUrl + metadataIdField.value.toString() + "/",
{
method: "DELETE",
- body: JSON.stringify({
- field: field,
- value: value
- }),
headers: {
"Accept": "application/json",
- "Content-Type": "application/json",
"X-CSRFToken": getCookie("csrftoken")
}
}
- ).then(
- response => response.json()
+
).then(function (response) {
- if (response.status === "success") {
- const row = document.getElementById("metadata-row-" + field + "-" + value);
- row.parentNode.removeChild(row);
+ if (!response.ok) {
+ throw new Error("Request failed: " + response.statusText);
}
- })
+ window.location.reload();
+
+ }).catch(function (error) {
+ console.log(error);
+
+ });
+}
+
+/**
+ * Update the star rating display.
+ *
+ * @param url Url to get quality level for a data source - defaults to global setting 'ratingUrl'
+ * @param badgeId Id of badge element to populate with stars - defaults to 'qualityLevelBadge'
+ */
+function updateLevelBadge(url, badgeId) {
+ "use strict";
+
+ if (!url) {
+ url = ratingUrl;
+ }
+
+ if (!badgeId) {
+ badgeId = "qualityLevelBadge";
+ }
+
+ fetch(url).then(
+ (response) => response.json()
+
+ ).then(function (data) {
+ const levelBadge = document.getElementById(badgeId);
+
+ // Clear existing rating
+ while (levelBadge.hasChildNodes()) {
+ levelBadge.removeChild(levelBadge.lastChild);
+ }
+
+ for (let i = 0; i < data.quality; i++) {
+ const star = document.createElement("i");
+ star.classList.add("fas", "fa-star");
+ levelBadge.appendChild(star);
+ }
+ });
}
diff --git a/datasources/templates/datasources/datasource/detail.html b/datasources/templates/datasources/datasource/detail.html
index d2a4c823b24511ab37f10a27af890603d72f5564..ab1c44aa2f02d474a7142aa56674adb288d2bc8e 100644
--- a/datasources/templates/datasources/datasource/detail.html
+++ b/datasources/templates/datasources/datasource/detail.html
@@ -2,15 +2,9 @@
{% load bootstrap4 %}
{% block extra_head %}
- {% if metadata_field_form %}
- {% load static %}
+ {% load static %}
- <script type="application/javascript" src="{% static 'js/metadata.js' %}"></script>
-
- <script type="application/javascript">
- setMetadataUrl("/datasources/{{ datasource.pk }}/metadata");
- </script>
- {% endif %}
+ <script type="application/javascript" src="{% static 'js/metadata.js' %}"></script>
{% endblock %}
{% block content %}
@@ -32,23 +26,28 @@
<div class="col-md-10 col-sm-8">
<h2>
{{ datasource.name }}
- {# TODO make this a template tag #}
- {% if datasource.licence %}
- <small>
+ <small>
+ {% if datasource.licence %}
<a href="{% url 'datasources:licence.detail' pk=datasource.licence.pk %}"
class="badge badge-info"
data-toggle="tooltip" data-placement="bottom" title="{{ datasource.licence.name }}">
{{ datasource.licence.short_name }}
</a>
- </small>
- {% else %}
- <small>
+ {% else %}
<span class="badge badge-warning"
data-toggle="tooltip" data-placement="bottom" title="No Licence">
No Licence
</span>
- </small>
- {% endif %}
+ {% endif %}
+
+ <span id="qualityLevelBadge" class="badge badge-primary"></span>
+
+ <script type="application/javascript">
+ document.addEventListener("DOMContentLoaded", function () {
+ updateLevelBadge("{% url 'api:datasource-quality' pk=datasource.pk %}");
+ });
+ </script>
+ </small>
</h2>
{% if datasource.description %}
@@ -69,6 +68,9 @@
<a href="{% url 'datasources:datasource.access.manage' pk=datasource.pk %}"
class="btn btn-block btn-primary" role="button">Manage Access</a>
+ <a href="{% url 'datasources:datasource.metadata' pk=datasource.pk %}"
+ class="btn btn-block btn-secondary" role="button">Manage Metadata</a>
+
<a href="{% url 'datasources:datasource.edit' datasource.id %}"
class="btn btn-block btn-success" role="button">Edit</a>
@@ -128,9 +130,6 @@
<thead>
<th scope="col" class="w-25 border-0"></th>
<th scope="col" class="border-0"></th>
- {% if metadata_field_form %}
- <th scope="col" class="w-25 border-0"></th>
- {% endif %}
</thead>
<tbody>
@@ -138,36 +137,10 @@
<tr id="metadata-row-{{ metadata_item.field.short_name }}-{{ metadata_item.value }}">
<td>{{ metadata_item.field.name }}</td>
<td>{{ metadata_item.value }}</td>
-
- {% if metadata_field_form %}
- <td>
- <button role="button"
- onclick="deleteMetadata('{{ metadata_item.field.short_name }}','{{ metadata_item.value }}')"
- class="btn btn-sm btn-danger float-right">Delete</button>
- </td>
- {% endif %}
</tr>
{% endfor %}
</tbody>
</table>
-
- {% if metadata_field_form %}
- <form id="formMetadata" class="form" action="javascript:postMetadata();">
- {% csrf_token %}
-
- <div class="row">
- <div class="col-3">
- {% bootstrap_field metadata_field_form.field layout='inline' %}
- </div>
- <div class="col-6">
- {% bootstrap_field metadata_field_form.value layout='inline' %}
- </div>
- <div class="col-3">
- <button type="submit" class="btn btn-block btn-success">Add</button>
- </div>
- </div>
- </form>
- {% endif %}
</div>
</div>
diff --git a/datasources/templates/datasources/datasource/list.html b/datasources/templates/datasources/datasource/list.html
index 771bf6d7d6315eef819457cb24251cffd2c0dd95..f06cac0638415f4568030eae77312e20809ee06c 100644
--- a/datasources/templates/datasources/datasource/list.html
+++ b/datasources/templates/datasources/datasource/list.html
@@ -1,6 +1,12 @@
{% extends "base.html" %}
{% load bootstrap4 %}
+{% block extra_head %}
+ {% load static %}
+
+ <script type="application/javascript" src="{% static 'js/metadata.js' %}"></script>
+{% endblock %}
+
{% block content %}
<nav aria-label="breadcrumb">
<ol class="breadcrumb">
@@ -56,6 +62,17 @@
No Licence
</span>
{% endif %}
+ <span id="qualityLevelBadge{{ datasource.pk }}" class="badge badge-primary"></span>
+
+ <script type="application/javascript">
+ // TODO Should this be a single function to loop over all data sources?
+ document.addEventListener("DOMContentLoaded", function () {
+ updateLevelBadge(
+ "{% url 'api:datasource-quality' pk=datasource.pk %}",
+ "qualityLevelBadge{{ datasource.pk }}"
+ );
+ });
+ </script>
</p>
<p class="pl-5">
{{ datasource.description|truncatechars:120 }}
diff --git a/datasources/templates/datasources/datasource/metadata.html b/datasources/templates/datasources/datasource/metadata.html
index 9cf8b36525b3410f05e3882aaa7821ada2b3c5af..99e5b8559a72ef6e12deb3a63dc8f9fc7a00384e 100644
--- a/datasources/templates/datasources/datasource/metadata.html
+++ b/datasources/templates/datasources/datasource/metadata.html
@@ -1,6 +1,17 @@
{% extends "base.html" %}
{% load bootstrap4 %}
+{% block extra_head %}
+ {% load static %}
+
+ <script type="application/javascript" src="{% static 'js/metadata.js' %}"></script>
+
+ <script type="application/javascript">
+ setMetadataUrl("{% url 'api:metadata-item-list' datasource_pk=datasource.pk %}");
+ setRatingUrl("{% url 'api:datasource-quality' pk=datasource.pk %}");
+ </script>
+{% endblock %}
+
{% block content %}
<nav aria-label="breadcrumb">
<ol class="breadcrumb">
@@ -19,11 +30,19 @@
</ol>
</nav>
- <h2>View Data Source - {{ datasource.name }}</h2>
+ <h2>
+ View Data Source - {{ datasource.name }}
- <p>
- Owner: <a href="#" role="link">{{ datasource.owner }}</a>
- </p>
+ <small>
+ <span id="qualityLevelBadge" class="badge badge-primary"></span>
+
+ <script type="application/javascript">
+ document.addEventListener("DOMContentLoaded", function () {
+ updateLevelBadge();
+ });
+ </script>
+ </small>
+ </h2>
{% if datasource.description %}
<p>{{ datasource.description }}</p>
@@ -31,46 +50,164 @@
<hr>
- <table class="table">
- <thead class="thead">
- <tr>
- <th>Relation</th>
- <th>Values</th>
- </tr>
- </thead>
+ {% load datasource_extras %}
+ <h2>Quality Metadata</h2>
+
+ <table class="table">
<tbody>
- {% for relation, values in metadata.items %}
+
+ {% for level in ruleset.levels.all %}
<tr>
- <td>{{ relation }}</td>
- <td>
- {% for value in values %}
- <p>{{ value }}</p>
- {% endfor %}
- </td>
+ <th class="row">
+ <div class="col-md-3">
+ Level {{ level.level }}
+ </div>
+
+ <div class="col-md-1">
+ {{ level.threshold }}
+ </div>
+ </th>
</tr>
+
+ {% for criterion in level.criteria.all %}
+ {% with metadata_items|access:criterion.metadata_field.short_name as metadata_field_group %}
+ <tr>
+ <td>
+ <div class="row">
+ <div class="col-md-3">
+ <span class="form-text">{{ criterion.metadata_field.name }}</span>
+ </div>
+
+ <div class="col-md-1">
+ {{ criterion.weight }}
+ </div>
+
+ <div class="col-md-8">
+ {% for item in metadata_field_group.list %}
+ <form class="row"
+ onsubmit="submitMetadataForm(event); event.preventDefault();"
+ onreset="deleteMetadataForm(event); event.preventDefault();">
+ <div class="col-md-8">
+ <input type="hidden" class="fieldDatasourceId" readonly
+ value="{{ datasource.id }}">
+
+ <input type="hidden" class="fieldMetadataId" readonly
+ value="{{ metadata_field_group.field.id }}">
+
+ <input type="hidden" class="fieldMetadataValueId" readonly
+ value="{{ item.id }}">
+
+ <input type="text" class="form-control mb-2 fieldMetadataValue"
+ value="{{ item.value }}" data-last-value="{{ item.value }}">
+ </div>
+
+ <div class="col-md-2">
+ <button type="submit" class="btn btn-block btn-success mb-2">Save</button>
+ </div>
+
+ <div class="col-md-2">
+ <button type="reset" class="btn btn-block btn-danger mb-2">Delete</button>
+ </div>
+ </form>
+ {% endfor %}
+
+ <form class="row"
+ onsubmit="submitMetadataForm(event); event.preventDefault();">
+ <div class="col-md-8">
+ <input type="hidden" class="fieldDatasourceId" readonly
+ value="{{ datasource.id }}">
+
+ <input type="hidden" class="fieldMetadataId" readonly
+ value="{{ criterion.metadata_field.id }}">
+
+ <input type="text" class="form-control mb-2 fieldMetadataValue">
+ </div>
+
+ <div class="col-md-2">
+ <button type="submit" class="btn btn-block btn-success mb-2">Save</button>
+ </div>
+ </form>
+
+ </div>
+ </div>
+ </td>
+ </tr>
+ {% endwith %}
+
+ {% endfor %}
+
{% endfor %}
+
</tbody>
</table>
<hr>
- <table class="table">
- <thead class="thead">
- <tr>
- <th>URL</th>
- <th>Metadata</th>
- </tr>
- </thead>
+ <h2>All Metadata</h2>
+ <table class="table">
<tbody>
- {% for dataset, dataset_metadata in datasets.items %}
- <tr>
- <td>{{ dataset }}</td>
- <td>{{ dataset_metadata }}</td>
- </tr>
- {% endfor %}
+ {% for metadata_field in metadata_fields %}
+ {% with metadata_items|access:metadata_field.short_name as metadata_field_group %}
+ <tr>
+ <td>
+ <div class="row">
+ <div class="col-md-4">
+ <span class="form-text">{{ metadata_field.name }}</span>
+ </div>
+
+ <div class="col-md-8">
+ {% for item in metadata_field_group.list %}
+ <form class="row"
+ onsubmit="submitMetadataForm(event); event.preventDefault();"
+ onreset="deleteMetadataForm(event); event.preventDefault();">
+ <div class="col-md-8">
+ <input type="hidden" class="fieldDatasourceId" readonly
+ value="{{ datasource.id }}">
+
+ <input type="hidden" class="fieldMetadataId" readonly
+ value="{{ metadata_field_group.field.id }}">
+
+ <input type="hidden" class="fieldMetadataValueId" readonly
+ value="{{ item.id }}">
+
+ <input type="text" class="form-control mb-2 fieldMetadataValue"
+ value="{{ item.value }}" data-last-value="{{ item.value }}">
+ </div>
+
+ <div class="col-md-2">
+ <button type="submit" class="btn btn-block btn-success mb-2">Save</button>
+ </div>
+
+ <div class="col-md-2">
+ <button type="reset" class="btn btn-block btn-danger mb-2">Delete</button>
+ </div>
+ </form>
+ {% endfor %}
+
+ <form class="row"
+ onsubmit="submitMetadataForm(event); event.preventDefault();">
+ <div class="col-md-8">
+ <input type="hidden" class="fieldDatasourceId" readonly
+ value="{{ datasource.id }}">
+
+ <input type="hidden" class="fieldMetadataId" readonly
+ value="{{ metadata_field.id }}">
+
+ <input type="text" class="form-control mb-2 fieldMetadataValue">
+ </div>
+
+ <div class="col-md-2">
+ <button type="submit" class="btn btn-block btn-success mb-2">Save</button>
+ </div>
+ </form>
+ </div>
+ </div>
+ </td>
+ </tr>
+ {% endwith %}
+ {% endfor %}
</tbody>
</table>
-
-{% endblock %}
\ No newline at end of file
+{% endblock %}
diff --git a/datasources/templatetags/__init__.py b/datasources/templatetags/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/datasources/templatetags/datasource_extras.py b/datasources/templatetags/datasource_extras.py
new file mode 100644
index 0000000000000000000000000000000000000000..997794bf550e95734d2c7d87cf7a9a26fcb44d22
--- /dev/null
+++ b/datasources/templatetags/datasource_extras.py
@@ -0,0 +1,27 @@
+from django.template import Library
+
+register = Library()
+
+
+@register.filter
+def access(value, arg):
+ """
+ Template filter to access a dictionary by a key.
+
+ :param value: Dictionary to access
+ :param arg: Key to look up
+ :return: Value of key in dictionary
+ """
+ try:
+ return value[arg]
+
+ except KeyError:
+ return None
+
+ except TypeError:
+ # Is a GroupedResult not a dictionary
+ for item in value:
+ if item.field.short_name == arg:
+ return item
+
+ return None
diff --git a/datasources/tests/test_quality.py b/datasources/tests/test_quality.py
new file mode 100644
index 0000000000000000000000000000000000000000..c71af1a62fa1e0aec1cddd28e72ed8247859fc0c
--- /dev/null
+++ b/datasources/tests/test_quality.py
@@ -0,0 +1,154 @@
+from django.contrib.auth import get_user_model
+from django.test import TestCase
+
+from datasources import models
+
+
+class QualityRulesetTest(TestCase):
+ @classmethod
+ def setUpClass(cls):
+ super().setUpClass()
+
+ cls.user = get_user_model().objects.create_user(
+ username='Test User 1'
+ )
+
+ cls.metadata_field = models.MetadataField.objects.create(
+ name='Test Metadata Field 1',
+ short_name='TMF1'
+ )
+
+ def setUp(self):
+ super().setUp()
+
+ self.ruleset = models.QualityRuleset.objects.create(
+ name='Test Ruleset 1',
+ short_name='TR1v1',
+ version='1'
+ )
+
+ self.datasource = models.DataSource.objects.create(
+ name='Test Data Source 1',
+ owner=self.user
+ )
+
+ def test_create_ruleset(self):
+ ruleset = models.QualityRuleset.objects.get(
+ short_name='TR1v1'
+ )
+
+ self.assertEqual('Test Ruleset 1', ruleset.name)
+ self.assertEqual('TR1v1', ruleset.short_name)
+ self.assertEqual('1', ruleset.version)
+
+ return ruleset
+
+ def test_ruleset_eval_null(self):
+ """
+ Test that an empty ruleset identifies any datasource as level 0.
+ """
+ level = self.ruleset(self.datasource)
+
+ self.assertEqual(0, level)
+
+ def test_ruleset_create_level(self):
+ """
+ Test that levels can be added to rulesets.
+ """
+ self.ruleset.levels.create(
+ level=1
+ )
+
+ self.assertEqual(1, self.ruleset.levels.count())
+
+ def test_ruleset_create_criterion(self):
+ """
+ Test that criteria can be added to quality levels.
+ """
+ level_1 = self.ruleset.levels.create(
+ level=1
+ )
+
+ level_1.criteria.create(
+ metadata_field=self.metadata_field
+ )
+
+ self.assertEqual(1, level_1.criteria.count())
+
+ def test_ruleset_eval_criterion(self):
+ """
+ Test that criteria are evaluated correctly.
+
+ Should return weight contributed to level threshold.
+ """
+ level_1 = self.ruleset.levels.create(
+ level=1,
+ threshold=1
+ )
+
+ criterion = level_1.criteria.create(
+ metadata_field=self.metadata_field,
+ weight=1
+ )
+
+ self.assertEqual(1, level_1.criteria.count())
+
+ self.assertAlmostEqual(0, criterion(self.datasource))
+
+ self.datasource.metadata_items.create(
+ field=self.metadata_field,
+ value=''
+ )
+
+ self.assertAlmostEqual(1, criterion(self.datasource))
+
+ def test_ruleset_eval_level(self):
+ """
+ Test that quality levels are evaluated correctly.
+
+ Should return whether data source passes level threshold.
+ """
+ level_1 = self.ruleset.levels.create(
+ level=1,
+ threshold=1
+ )
+
+ level_1.criteria.create(
+ metadata_field=self.metadata_field,
+ weight=1
+ )
+
+ self.assertFalse(level_1(self.datasource))
+
+ self.datasource.metadata_items.create(
+ field=self.metadata_field,
+ value=''
+ )
+
+ self.assertTrue(level_1(self.datasource))
+
+ def test_ruleset_eval(self):
+ """
+ Test that rulesets are evaluated correctly.
+
+ Should return quality level of data source.
+ """
+ level_1 = self.ruleset.levels.create(
+ level=1,
+ threshold=1
+ )
+
+ level_1.criteria.create(
+ metadata_field=self.metadata_field,
+ weight=1
+ )
+
+ self.assertEqual(0, self.ruleset(self.datasource))
+
+ self.datasource.metadata_items.create(
+ field=self.metadata_field,
+ value=''
+ )
+
+ self.assertEqual(1, self.ruleset(self.datasource))
+
diff --git a/datasources/urls.py b/datasources/urls.py
index f8f94d360cdfd75d0baed7a309fe37363be74c2a..0974dc716b7efeaf8cbf9f67546f2b5d85db3925 100644
--- a/datasources/urls.py
+++ b/datasources/urls.py
@@ -26,7 +26,7 @@ urlpatterns = [
name='datasource.delete'),
path('<int:pk>/metadata',
- views.datasource.DataSourceMetadataAjaxView.as_view(),
+ views.datasource.DataSourceMetadataView.as_view(),
name='datasource.metadata'),
path('<int:pk>/explorer',
diff --git a/datasources/views/datasource.py b/datasources/views/datasource.py
index 1ea6417e764043f439af276d276805c6458c6426..0b2b58eb3b231ea7d9e06668044acb52145d4ac3 100644
--- a/datasources/views/datasource.py
+++ b/datasources/views/datasource.py
@@ -1,4 +1,7 @@
+from collections import namedtuple
+
from django.contrib import messages
+from django.contrib.auth import get_user_model
from django.contrib.auth.mixins import PermissionRequiredMixin
from django.http import HttpResponse
from django.urls import reverse_lazy
@@ -36,8 +39,6 @@ class DataSourceDetailView(DetailView):
context = super().get_context_data(**kwargs)
context['has_edit_permission'] = self.request.user.is_superuser or self.request.user == self.object.owner
- if context['has_edit_permission']:
- context['metadata_field_form'] = forms.MetadataFieldForm()
try:
context['is_catalogue'] = self.object.is_catalogue
@@ -132,64 +133,31 @@ class DataSourceDataSetSearchView(HasPermissionLevelMixin, DetailView):
return context
-class DataSourceMetadataAjaxView(OwnerPermissionMixin, APIView):
+FieldValueSet = namedtuple('FieldValueSet', ['field', 'list'])
+
+
+class DataSourceMetadataView(OwnerPermissionMixin, DetailView):
model = models.DataSource
+ template_name = 'datasources/datasource/metadata.html'
+ context_object_name = 'datasource'
- # Don't redirect to login page if unauthorised
- raise_exception = True
-
- class MetadataSerializer(serializers.ModelSerializer):
- class Meta:
- model = models.MetadataItem
- fields = '__all__'
-
- def get_object(self, pk):
- return self.model.objects.get(pk=pk)
-
- def post(self, request, pk, format=None):
- """
- Create a new MetadataItem associated with this DataSource.
- """
- datasource = self.get_object(pk)
- if 'datasource' not in request.data:
- request.data['datasource'] = datasource.pk
-
- serializer = self.MetadataSerializer(data=request.data)
-
- if serializer.is_valid():
- obj = serializer.save()
-
- return Response({
- 'status': 'success',
- 'data': {
- 'datasource': datasource.pk,
- 'field': obj.field.name,
- 'field_short': obj.field.short_name,
- 'value': obj.value,
- }
- })
-
- return Response({'status': 'failure'}, status=400)
-
- def delete(self, request, pk, format=None):
- """
- Delete a MetadataItem associated with this DataSource.
- """
- datasource = self.get_object(pk)
- if 'datasource' not in request.data:
- request.data['datasource'] = datasource.pk
-
- metadata_item = models.MetadataItem.objects.get(
- datasource=datasource,
- field__short_name=self.request.data['field'],
- value=self.request.data['value']
- )
+ def get_context_data(self, **kwargs):
+ context = super().get_context_data(**kwargs)
- metadata_item.delete()
+ context['ruleset'] = get_user_model().get_quality_ruleset()
+ context['metadata_fields'] = models.MetadataField.objects.all()
- return Response({
- 'status': 'success'
- })
+ items = []
+ present_fields = {item.field for item in self.object.metadata_items.all()}
+ for field in present_fields:
+ items.append(
+ FieldValueSet(field, [
+ item for item in self.object.metadata_items.all() if item.field == field
+ ])
+ )
+ context['metadata_items'] = items
+
+ return context
class DataSourceExplorerView(HasPermissionLevelMixin, DetailView):
diff --git a/profiles/models.py b/profiles/models.py
index 738fa71aaca9294d9327cb4476253935c4d89972..6afff109d5152d7ddcb09a2db92691c91172d12e 100644
--- a/profiles/models.py
+++ b/profiles/models.py
@@ -6,6 +6,8 @@ from django.urls import reverse
from rest_framework.authtoken.models import Token
+from datasources.models.quality import QualityRuleset
+
class User(AbstractUser):
"""
@@ -35,3 +37,12 @@ class User(AbstractUser):
"""
self.auth_token.delete()
+ # TODO ruleset should be configurable by user
+ @staticmethod
+ def get_quality_ruleset():
+ try:
+ return QualityRuleset.objects.first()
+
+ except QualityRuleset.DoesNotExist:
+ return None
+
diff --git a/profiles/templates/index.html b/profiles/templates/index.html
index 732e933b68e7d3cb73856451f423433a64d5ed08..152b53de15c013cdc697f45e91ce115ae4864e3b 100644
--- a/profiles/templates/index.html
+++ b/profiles/templates/index.html
@@ -3,7 +3,10 @@
{% block extra_head %}
{% load static %}
+
<link rel="stylesheet" href="{% static 'css/masthead.css' %}">
+
+ <script type="application/javascript" src="{% static 'js/metadata.js' %}"></script>
{% endblock %}
{% block pre_content %}
@@ -148,7 +151,22 @@
style="width: 150px;">
<div class="card-body d-flex flex-column pb-2">
- <h5 class="card-title">{{ datasource.name }}</h5>
+ <h5 class="card-title">
+ {{ datasource.name }}
+
+ <span id="qualityLevelBadge{{ datasource.pk }}"
+ class="badge badge-primary float-right"></span>
+
+ <script type="application/javascript">
+ // TODO Should this be a single function to loop over all data sources?
+ document.addEventListener("DOMContentLoaded", function () {
+ updateLevelBadge(
+ "{% url 'api:datasource-quality' pk=datasource.pk %}",
+ "qualityLevelBadge{{ datasource.pk }}"
+ );
+ });
+ </script>
+ </h5>
<p class="card-text">{{ datasource.description|truncatechars:120 }}</p>
<p class="card-text text-right mt-auto">
diff --git a/requirements.txt b/requirements.txt
index 4edb81020e50bcc916a03754f8089bb582e121e1..8945d1d5ddd9dcd08e4c71477e0c6789ebbf1095 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -14,6 +14,7 @@ django-cors-headers==2.4.0
django-haystack==2.8.1
djangorestframework==3.9.1
docutils==0.14
+drf-nested-routers==0.91
idna==2.7
imagesize==1.0.0
isodate==0.6.0