From c042ba85c5cd2e08d61e6fb301f42638a3632608 Mon Sep 17 00:00:00 2001
From: James Graham <J.Graham@software.ac.uk>
Date: Mon, 29 Apr 2019 16:09:44 +0100
Subject: [PATCH] Give metadata fields human names and add docstrings
---
datasources/models/metadata.py | 20 +++++++++++++-------
datasources/models/pipeline.py | 9 +++++++--
datasources/pipeline/base.py | 13 +++++++++++--
3 files changed, 31 insertions(+), 11 deletions(-)
diff --git a/datasources/models/metadata.py b/datasources/models/metadata.py
index 156ba5c..90da0ea 100644
--- a/datasources/models/metadata.py
+++ b/datasources/models/metadata.py
@@ -54,16 +54,22 @@ class MetadataField(models.Model):
This is called from within the AppConfig.
"""
fixtures = (
- ('data_query_param', 'data_query_param', True),
- ('indexed_field', 'indexed_field', True),
- ('schema', 'schema', True),
+ ('Accepted Query Parameter', 'data_query_param', True),
+ ('Indexed Field', 'indexed_field', True),
+ ('Validation Schema', 'schema', True),
)
for name, short_name, operational in fixtures:
- obj, created = cls.objects.get_or_create(
- name=name,
- short_name=short_name
- )
+ try:
+ obj = cls.objects.get(short_name=short_name)
+
+ except cls.DoesNotExist:
+ obj = cls.objects.create(
+ name=name,
+ short_name=short_name
+ )
+
+ obj.name = name
obj.operational = operational
obj.save()
diff --git a/datasources/models/pipeline.py b/datasources/models/pipeline.py
index fa9c23c..6076e01 100644
--- a/datasources/models/pipeline.py
+++ b/datasources/models/pipeline.py
@@ -20,9 +20,11 @@ class PipelineSetupError(BaseException):
class Pipeline(models.Model):
- class Meta:
- pass
+ """
+ Model representing a data pipeline.
+ A pipeline may contain multiple stages.
+ """
# Prevent template engine from trying to call the model
do_not_call_in_templates = True
@@ -57,6 +59,9 @@ class Pipeline(models.Model):
class PipelineStage(models.Model):
+ """
+ Model representing a stage of a data pipeline.
+ """
class Meta:
order_with_respect_to = 'pipeline'
diff --git a/datasources/pipeline/base.py b/datasources/pipeline/base.py
index 205d885..1a53d2b 100644
--- a/datasources/pipeline/base.py
+++ b/datasources/pipeline/base.py
@@ -13,6 +13,9 @@ from .. import models
class BasePipelineStage(metaclass=plugin.Plugin):
+ """
+ Base class for pipeline stages. Allows the defined classes to be used as plugins.
+ """
#: Help string to be shown when a user is building a pipeline
description = None
@@ -21,6 +24,12 @@ class BasePipelineStage(metaclass=plugin.Plugin):
@abc.abstractmethod
def __call__(self, data: typing.Mapping) -> typing.Mapping:
+ """
+ Pass data through the pipeline stage.
+
+ :param data: Data to process
+ :return: Processed data
+ """
raise NotImplementedError
@@ -37,10 +46,10 @@ class NullPipelineStage(BasePipelineStage):
class JsonValidationPipelineStage(BasePipelineStage):
"""
- Always raises an error.
+ Validate returned data against a JSON schema.
"""
#: Help string to be shown when a user is building a pipeline
- description = 'Raise an error'
+ description = 'Validates data against a JSON schema'
def __init__(self, options: typing.Optional[typing.Mapping] = None):
super().__init__(options)
--
GitLab