Skip to content
Snippets Groups Projects
Commit c042ba85 authored by James Graham's avatar James Graham
Browse files

Give metadata fields human names and add docstrings

parent 0501dc0c
No related branches found
No related tags found
No related merge requests found
...@@ -54,16 +54,22 @@ class MetadataField(models.Model): ...@@ -54,16 +54,22 @@ class MetadataField(models.Model):
This is called from within the AppConfig. This is called from within the AppConfig.
""" """
fixtures = ( fixtures = (
('data_query_param', 'data_query_param', True), ('Accepted Query Parameter', 'data_query_param', True),
('indexed_field', 'indexed_field', True), ('Indexed Field', 'indexed_field', True),
('schema', 'schema', True), ('Validation Schema', 'schema', True),
) )
for name, short_name, operational in fixtures: for name, short_name, operational in fixtures:
obj, created = cls.objects.get_or_create( try:
name=name, obj = cls.objects.get(short_name=short_name)
short_name=short_name
) except cls.DoesNotExist:
obj = cls.objects.create(
name=name,
short_name=short_name
)
obj.name = name
obj.operational = operational obj.operational = operational
obj.save() obj.save()
......
...@@ -20,9 +20,11 @@ class PipelineSetupError(BaseException): ...@@ -20,9 +20,11 @@ class PipelineSetupError(BaseException):
class Pipeline(models.Model): class Pipeline(models.Model):
class Meta: """
pass Model representing a data pipeline.
A pipeline may contain multiple stages.
"""
# Prevent template engine from trying to call the model # Prevent template engine from trying to call the model
do_not_call_in_templates = True do_not_call_in_templates = True
...@@ -57,6 +59,9 @@ class Pipeline(models.Model): ...@@ -57,6 +59,9 @@ class Pipeline(models.Model):
class PipelineStage(models.Model): class PipelineStage(models.Model):
"""
Model representing a stage of a data pipeline.
"""
class Meta: class Meta:
order_with_respect_to = 'pipeline' order_with_respect_to = 'pipeline'
......
...@@ -13,6 +13,9 @@ from .. import models ...@@ -13,6 +13,9 @@ from .. import models
class BasePipelineStage(metaclass=plugin.Plugin): class BasePipelineStage(metaclass=plugin.Plugin):
"""
Base class for pipeline stages. Allows the defined classes to be used as plugins.
"""
#: Help string to be shown when a user is building a pipeline #: Help string to be shown when a user is building a pipeline
description = None description = None
...@@ -21,6 +24,12 @@ class BasePipelineStage(metaclass=plugin.Plugin): ...@@ -21,6 +24,12 @@ class BasePipelineStage(metaclass=plugin.Plugin):
@abc.abstractmethod @abc.abstractmethod
def __call__(self, data: typing.Mapping) -> typing.Mapping: def __call__(self, data: typing.Mapping) -> typing.Mapping:
"""
Pass data through the pipeline stage.
:param data: Data to process
:return: Processed data
"""
raise NotImplementedError raise NotImplementedError
...@@ -37,10 +46,10 @@ class NullPipelineStage(BasePipelineStage): ...@@ -37,10 +46,10 @@ class NullPipelineStage(BasePipelineStage):
class JsonValidationPipelineStage(BasePipelineStage): class JsonValidationPipelineStage(BasePipelineStage):
""" """
Always raises an error. Validate returned data against a JSON schema.
""" """
#: Help string to be shown when a user is building a pipeline #: Help string to be shown when a user is building a pipeline
description = 'Raise an error' description = 'Validates data against a JSON schema'
def __init__(self, options: typing.Optional[typing.Mapping] = None): def __init__(self, options: typing.Optional[typing.Mapping] = None):
super().__init__(options) super().__init__(options)
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment