diff --git a/datasources/models/metadata.py b/datasources/models/metadata.py index 156ba5cbd56737538e00bf4c5583e149aafa3b07..90da0ea692985a74dddd96112aa4832020b229e7 100644 --- a/datasources/models/metadata.py +++ b/datasources/models/metadata.py @@ -54,16 +54,22 @@ class MetadataField(models.Model): This is called from within the AppConfig. """ fixtures = ( - ('data_query_param', 'data_query_param', True), - ('indexed_field', 'indexed_field', True), - ('schema', 'schema', True), + ('Accepted Query Parameter', 'data_query_param', True), + ('Indexed Field', 'indexed_field', True), + ('Validation Schema', 'schema', True), ) for name, short_name, operational in fixtures: - obj, created = cls.objects.get_or_create( - name=name, - short_name=short_name - ) + try: + obj = cls.objects.get(short_name=short_name) + + except cls.DoesNotExist: + obj = cls.objects.create( + name=name, + short_name=short_name + ) + + obj.name = name obj.operational = operational obj.save() diff --git a/datasources/models/pipeline.py b/datasources/models/pipeline.py index fa9c23c81b88e30dcd47acd8576086aabc83d608..6076e010e1cc1c6136a356f2a32955bc23b65dde 100644 --- a/datasources/models/pipeline.py +++ b/datasources/models/pipeline.py @@ -20,9 +20,11 @@ class PipelineSetupError(BaseException): class Pipeline(models.Model): - class Meta: - pass + """ + Model representing a data pipeline. + A pipeline may contain multiple stages. + """ # Prevent template engine from trying to call the model do_not_call_in_templates = True @@ -57,6 +59,9 @@ class Pipeline(models.Model): class PipelineStage(models.Model): + """ + Model representing a stage of a data pipeline. + """ class Meta: order_with_respect_to = 'pipeline' diff --git a/datasources/pipeline/base.py b/datasources/pipeline/base.py index 205d8856cf017ceb74f0b7a7dd6140a9df4720b6..1a53d2b54d740bef26c4f7ea1525467a6a5f935f 100644 --- a/datasources/pipeline/base.py +++ b/datasources/pipeline/base.py @@ -13,6 +13,9 @@ from .. import models class BasePipelineStage(metaclass=plugin.Plugin): + """ + Base class for pipeline stages. Allows the defined classes to be used as plugins. + """ #: Help string to be shown when a user is building a pipeline description = None @@ -21,6 +24,12 @@ class BasePipelineStage(metaclass=plugin.Plugin): @abc.abstractmethod def __call__(self, data: typing.Mapping) -> typing.Mapping: + """ + Pass data through the pipeline stage. + + :param data: Data to process + :return: Processed data + """ raise NotImplementedError @@ -37,10 +46,10 @@ class NullPipelineStage(BasePipelineStage): class JsonValidationPipelineStage(BasePipelineStage): """ - Always raises an error. + Validate returned data against a JSON schema. """ #: Help string to be shown when a user is building a pipeline - description = 'Raise an error' + description = 'Validates data against a JSON schema' def __init__(self, options: typing.Optional[typing.Mapping] = None): super().__init__(options)