Skip to content
GitLab
Explore
Sign in
Register
Primary navigation
Search or go to…
Project
P
PEDASI
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Deploy
Releases
Model registry
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
Research Software Group
PEDASI
Commits
c042ba85
Commit
c042ba85
authored
6 years ago
by
James Graham
Browse files
Options
Downloads
Patches
Plain Diff
Give metadata fields human names and add docstrings
parent
0501dc0c
No related branches found
No related tags found
No related merge requests found
Changes
3
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
datasources/models/metadata.py
+13
-7
13 additions, 7 deletions
datasources/models/metadata.py
datasources/models/pipeline.py
+7
-2
7 additions, 2 deletions
datasources/models/pipeline.py
datasources/pipeline/base.py
+11
-2
11 additions, 2 deletions
datasources/pipeline/base.py
with
31 additions
and
11 deletions
datasources/models/metadata.py
+
13
−
7
View file @
c042ba85
...
@@ -54,16 +54,22 @@ class MetadataField(models.Model):
...
@@ -54,16 +54,22 @@ class MetadataField(models.Model):
This is called from within the AppConfig.
This is called from within the AppConfig.
"""
"""
fixtures
=
(
fixtures
=
(
(
'
data_q
uery
_p
aram
'
,
'
data_query_param
'
,
True
),
(
'
Accepted Q
uery
P
aram
eter
'
,
'
data_query_param
'
,
True
),
(
'
i
ndexed
_f
ield
'
,
'
indexed_field
'
,
True
),
(
'
I
ndexed
F
ield
'
,
'
indexed_field
'
,
True
),
(
'
s
chema
'
,
'
schema
'
,
True
),
(
'
Validation S
chema
'
,
'
schema
'
,
True
),
)
)
for
name
,
short_name
,
operational
in
fixtures
:
for
name
,
short_name
,
operational
in
fixtures
:
obj
,
created
=
cls
.
objects
.
get_or_create
(
try
:
name
=
name
,
obj
=
cls
.
objects
.
get
(
short_name
=
short_name
)
short_name
=
short_name
)
except
cls
.
DoesNotExist
:
obj
=
cls
.
objects
.
create
(
name
=
name
,
short_name
=
short_name
)
obj
.
name
=
name
obj
.
operational
=
operational
obj
.
operational
=
operational
obj
.
save
()
obj
.
save
()
...
...
This diff is collapsed.
Click to expand it.
datasources/models/pipeline.py
+
7
−
2
View file @
c042ba85
...
@@ -20,9 +20,11 @@ class PipelineSetupError(BaseException):
...
@@ -20,9 +20,11 @@ class PipelineSetupError(BaseException):
class
Pipeline
(
models
.
Model
):
class
Pipeline
(
models
.
Model
):
class
Meta
:
"""
pass
Model representing a data pipeline.
A pipeline may contain multiple stages.
"""
# Prevent template engine from trying to call the model
# Prevent template engine from trying to call the model
do_not_call_in_templates
=
True
do_not_call_in_templates
=
True
...
@@ -57,6 +59,9 @@ class Pipeline(models.Model):
...
@@ -57,6 +59,9 @@ class Pipeline(models.Model):
class
PipelineStage
(
models
.
Model
):
class
PipelineStage
(
models
.
Model
):
"""
Model representing a stage of a data pipeline.
"""
class
Meta
:
class
Meta
:
order_with_respect_to
=
'
pipeline
'
order_with_respect_to
=
'
pipeline
'
...
...
This diff is collapsed.
Click to expand it.
datasources/pipeline/base.py
+
11
−
2
View file @
c042ba85
...
@@ -13,6 +13,9 @@ from .. import models
...
@@ -13,6 +13,9 @@ from .. import models
class
BasePipelineStage
(
metaclass
=
plugin
.
Plugin
):
class
BasePipelineStage
(
metaclass
=
plugin
.
Plugin
):
"""
Base class for pipeline stages. Allows the defined classes to be used as plugins.
"""
#: Help string to be shown when a user is building a pipeline
#: Help string to be shown when a user is building a pipeline
description
=
None
description
=
None
...
@@ -21,6 +24,12 @@ class BasePipelineStage(metaclass=plugin.Plugin):
...
@@ -21,6 +24,12 @@ class BasePipelineStage(metaclass=plugin.Plugin):
@abc.abstractmethod
@abc.abstractmethod
def
__call__
(
self
,
data
:
typing
.
Mapping
)
->
typing
.
Mapping
:
def
__call__
(
self
,
data
:
typing
.
Mapping
)
->
typing
.
Mapping
:
"""
Pass data through the pipeline stage.
:param data: Data to process
:return: Processed data
"""
raise
NotImplementedError
raise
NotImplementedError
...
@@ -37,10 +46,10 @@ class NullPipelineStage(BasePipelineStage):
...
@@ -37,10 +46,10 @@ class NullPipelineStage(BasePipelineStage):
class
JsonValidationPipelineStage
(
BasePipelineStage
):
class
JsonValidationPipelineStage
(
BasePipelineStage
):
"""
"""
Always raises an error
.
Validate returned data against a JSON schema
.
"""
"""
#: Help string to be shown when a user is building a pipeline
#: Help string to be shown when a user is building a pipeline
description
=
'
Raise an error
'
description
=
'
Validates data against a JSON schema
'
def
__init__
(
self
,
options
:
typing
.
Optional
[
typing
.
Mapping
]
=
None
):
def
__init__
(
self
,
options
:
typing
.
Optional
[
typing
.
Mapping
]
=
None
):
super
().
__init__
(
options
)
super
().
__init__
(
options
)
...
...
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment