Skip to content
GitLab
Explore
Sign in
Register
Primary navigation
Search or go to…
Project
P
PEDASI
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Deploy
Releases
Model registry
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
GitLab community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
Research Software Group
PEDASI
Commits
b6d78946
Commit
b6d78946
authored
6 years ago
by
James Graham
Browse files
Options
Downloads
Patches
Plain Diff
Refactor UnmanagedSqlConnector to use dummy http response intermediate
parent
e57adf03
No related branches found
No related tags found
No related merge requests found
Changes
3
Show whitespace changes
Inline
Side-by-side
Showing
3 changed files
datasources/connectors/base.py
+21
-1
21 additions, 1 deletion
datasources/connectors/base.py
datasources/connectors/csv.py
+2
-14
2 additions, 14 deletions
datasources/connectors/csv.py
datasources/connectors/sql.py
+8
-5
8 additions, 5 deletions
datasources/connectors/sql.py
with
31 additions
and
20 deletions
datasources/connectors/base.py
+
21
−
1
View file @
b6d78946
...
...
@@ -8,6 +8,7 @@ import abc
from
collections
import
abc
as
collections_abc
from
collections
import
OrderedDict
import
enum
import
json
import
typing
import
requests
...
...
@@ -49,6 +50,22 @@ REQUEST_AUTH_FUNCTIONS = OrderedDict([
])
class
DummyRequestsResponse
:
def
__init__
(
self
,
text
:
str
,
status_code
:
int
,
content_type
:
str
):
self
.
text
=
text
self
.
status_code
=
status_code
self
.
headers
=
{
'
content-type
'
:
content_type
,
'
Content-Type
'
:
content_type
,
}
def
json
(
self
):
return
json
.
loads
(
self
.
text
)
class
BaseDataConnector
(
metaclass
=
plugin
.
Plugin
):
"""
Base class of data connectors which provide access to data / metadata via an external API.
...
...
@@ -178,7 +195,10 @@ class DataSetConnector(BaseDataConnector):
"""
response
=
self
.
get_response
(
params
)
if
'
json
'
in
response
.
headers
[
'
Content-Type
'
]
:
try
:
return
response
.
json
()
except
(
AttributeError
,
json
.
decoder
.
JSONDecodeError
):
pass
return
response
.
text
This diff is collapsed.
Click to expand it.
datasources/connectors/csv.py
+
2
−
14
View file @
b6d78946
import
typing
from
.base
import
DataSetConnector
class
DummyResponse
:
def
__init__
(
self
,
text
:
str
,
status_code
:
int
,
content_type
:
str
):
self
.
text
=
text
self
.
status_code
=
status_code
self
.
headers
=
{
'
content-type
'
:
content_type
,
}
from
.base
import
DataSetConnector
,
DummyRequestsResponse
class
CsvConnector
(
DataSetConnector
):
...
...
@@ -28,4 +16,4 @@ class CsvConnector(DataSetConnector):
:return: Requested data
"""
with
open
(
self
.
location
,
'
r
'
)
as
f
:
return
DummyResponse
(
f
.
read
(),
200
,
'
text/plain
'
)
return
DummyRe
questsRe
sponse
(
f
.
read
(),
200
,
'
text/plain
'
)
This diff is collapsed.
Click to expand it.
datasources/connectors/sql.py
+
8
−
5
View file @
b6d78946
...
...
@@ -4,12 +4,13 @@ This module contains data connectors for SQL tables.
These tables may be external or automatically manage by PEDASI.
"""
import
json
import
typing
import
sqlalchemy
import
sqlalchemy.orm
from
.base
import
DataSetConnector
from
.base
import
DataSetConnector
,
DummyRequestsResponse
# TODO protect Django SQL database with proper password - this connector allows it to be retrieved
...
...
@@ -32,9 +33,8 @@ class UnmanagedSqlConnector(DataSetConnector):
self
.
_table_meta
=
sqlalchemy
.
MetaData
(
self
.
_engine
)
self
.
_table
=
sqlalchemy
.
Table
(
table
,
self
.
_table_meta
,
autoload
=
True
)
def
get_
data
(
self
,
def
get_
response
(
self
,
params
:
typing
.
Optional
[
typing
.
Mapping
[
str
,
str
]]
=
None
):
# TODO select where params
"""
Select and return all contents of table.
"""
...
...
@@ -54,7 +54,10 @@ class UnmanagedSqlConnector(DataSetConnector):
result
=
session
.
execute
(
query
)
return
[
dict
(
row
)
for
row
in
result
]
return
DummyRequestsResponse
(
json
.
dumps
([
dict
(
row
)
for
row
in
result
],
default
=
str
),
200
,
content_type
=
'
application/json
'
)
def
get_metadata
(
self
,
params
:
typing
.
Optional
[
typing
.
Mapping
[
str
,
str
]]
=
None
):
...
...
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment