Allow storing outputs in database

.gitignore

@@ -16,3 +16,5 @@ docs/_build
 
 *.orig
 .coverage
+example.db
+*.cfg

default-sample.cfg

@@ -28,9 +28,10 @@ maxrequestsize=3mb
 url=http://localhost:5000/wps
 outputurl=http://localhost:5000/outputs/
 outputpath=outputs
-workdir=workdir
+workdir=/tmp
 maxprocesses=10
 parallelprocesses=2
+store_type=db
 
 [processing]
 mode=default
@@ -44,3 +45,15 @@ format=%(asctime)s] [%(levelname)s] file=%(pathname)s line=%(lineno)s module=%(m
 
 [grass]
 gisbase=/usr/local/grass-7.3.svn/
+
+
+[db]
+db_type=pg
+dbname=dbname
+user=username
+password=password
+host=localhost
+port=5432
+dblocation=/tmp/db.sqlite
+schema_name=test_schema
+

pywps/configuration.py

@@ -131,6 +131,8 @@ def load_configuration(cfgfiles=None):
     CONFIG.add_section('grass')
     CONFIG.set('grass', 'gisbase', '')
 
+    CONFIG.add_section('db')
+
     if not cfgfiles:
         cfgfiles = _get_default_config_files_location()
 

pywps/inout/formats/__init__.py

@@ -11,16 +11,29 @@
 # based on Web Processing Service Best Practices Discussion Paper, OGC 12-029
 # http://opengeospatial.org/standards/wps
 
+from enum import Enum
 from collections import namedtuple
 import mimetypes
 from pywps.validator.mode import MODE
 from pywps.validator.base import emptyvalidator
 
-
 _FORMATS = namedtuple('FORMATS', 'GEOJSON, JSON, SHP, GML, GEOTIFF, WCS,'
                                  'WCS100, WCS110, WCS20, WFS, WFS100,'
                                  'WFS110, WFS20, WMS, WMS130, WMS110,'
-                                 'WMS100, TEXT, NETCDF, LAZ, LAS')
+                                 'WMS100, TEXT, CSV, NETCDF, LAZ, LAS')
+
+
+# this should be Enum type (only compatible with Python 3)
+class DATA_TYPE(Enum):
+    VECTOR = 0
+    RASTER = 1
+    OTHER = 2
+
+    def is_valid_datatype(data_type):
+
+        known_values = [datatype for datatype in DATA_TYPE]
+        if data_type not in known_values:
+            raise Exception("Unknown data type")
 
 
 class Format(object):
@@ -39,20 +52,22 @@ class Format(object):
     def __init__(self, mime_type,
                  schema=None, encoding=None,
                  validate=emptyvalidator, mode=MODE.SIMPLE,
-                 extension=None):
+                 extension=None, data_type=None):
         """Constructor
         """
 
         self._mime_type = None
         self._encoding = None
         self._schema = None
         self._extension = None
+        self._data_type = None
 
         self.mime_type = mime_type
         self.encoding = encoding
         self.schema = schema
         self.validate = validate
         self.extension = extension
+        self.data_type = data_type
 
     @property
     def mime_type(self):
@@ -62,6 +77,20 @@ def mime_type(self):
 
         return self._mime_type
 
+    @property
+    def data_type(self):
+        """Get format data type
+        """
+
+        return self._data_type
+
+    @data_type.setter
+    def data_type(self, data_type):
+        """Set format encoding
+        """
+
+        self._data_type = data_type
+
     @mime_type.setter
     def mime_type(self, mime_type):
         """Set format mime type
@@ -143,7 +172,8 @@ def json(self):
             'mime_type': self.mime_type,
             'encoding': self.encoding,
             'schema': self.schema,
-            'extension': self.extension
+            'extension': self.extension,
+            'data_type': self.data_type
         }
 
     @json.setter
@@ -156,30 +186,32 @@ def json(self, jsonin):
         self.encoding = jsonin['encoding']
         self.schema = jsonin['schema']
         self.extension = jsonin['extension']
+        self.data_type = jsonin['data_type']
 
 
 FORMATS = _FORMATS(
-    Format('application/vnd.geo+json', extension='.geojson'),
-    Format('application/json', extension='.json'),
-    Format('application/x-zipped-shp', extension='.zip'),
-    Format('application/gml+xml', extension='.gml'),
-    Format('image/tiff; subtype=geotiff', extension='.tiff'),
-    Format('application/xogc-wcs', extension='.xml'),
-    Format('application/x-ogc-wcs; version=1.0.0', extension='.xml'),
-    Format('application/x-ogc-wcs; version=1.1.0', extension='.xml'),
-    Format('application/x-ogc-wcs; version=2.0', extension='.xml'),
-    Format('application/x-ogc-wfs', extension='.xml'),
-    Format('application/x-ogc-wfs; version=1.0.0', extension='.xml'),
-    Format('application/x-ogc-wfs; version=1.1.0', extension='.xml'),
-    Format('application/x-ogc-wfs; version=2.0', extension='.xml'),
-    Format('application/x-ogc-wms', extension='.xml'),
-    Format('application/x-ogc-wms; version=1.3.0', extension='.xml'),
-    Format('application/x-ogc-wms; version=1.1.0', extension='.xml'),
-    Format('application/x-ogc-wms; version=1.0.0', extension='.xml'),
-    Format('text/plain', extension='.txt'),
-    Format('application/x-netcdf', extension='.nc'),
-    Format('application/octet-stream', extension='.laz'),
-    Format('application/octet-stream', extension='.las'),
+    Format('application/vnd.geo+json', extension='.geojson', data_type=DATA_TYPE.VECTOR),
+    Format('application/json', extension='.json', data_type=DATA_TYPE.VECTOR),
+    Format('application/x-zipped-shp', extension='.zip', data_type=DATA_TYPE.VECTOR),
+    Format('application/gml+xml', extension='.gml', data_type=DATA_TYPE.VECTOR),
+    Format('image/tiff; subtype=geotiff', extension='.tiff', data_type=DATA_TYPE.RASTER),
+    Format('application/xogc-wcs', extension='.xml', data_type=DATA_TYPE.VECTOR),
+    Format('application/x-ogc-wcs; version=1.0.0', extension='.xml', data_type=DATA_TYPE.VECTOR),
+    Format('application/x-ogc-wcs; version=1.1.0', extension='.xml', data_type=DATA_TYPE.VECTOR),
+    Format('application/x-ogc-wcs; version=2.0', extension='.xml', data_type=DATA_TYPE.VECTOR),
+    Format('application/x-ogc-wfs', extension='.xml', data_type=DATA_TYPE.VECTOR),
+    Format('application/x-ogc-wfs; version=1.0.0', extension='.xml', data_type=DATA_TYPE.VECTOR),
+    Format('application/x-ogc-wfs; version=1.1.0', extension='.xml', data_type=DATA_TYPE.VECTOR),
+    Format('application/x-ogc-wfs; version=2.0', extension='.xml', data_type=DATA_TYPE.VECTOR),
+    Format('application/x-ogc-wms', extension='.xml', data_type=DATA_TYPE.VECTOR),
+    Format('application/x-ogc-wms; version=1.3.0', extension='.xml', data_type=DATA_TYPE.VECTOR),
+    Format('application/x-ogc-wms; version=1.1.0', extension='.xml', data_type=DATA_TYPE.VECTOR),
+    Format('application/x-ogc-wms; version=1.0.0', extension='.xml', data_type=DATA_TYPE.VECTOR),
+    Format('text/plain', extension='.txt', data_type=DATA_TYPE.OTHER),
+    Format('text/csv', extension='.csv', data_type=DATA_TYPE.OTHER),
+    Format('application/x-netcdf', extension='.nc', data_type=DATA_TYPE.VECTOR),
+    Format('application/octet-stream', extension='.laz', data_type=DATA_TYPE.VECTOR),
+    Format('application/octet-stream', extension='.las', data_type=DATA_TYPE.VECTOR),
 )
 
 

pywps/inout/outputs.py

@@ -9,7 +9,9 @@
 import lxml.etree as etree
 import six
 from pywps.inout import basic
-from pywps.inout.storage import FileStorage
+from pywps.inout.storage.file import FileStorage
+from pywps.inout.storage.db import DbStorage
+from pywps import configuration as config
 from pywps.validator.mode import MODE
 
 
@@ -99,9 +101,21 @@ def _json_reference(self, data):
         data["type"] = "reference"
 
         # get_url will create the file and return the url for it
-        self.storage = FileStorage()
         data["href"] = self.get_url()
 
+        store_type = config.get_config_value('server', 'store_type')
+        self.storage = None
+        if store_type == 'db':
+            db_storage_instance = DbStorage()
+            self.storage = db_storage_instance.get_db_type()
+        else:
+            self.storage = FileStorage()
+
+        #to be implemented:
+        #elif store_type == 's3' and \
+        #   config.get_config_value('s3', 'bucket_name'):
+        #   self.storage = S3Storage()
+
         if self.data_format:
             if self.data_format.mime_type:
                 data['mimetype'] = self.data_format.mime_type

pywps/inout/storage/__init__.py

@@ -0,0 +1,46 @@
+##################################################################
+# Copyright 2018 Open Source Geospatial Foundation and others    #
+# licensed under MIT, Please consult LICENSE.txt for details     #
+##################################################################
+
+from abc import ABCMeta, abstractmethod
+
+
+class STORE_TYPE:
+    PATH = 0
+    DB = 1
+
+
+class StorageAbstract(object):
+    """Data storage abstract class
+    """
+
+    __metaclass__ = ABCMeta
+
+    @abstractmethod
+    def store(self, output):
+        """
+        :param output: of type IOHandler
+        :returns: (type, store, url) where
+            type - is type of STORE_TYPE - number
+            store - string describing storage - file name, database connection
+            url - url, where the data can be downloaded
+        """
+        pass
+
+
+class DummyStorage(StorageAbstract):
+    """Dummy empty storage implementation, does nothing
+
+    Default instance, for non-reference output request
+
+    >>> store = DummyStorage()
+    >>> assert store.store
+    """
+
+    def __init__(self):
+        """
+        """
+
+    def store(self, output):
+        pass

pywps/inout/storage/db/__init__.py

@@ -0,0 +1,58 @@
+##################################################################
+# Copyright 2018 Open Source Geospatial Foundation and others    #
+# licensed under MIT, Please consult LICENSE.txt for details     #
+##################################################################
+
+import logging
+from abc import ABCMeta, abstractmethod
+from pywps import configuration as config
+from pywps.inout.formats import DATA_TYPE
+from pywps.exceptions import NoApplicableCode
+from .. import STORE_TYPE
+from .. import StorageAbstract
+import sqlalchemy
+
+
+LOGGER = logging.getLogger('PYWPS')
+
+
+class DbStorage(StorageAbstract):
+
+    def __init__(self):
+        # get db_type from configuration
+        try:
+            self.db_type = config.get_config_value('db', 'db_type').lower()
+        except KeyError:
+            raise Exception("Database type has not been specified")
+
+    @staticmethod
+    def get_db_type():
+        from . import sqlite
+        from . import pg
+        # create an instance of the appropriate class
+        db_type = config.get_config_value('db', 'db_type').lower()
+        if db_type == "pg":
+            storage = pg.PgStorage()
+        elif db_type == "sqlite":
+            storage = sqlite.SQLiteStorage()
+        else:
+            raise Exception("Unknown database type: '{}'".format(config.get_config_value('db', 'db_type').lower()))
+
+        return storage
+
+    def initdb(self):
+        pass
+
+    def store(self, output):
+        """ Creates reference that is returned to the client
+        """
+        pass
+
+    def store_vector_output(self, file_name, identifier):
+        pass
+
+    def store_raster_output(self, file_name, identifier):
+        pass
+
+    def store_other_output(self, file_name, identifier, uuid):
+        pass