WIP Adding file/bytes parameter API support.

brew_view/__init__.py

@@ -249,6 +249,8 @@ def _setup_tornado_app():
         CommandAPI,
         CommandListAPI,
         ConfigHandler,
+        FileListAPI,
+        FileAPI,
         InstanceAPI,
         QueueAPI,
         QueueListAPI,
@@ -303,6 +305,8 @@ def _setup_tornado_app():
         (r"{0}api/v1/config/logging/?".format(prefix), LoggingConfigAPI),
         # Beta
         (r"{0}api/vbeta/events/?".format(prefix), EventPublisherAPI),
+        (r"{0}api/vbeta/files/?".format(prefix), FileListAPI),
+        (r"{0}api/vbeta/files/(\w+)/?".format(prefix), FileAPI),
         # Deprecated
         (r"{0}api/v1/admin/system/?".format(prefix), OldAdminAPI),
         (r"{0}api/v1/admin/queues/?".format(prefix), OldQueueListAPI),

brew_view/authorization.py

@@ -23,6 +23,10 @@ class Permissions(Enum):
     EVENT_READ = "bg-event-read"
     EVENT_UPDATE = "bg-event-update"
     EVENT_DELETE = "bg-event-delete"
+    FILE_CREATE = "bg-file-create"
+    FILE_READ = "bg-file-read"
+    FILE_UPDATE = "bg-file-update"
+    FILE_DELETE = "bg-file-delete"
     INSTANCE_CREATE = "bg-instance-create"
     INSTANCE_READ = "bg-instance-read"
     INSTANCE_UPDATE = "bg-instance-update"

brew_view/base_handler.py

@@ -58,6 +58,13 @@ class BaseHandler(AuthMixin, RequestHandler):
         socket.timeout: {"status_code": 504, "message": "Backend request timed out"},
     }
 
+    def get_bool_header(self, key, default_value):
+        value = self.request.headers.get(key)
+        if value is None:
+            return default_value
+
+        return str(value).lower() in ["1", "true", "t", "yes", "y"]
+
     def get_refresh_id_from_cookie(self):
         token_id = self.get_secure_cookie(self.REFRESH_COOKIE_NAME)
         if token_id:
@@ -125,21 +132,27 @@ def prepare(self):
             content_type = content_type.split(";")
 
             self.request.mime_type = content_type[0]
-            if self.request.mime_type not in [
+            if self.request.mime_type in [
                 "application/json",
                 "application/x-www-form-urlencoded",
             ]:
+                # Attempt to parse out the charset and decode the body, default to utf-8
+                charset = "utf-8"
+                if len(content_type) > 1:
+                    search_result = self.charset_re.search(content_type[1])
+                    if search_result:
+                        charset = search_result.group(1)
+                self.request.charset = charset
+                self.request.decoded_body = self.request.body.decode(charset)
+            elif (
+                self.request.mime_type == "multipart/form-data"
+                and "files" in self.request.path
+            ):
+                # No need to decode the request body, we're just reading bytes
+                pass
+            else:
                 raise ModelValidationError("Unsupported or missing content-type header")
 
-            # Attempt to parse out the charset and decode the body, default to utf-8
-            charset = "utf-8"
-            if len(content_type) > 1:
-                search_result = self.charset_re.search(content_type[1])
-                if search_result:
-                    charset = search_result.group(1)
-            self.request.charset = charset
-            self.request.decoded_body = self.request.body.decode(charset)
-
     def on_finish(self):
         """Called after a handler completes processing"""
         # This is gross, but in some cases we have to do these in the handler

brew_view/controllers/__init__.py

@@ -24,3 +24,4 @@
 from brew_view.controllers.system_list_api import SystemListAPI
 from brew_view.controllers.token_api import TokenAPI, TokenListAPI
 from brew_view.controllers.users_api import UserAPI, UsersAPI
+from brew_view.controllers.file_api import FileListAPI, FileAPI

brew_view/controllers/file_api.py

@@ -0,0 +1,125 @@
+import logging
+
+from tornado.gen import coroutine
+
+from brew_view import file_writers
+from brew_view import file_readers
+from bg_utils.mongo.parser import MongoParser
+from brew_view.authorization import authenticated, Permissions
+from brew_view.base_handler import BaseHandler
+from brewtils.models import Events
+
+
+class FileListAPI(BaseHandler):
+
+    parser = MongoParser()
+    logger = logging.getLogger(__name__)
+
+    @coroutine
+    @authenticated(permissions=[Permissions.FILE_CREATE])
+    def post(self):
+        """
+        ---
+        summary: Create a new file
+        description: |
+            Creates a new file based on the default driver. You can specify
+            a different driver with a header.
+        consumes:
+          - multipart/form-data
+        parameters:
+          - name: file
+            in: formData
+            type: file
+            description: The file to upload.
+        responses:
+          201:
+            description: A new file has been created
+          400:
+            $ref: '#/definitions/400Error'
+          50x:
+            $ref: '#/definitions/50xError'
+        tags:
+          - Beta
+        """
+        file_writer = self._get_file_writer()
+
+        response = {}
+        for key in self.request.files:
+            self.request.event.name = Events.FILE_CREATED.name
+            fileinfo = self.request.files[key][0]
+            filename = fileinfo["filename"]
+            body = fileinfo["body"]
+            file_id = file_writer.write(body, filename=filename)
+            response[key] = {
+                "id": file_id,
+                "storage_type": "gridfs",
+                "filename": filename,
+            }
+
+        self.set_status(201)
+        self.write(response)
+
+    def _get_file_writer(self):
+        storage_type = self.request.headers.get("X-BG-Storage-Type", "gridfs")
+        return file_writers.get(storage_type)
+
+
+class FileAPI(BaseHandler):
+
+    parser = MongoParser()
+    logger = logging.getLogger(__name__)
+
+    @authenticated(permissions=[Permissions.FILE_READ])
+    def get(self, file_id):
+        """
+        ---
+        summary: Retrieve a specific file, or its metadata
+        description: |
+            If your storage_type is anything other than gridfs, you probably
+            only want to use this endpoint to retrieve metadata, however if
+            your storage type is gridfs, then you want to retrieve the actual
+            file.
+        parameters:
+          - name: file_id
+            in: path
+            required: true
+            description: The ID of the file
+            type: string
+          - name: X-BG-file-meta-only
+            in: header
+            required: false
+            description: If true, will only return JSON metadata about file
+            type: string
+        produces:
+          - application/json
+          - application/octet-stream
+        responses:
+          200:
+            description: Will return JSON if X-BG-file-meta-only is true,
+              otherwise, it will return the actual file.
+          404:
+            $ref: '#/definitions/404Error'
+          50x:
+            $ref: '#/definitions/50xError'
+        tags:
+          - Beta
+        """
+        meta_only = self.get_bool_header("X-BG-file-meta-only", False)
+        request_file, body = file_readers.get("gridfs").read(
+            file_id, meta_only=meta_only
+        )
+        if meta_only:
+            self.write(
+                self.parser.serialize_request_file(request_file, to_string=False)
+            )
+        else:
+            size = 4096
+            self.set_header("Content-Type", "application/octet-stream")
+            self.set_header(
+                "Content-Disposition", "attachment; filename=%s" % request_file.filename
+            )
+            while True:
+                data = body.read(size)
+                if not data:
+                    break
+                self.write(data)

brew_view/file_readers/__init__.py

@@ -0,0 +1,10 @@
+from brew_view.file_readers import gridfs
+
+
+def get(name):
+    """Get a file_reader"""
+    fname = str(name).lower()
+    if fname == "gridfs":
+        return gridfs
+    else:
+        raise TypeError("Invalid reader type: %s" % name)

brew_view/file_readers/gridfs.py

@@ -0,0 +1,10 @@
+from bg_utils.mongo.models import RequestFile
+
+
+def read(file_id, **kwargs):
+    """Read a file from gridfs."""
+    r_file = RequestFile.objects.get(id=file_id)
+    if kwargs.get("meta_only", False):
+        return r_file, None
+
+    return r_file, r_file.body

brew_view/file_writers/__init__.py

@@ -0,0 +1,10 @@
+from brew_view.file_writers import gridfs
+
+
+def get(name):
+    """Get a file writer"""
+    fname = str(name).lower()
+    if fname == "gridfs":
+        return gridfs
+    else:
+        raise TypeError("Invalid writer type: %s" % name)

brew_view/file_writers/gridfs.py

@@ -0,0 +1,28 @@
+from bg_utils.mongo.models import RequestFile
+
+
+def write(body, **kwargs):
+    """Write the given body to gridfs.
+
+    Args:
+        body: Bytes
+
+    Keyword Args:
+        filename: required, name of the file.
+        encoding: default is None
+
+    Returns:
+        The ID of the RequestFile that was written to the database.
+
+    """
+    filename = kwargs.get("filename")
+    if not filename:
+        raise ValueError("Cannot write a gridfs document without a filename.")
+
+    encoding = kwargs.get("encoding")
+    r_file = RequestFile(filename=filename)
+    r_file.body.new_file(encoding=encoding)
+    r_file.body.write(body)
+    r_file.body.close()
+    r_file.save()
+    return str(r_file.id)

test/conftest.py

@@ -69,6 +69,11 @@ def thrift_context(thrift_client):
     )
 
 
+@pytest.fixture
+def mongo_request_file(bg_request_file):
+    return brew2mongo(bg_request_file)
+
+
 @pytest.fixture
 def mongo_system(bg_system):
     return brew2mongo(bg_system)

test/unit/controllers/file_api_test.py

@@ -0,0 +1,95 @@
+from uuid import uuid4
+
+import pytest
+import json
+from functools import partial
+from tornado import gen
+
+from bg_utils.mongo.models import RequestFile
+
+
+@pytest.fixture(autouse=True)
+def drop_files(app):
+    RequestFile.drop_collection()
+
+
+class TestFileAPI(object):
+    @pytest.mark.gen_test
+    def test_get_meta_only(
+        self, http_client, base_url, request_file_dict, mongo_request_file
+    ):
+        mongo_request_file.save()
+
+        response = yield http_client.fetch(
+            base_url + "/api/vbeta/files/" + str(mongo_request_file.id),
+            headers={"X-BG-file-meta-only": "true"},
+        )
+        assert 200 == response.code
+        assert request_file_dict == json.loads(response.body.decode("utf-8"))
+
+    @pytest.mark.gen_test
+    def test_get_file(self, http_client, base_url, mongo_request_file):
+        mongo_request_file.save()
+        response = yield http_client.fetch(
+            base_url + "/api/vbeta/files/" + str(mongo_request_file.id)
+        )
+        assert 200 == response.code
+        assert "Content-Type" in response.headers
+        assert "Content-Disposition" in response.headers
+        assert (
+            response.headers["Content-Disposition"]
+            == "attachment; filename=%s" % mongo_request_file.filename
+        )
+        assert response.body == b""
+
+    @pytest.mark.gen_test
+    def test_get_404(self, http_client, base_url, system_id):
+        response = yield http_client.fetch(
+            base_url + "/api/vbeta/files/" + system_id, raise_error=False
+        )
+        assert 404 == response.code
+
+    @pytest.fixture
+    def boundary(self):
+        return uuid4().hex
+
+    @staticmethod
+    @gen.coroutine
+    def multipart_producer(boundary, filename, body, write):
+        boundary_bytes = boundary.encode()
+        filename_bytes = filename.encode()
+        mtype = "application/octet-stream"
+        buf = (
+            (b"--%s\r\n" % boundary_bytes)
+            + (
+                b'Content-Disposition: form-data; name="%s"; filename="%s"\r\n'
+                % (filename_bytes, filename_bytes)
+            )
+            + (b"Content-Type: %s\r\n" % mtype.encode())
+            + b"\r\n"
+        )
+        yield write(buf)
+        yield write(body)
+        yield write(b"\r\n")
+        yield write(b"--%s--\r\n" % boundary_bytes)
+
+    @pytest.mark.gen_test
+    @pytest.mark.skip
+    def test_post(self, http_client, base_url, boundary, request_file_dict):
+        """Skipping until mongomock supports gridfs"""
+        headers = {"Content-Type": "multipart/form-data; boundary=%s" % boundary}
+        producer = partial(
+            self.multipart_producer, boundary, request_file_dict["filename"], b"0x01"
+        )
+        response = yield http_client.fetch(
+            base_url + "/api/vbeta/files/",
+            method="POST",
+            headers=headers,
+            body_producer=producer,
+        )
+        assert 201 == response.code
+        json_response = json.loads(response.body.decode("utf-8"))
+        assert "id" in json_response
+        assert "storage_type" in json_response
+        assert "filename" in json_response
+        assert "content_type" in json_response