Multipart

.gitignore

@@ -3,6 +3,7 @@ pom.xml*
 /lib/
 /classes/
 /target
+modules/*/target
 .lein-failures
 .lein-deps-sum
 .lein-repl-history

modules/muuntaja-multipart/project.clj

@@ -0,0 +1,11 @@
+(defproject metosin/muuntaja-multipart "0.6.0"
+  :description "Multipart format for Muuntaja"
+  :url "https://github.com/metosin/muuntaja"
+  :license {:name "Eclipse Public License"
+            :url "http://www.eclipse.org/legal/epl-v10.html"}
+  :plugins [[lein-parent "0.3.2"]]
+  :parent-project {:path "../../project.clj"
+                   :inherit [:deploy-repositories :managed-dependencies]}
+  :dependencies [[metosin/muuntaja]
+                 [org.synchronoss.cloud/nio-multipart-parser]])
+

modules/muuntaja-multipart/src/muuntaja/format/multipart.clj

@@ -0,0 +1,97 @@
+(ns muuntaja.format.multipart
+  (:refer-clojure :exclude [format])
+  (:require [muuntaja.format.core :as core]
+            [clojure.java.io :as io])
+  (:import [java.io PipedInputStream PipedOutputStream]
+           [org.synchronoss.cloud.nio.multipart
+            Multipart MultipartContext MultipartUtils
+            NioMultipartParserListener DefaultPartBodyStreamStorageFactory
+            PartBodyStreamStorageFactory]
+           [org.synchronoss.cloud.nio.stream.storage
+            StreamStorage StreamStorageFactory]))
+
+(defn stream-storage []
+  (reify PartBodyStreamStorageFactory
+    (newStreamStorageForPartBody [this headers partIndex]
+      (let [is (PipedInputStream.)
+            os (PipedOutputStream.)]
+        (.connect is os)
+        (proxy [StreamStorage] []
+          (write
+            ([b]
+             (.write os b))
+            ([b off len]
+             (.write os b off len)))
+          (close []
+            (.close os))
+          (flush []
+            nil)
+          (getInputStream []
+            is))))))
+
+(defn decoder [{:keys [;; ring middleware options
+                       ;; FIXME: Do these make sense?
+                       store fallback-encoding encoding
+                       ;; nio-multipart options
+                       buffer-size headers-size-limit
+                       max-memory-usage-per-body-part
+                       limit-nesting-parts-to]
+                :as options}]
+  (reify
+    core/Decode
+    (decode [this data charset]
+      ;; FIXME: This needs the request map to get proper content-type and length
+      (let [store (or store identity)
+            context (MultipartContext. "multipart/form-data; boundary=XXXX" 10000 (or encoding
+                                                                                      charset
+                                                                                      fallback-encoding))
+            result (atom {})
+            listener (reify NioMultipartParserListener
+                       (onPartFinished [this partBodyStreamStorage, headersFromPart]
+                         (let [fieldName (MultipartUtils/getFieldName headersFromPart)]
+                           (let [data (if (MultipartUtils/isFormField headersFromPart context)
+                                        (slurp (.getInputStream partBodyStreamStorage) :encoding (or encoding
+                                                                                                     (MultipartUtils/getCharEncoding headersFromPart)
+                                                                                                     fallback-encoding))
+                                        ;; TODO: If content-type is known by Muuntaja, parse the data.
+                                        ;; FIXME: needs the muuntaja instance.
+                                        (store {:content-type (MultipartUtils/getContentType headersFromPart)
+                                                :filename (MultipartUtils/getFileName headersFromPart)
+                                                :stream (.getInputStream partBodyStreamStorage)}))]
+                             (swap! result update fieldName (fn [v]
+                                                              ;; If result already contains same key,
+                                                              ;; append to vec to vector or convert the value to vector.
+                                                              (if v
+                                                                (if (vector? v)
+                                                                  (conj v data)
+                                                                  [v data])
+                                                                data))))))
+                       (onNestedPartStarted [this headersFromParentPart]
+                         nil)
+                       (onNestedPartFinished [this]
+                         nil)
+                       (onAllPartsFinished [this]
+                         nil)
+                       (onError [this message cause]
+                         ;; Not sure if works, is this thrown in main thread?
+                         (throw (Exception. message cause))))
+            parser (-> (Multipart/multipart context)
+                       ;; Setup always our own piped-input-stream storage
+                       ;; Ring-style :store function can be used to convert input-stream to file.
+                       (.usePartBodyStreamStorageFactory (stream-storage))
+                       (doto (cond->
+                               buffer-size (.setBufferSize buffer-size)
+                               headers-size-limit (.setHeadersSizeLimit headers-size-limit)
+                               max-memory-usage-per-body-part (.setMaxMemoryUsagePerBodyPart max-memory-usage-per-body-part)
+                               limit-nesting-parts-to (.setLimitNestingPartsTo limit-nesting-parts-to)))
+                       (.forNIO listener))]
+        (try
+          (io/copy data parser)
+          (finally
+            (.close parser)))
+        @result))))
+
+(def format
+  (core/map->Format
+    {:name "multipart/form-data"
+     :decoder [decoder]}))

modules/muuntaja/src/muuntaja/core.clj

@@ -85,6 +85,7 @@
      (util/throw! m format "encoder not found for"))))
 
 (defn decode
+  ;; FIXME: data is input stream? Returns the decoded value.
   "Decode data into the given format. Returns InputStream or throws."
   ([m format data]
    (decode m format data (default-charset m)))

project.clj

@@ -11,7 +11,8 @@
                          [com.cognitect/transit-clj "0.8.313"]
                          [cheshire "5.8.0"]
                          [circleci/clj-yaml "0.5.6"]
-                         [clojure-msgpack "1.2.1" :exclusions [org.clojure/clojure]]]
+                         [clojure-msgpack "1.2.1" :exclusions [org.clojure/clojure]]
+                         [org.synchronoss.cloud/nio-multipart-parser "1.1.0"]]
   :dependencies []
   :plugins [[lein-codox "0.10.3"]]
   :codox {:src-uri "http://github.com/metosin/muuntaja/blob/master/{filepath}#L{line}"
@@ -26,7 +27,8 @@
                    :source-paths ["modules/muuntaja/src"
                                   "modules/muuntaja-cheshire/src"
                                   "modules/muuntaja-yaml/src"
-                                  "modules/muuntaja-msgpack/src"]
+                                  "modules/muuntaja-msgpack/src"
+                                  "modules/muuntaja-multipart/src"]
 
                    :dependencies [[org.clojure/clojure "1.9.0"]
                                   [ring/ring-core "1.6.3"]
@@ -39,6 +41,7 @@
                                   [metosin/muuntaja-cheshire "0.6.0"]
                                   [metosin/muuntaja-msgpack "0.6.0"]
                                   [metosin/muuntaja-yaml "0.6.0"]
+                                  [metosin/muuntaja-multipart "0.6.0"]
 
                                   ;; Sieppari
                                   [metosin/sieppari "0.0.0-alpha4"]

scripts/lein-modules

@@ -7,6 +7,7 @@ for ext in \
   muuntaja \
   muuntaja-cheshire \
   muuntaja-msgpack \
-  muuntaja-yaml; do
+  muuntaja-yaml \
+  muuntaja-multipart; do
   cd modules/$ext; lein "$@"; cd ../..;
 done

test/muuntaja/multipart_ring_test.clj

@@ -0,0 +1,220 @@
+(ns muuntaja.multipart-ring-test
+  "Ring multipart middleware test suite"
+  (:require [clojure.test :refer :all]
+            [muuntaja.core :as m]
+            [muuntaja.middleware :as middleware]
+            [muuntaja.format.multipart :as multipart]
+            [ring.util.io :refer [string-input-stream]]
+            [ring.middleware.multipart-params.byte-array :refer [byte-array-store]]))
+
+;; Mock Ring style middleware and request functions
+
+(defn multipart-request [req muuntaja-options]
+  (let [m (m/create muuntaja-options)
+        req (->> req
+                 (m/negotiate-request-response m)
+                 (m/format-request m))]
+    (-> req
+        (assoc :multipart-params (:body-params req))
+        (update :params merge (:body-params req)))))
+
+(def wrap-multipart-params
+  (fn
+    ([handler]
+     (wrap-multipart-params handler nil))
+    ([handler options]
+     (let [muuntaja-options (assoc-in m/default-options [:formats "multipart/form-data"] (assoc multipart/format :decoder-opts options))]
+       (-> handler
+           ((fn [handler]
+              (fn
+                ([req]
+                 (handler (multipart-request req muuntaja-options)))
+                ([req respond raise]
+                 (handler (multipart-request req muuntaja-options) respond raise)))))
+           (middleware/wrap-format-request muuntaja-options)
+           (middleware/wrap-format-negotiate muuntaja-options))))))
+
+(defn multipart-params-request
+  ([req]
+   (multipart-params-request req nil))
+  ([req options]
+   ((wrap-multipart-params identity options) req)))
+
+;; Ring ring.middleware.test.multipart-params
+
+(defn string-store [item]
+  (-> (select-keys item [:filename :content-type])
+      (assoc :content (slurp (:stream item)))))
+
+(deftest test-wrap-multipart-params
+  (let [form-body (str "--XXXX\r\n"
+                       "Content-Disposition: form-data;"
+                       "name=\"upload\"; filename=\"test.txt\"\r\n"
+                       "Content-Type: text/plain\r\n\r\n"
+                       "foo\r\n"
+                       "--XXXX\r\n"
+                       "Content-Disposition: form-data;"
+                       "name=\"baz\"\r\n\r\n"
+                       "qux\r\n"
+                       "--XXXX--")
+        handler (wrap-multipart-params identity {:store string-store})
+        request {:headers {"content-type" "multipart/form-data; boundary=XXXX"
+                           "content-length" (str (count form-body))}
+                 :params {"foo" "bar"}
+                 :body (string-input-stream form-body)}
+        response (handler request)]
+
+    (is (= (get-in response [:params "foo"]) "bar"))
+    (is (= (get-in response [:params "baz"]) "qux"))
+    (let [upload (get-in response [:params "upload"])]
+      (is (= (:filename upload)     "test.txt"))
+      (is (= (:content-type upload) "text/plain"))
+      (is (= (:content upload)      "foo")))))
+
+(deftest test-multiple-params
+  (let [form-body (str "--XXXX\r\n"
+                       "Content-Disposition: form-data;"
+                       "name=\"foo\"\r\n\r\n"
+                       "bar\r\n"
+                       "--XXXX\r\n"
+                       "Content-Disposition: form-data;"
+                       "name=\"foo\"\r\n\r\n"
+                       "baz\r\n"
+                       "--XXXX--")
+        handler (wrap-multipart-params identity {:store string-store})
+        request {:headers {"content-type" "multipart/form-data; boundary=XXXX"
+                           "content-length" (str (count form-body))}
+                 :body (string-input-stream form-body)}
+        response (handler request)]
+    (is (= (get-in response [:params "foo"])
+           ["bar" "baz"]))))
+
+(defn all-threads []
+  (.keySet (Thread/getAllStackTraces)))
+
+(deftest test-multipart-threads
+  (testing "no thread leakage when handler called"
+    (let [handler (wrap-multipart-params identity)]
+      (dotimes [_ 200]
+        (handler {}))
+      (is (< (count (all-threads))
+             100))))
+
+  (testing "no thread leakage from default store"
+    (let [form-body (str "--XXXX\r\n"
+                         "Content-Disposition: form-data;"
+                         "name=\"upload\"; filename=\"test.txt\"\r\n"
+                         "Content-Type: text/plain\r\n\r\n"
+                         "foo\r\n"
+                         "--XXXX--")]
+      (dotimes [_ 200]
+        (let [handler (wrap-multipart-params identity)
+              request {:headers {"content-type" "multipart/form-data; boundary=XXXX"
+                                 "content-length" (str (count form-body))}
+                       :body (string-input-stream form-body)}]
+          (handler request))))
+    (is (< (count (all-threads))
+           100))))
+
+(deftest wrap-multipart-params-cps-test
+  (let [handler   (wrap-multipart-params (fn [req respond _] (respond req)))
+        form-body (str "--XXXX\r\n"
+                       "Content-Disposition: form-data;"
+                       "name=\"foo\"\r\n\r\n"
+                       "bar\r\n"
+                       "--XXXX--")
+        request   {:headers {"content-type" "multipart/form-data; boundary=XXXX"}
+                   :body    (string-input-stream form-body "UTF-8")}
+        response  (promise)
+        exception (promise)]
+        (handler request response exception)
+        (is (= (get-in @response [:multipart-params "foo"]) "bar"))
+        (is (not (realized? exception)))))
+
+(deftest multipart-params-request-test
+  (is (fn? multipart-params-request)))
+
+(deftest decode-with-utf8-by-default
+  (let [form-body (str "--XXXX\r\n"
+                       "Content-Disposition: form-data;"
+                       "name=\"foo\"\r\n\r\n"
+                       "Øæß箣èé\r\n"
+                       "--XXXX--")
+        request {:headers {"content-type"
+                           (str "multipart/form-data; boundary=XXXX")}
+                 :body (string-input-stream form-body "UTF-8")}
+        request* (multipart-params-request request)]
+        (is (= (get-in request* [:multipart-params "foo"]) "Øæß箣èé"))))
+
+(deftest parts-may-have-invidual-charsets-in-content-type
+  (let [form-body (str "--XXXX\r\n"
+                       "Content-Disposition: form-data;"
+                       "name=\"foo\"\r\n"
+                       "Content-Type: text/plain; charset=ISO-8859-15\r\n\r\n"
+                       "äÄÖöÅå€\r\n"
+                       "--XXXX--")
+        request {:headers {"content-type"
+                           (str "multipart/form-data; boundary=XXXX")}
+                 :body (string-input-stream form-body "ISO-8859-15")}
+        request* (multipart-params-request request)]
+    (println request*)
+    (is (= (get-in request* [:multipart-params "foo"]) "äÄÖöÅå€"))))
+
+;; FIXME: this has not been implemented but utf8 is default or something, so this works accidentally.
+(deftest charset-may-be-defined-html5-style-parameter
+  (let [form-body (str "--XXXX\r\n"
+                       "Content-Disposition: form-data;"
+                       "name=\"foo\"\r\n\r\n"
+                       "Øæß箣èé\r\n"
+                       "--XXXX\r\n"
+                       "Content-Disposition: form-data;"
+                       "name=\"_charset_\"\r\n\r\n"
+                       "UTF-8\r\n"
+                       "--XXXX--")
+        request {:headers {"content-type"
+                           (str "multipart/form-data; boundary=XXXX; charset=US-ASCII")}
+                 :body (string-input-stream form-body "UTF-8")}
+        request* (multipart-params-request request)]
+    (is (= (get-in request* [:multipart-params "foo"]) "Øæß箣èé"))))
+
+(deftest form-field-recognition-test
+  (let [form-body (str "--XXXX\r\n"
+                       "Content-Disposition: form-data;"
+                       "name=\"upload\"; filename=\"test.txt\"\r\n"
+                       "Content-Type: text/plain\r\n\r\n"
+                       "foo\r\n"
+                       "--XXXX--")
+        handler (wrap-multipart-params identity {:store (byte-array-store)})
+        request {:headers {"content-type" "multipart/form-data; boundary=XXXX"
+                           "content-length" (str (count form-body))}
+                 :body (string-input-stream form-body)}
+        response (handler request)]
+    (let [upload (get-in response [:multipart-params "upload"])]
+      (is (java.util.Arrays/equals (:bytes upload) (.getBytes "foo"))))))
+
+(deftest forced-encoding-option-works
+  (let [form-body (str "--XXXX\r\n"
+                       "Content-Disposition: form-data;"
+                       "name=\"foo\"\r\n"
+                       ;; application/json field is not considered form-value so it would be handled as file.
+                       ; "Content-Type: application/json; charset=UTF-8\r\n\r\n"
+                       "Content-Type: text/plain; charset=UTF-8\r\n\r\n"
+                       "{\"åå\":\"ÄÖ\"}\r\n"
+                       "--XXXX--")
+        request {:headers {"content-type"
+                           (str "multipart/form-data; boundary=XXXX; charset=US-ASCII")}
+                 :body (string-input-stream form-body "ISO-8859-1")}
+        request* (multipart-params-request request {:encoding "ISO-8859-1"})]
+    (is (= (get-in request* [:multipart-params "foo"]) "{\"åå\":\"ÄÖ\"}"))))
+
+(deftest fallback-encoding-option-works
+  (let [form-body (str "--XXXX\r\n"
+                       "Content-Disposition: form-data;"
+                       "name=\"foo\"\r\n\r\n"
+                       "äÄÖöÅå€\r\n"
+                       "--XXXX--")
+        request {:headers {"content-type"
+                           (str "multipart/form-data; boundary=XXXX; charset=US-ASCII")}
+                 :body (string-input-stream form-body "ISO-8859-15")}
+        request* (multipart-params-request request {:fallback-encoding "ISO-8859-15"})]
+    (is (= (get-in request* [:multipart-params "foo"]) "äÄÖöÅå€"))))

test/muuntaja/multipart_test.clj

@@ -0,0 +1,12 @@
+(ns muuntaja.multipart-test
+  (:require [clojure.test :refer :all]
+            [muuntaja.core :as m]
+            [muuntaja.middleware :as middleware]
+            [muuntaja.format.multipart :as multipart]
+            [ring.util.io :refer [string-input-stream]]))
+
+;; Add out own tests:
+;; multipart/mixed
+;; multipart inside multipart
+;; decode json etc. inside multipart
+;; images etc. shouldn't be decoded