Special characters in object name in list objects (#959)

minio · Dec 7, 2021 · 6acf1ce · 6acf1ce
1 parent ba3bfd9
commit 6acf1ce
Show file tree

Hide file tree

Showing 5 changed files with 233 additions and 15 deletions.
diff --git a/src/main/extensions.js b/src/main/extensions.js
@@ -130,6 +130,7 @@ export default class extensions {
 
     // Call for listing objects v2 API
     queries.push(`list-type=2`)
+    queries.push(`encoding-type=url`)
     // escape every value in query string, except maxKeys
     queries.push(`prefix=${uriEscape(prefix)}`)
     queries.push(`delimiter=${uriEscape(delimiter)}`)

diff --git a/src/main/helpers.js b/src/main/helpers.js
@@ -396,10 +396,28 @@ export const toMd5=(payload)=>{
   let payLoadBuf = objectToBuffer(payload)
   // use string from browser and buffer from nodejs
   // browser support is tested only against minio server
-  payLoadBuf = isBrowser ? payLoadBuf.toString() : payLoadBuf 
+  payLoadBuf = isBrowser ? payLoadBuf.toString() : payLoadBuf
   return Crypto.createHash('md5').update(payLoadBuf).digest().toString('base64')
 }
 
 export const toSha256=(payload)=>{
   return Crypto.createHash('sha256').update(payload).digest('hex')
 }
+
+// toArray returns a single element array with param being the element,
+// if param is just a string, and returns 'param' back if it is an array
+// So, it makes sure param is always an array
+export const toArray = (param) => {
+  if (!Array.isArray(param)) {
+    return Array(param)
+  }
+  return param
+}
+
+
+export const sanitizeObjectKey=(objectName)=>{
+  // + symbol characters are not decoded as spaces in JS. so replace them first and decode to get the correct result.
+  let asStrName= (objectName || "").replace(/\+/g, ' ')
+  const sanitizedName = decodeURIComponent(asStrName)
+  return sanitizedName
+}
diff --git a/src/main/minio.js b/src/main/minio.js
@@ -1269,6 +1269,7 @@ export class Client {
     // escape every value in query string, except maxKeys
     queries.push(`prefix=${uriEscape(prefix)}`)
     queries.push(`delimiter=${uriEscape(Delimiter)}`)
+    queries.push(`encoding-type=url`)
 
     if (IncludeVersion) {
       queries.push(`versions`)
@@ -1406,6 +1407,7 @@ export class Client {
 
     // Call for listing objects v2 API
     queries.push(`list-type=2`)
+    queries.push(`encoding-type=url`)
 
     // escape every value in query string, except maxKeys
     queries.push(`prefix=${uriEscape(prefix)}`)

diff --git a/src/main/xml-parsers.js b/src/main/xml-parsers.js
@@ -20,6 +20,8 @@ import * as errors from './errors.js'
 import {
   isObject,
   sanitizeETag,
+  toArray,
+  sanitizeObjectKey,
   RETENTION_VALIDITY_UNITS
 } from "./helpers"
 
@@ -33,16 +35,6 @@ var parseXml = (xml) => {
   return result
 }
 
-// toArray returns a single element array with param being the element,
-// if param is just a string, and returns 'param' back if it is an array
-// So, it makes sure param is always an array
-var toArray = (param) => {
-  if (!Array.isArray(param)) {
-    return Array(param)
-  }
-  return param
-}
-
 // Parse XML and return information as Javascript types
 
 // parse error XML response
@@ -299,7 +291,7 @@ const formatObjInfo = (content, opts={}) => {
     opts = {}
   }
 
-  const name = toArray(Key)[0]
+  const name = sanitizeObjectKey(toArray(Key)[0])
   const lastModified = new Date(toArray(LastModified)[0])
   const etag = sanitizeETag(toArray(ETag)[0])
 
@@ -342,7 +334,7 @@ export function parseListObjects(xml) {
     }
     if (listBucketResult.Contents) {
       toArray(listBucketResult.Contents).forEach(content => {
-        const name = toArray(content.Key)[0]
+        const name = sanitizeObjectKey(toArray(content.Key)[0])
         const lastModified = new Date(toArray(content.LastModified)[0])
         const etag = sanitizeETag(toArray(content.ETag)[0])
         const size = content.Size
@@ -403,7 +395,7 @@ export function parseListObjectsV2(xml) {
   if (xmlobj.NextContinuationToken) result.nextContinuationToken = xmlobj.NextContinuationToken
   if (xmlobj.Contents) {
     toArray(xmlobj.Contents).forEach(content => {
-      var name = content.Key
+      var name = sanitizeObjectKey(toArray(content.Key)[0])
       var lastModified = new Date(content.LastModified)
       var etag = sanitizeETag(content.ETag)
       var size = content.Size
@@ -436,7 +428,7 @@ export function parseListObjectsV2WithMetadata(xml) {
 
   if (xmlobj.Contents) {
     toArray(xmlobj.Contents).forEach(content => {
-      var name = content.Key
+      var name = sanitizeObjectKey(content.Key)
       var lastModified = new Date(content.LastModified)
       var etag = sanitizeETag(content.ETag)
       var size = content.Size

diff --git a/src/test/functional/functional-tests.js b/src/test/functional/functional-tests.js
@@ -2512,4 +2512,209 @@ describe('functional tests', function() {
       })
 
     })})
+
+  describe('Object Name special characters test without Prefix', ()=> {
+    // Isolate the bucket/object for easy debugging and tracking.
+    const bucketNameForSpCharObjects = "minio-js-test-obj-spwpre-" + uuid.v4()
+    before((done) => client.makeBucket(bucketNameForSpCharObjects, '', done))
+    after((done) => client.removeBucket(bucketNameForSpCharObjects, done))
+
+    // Reference:: https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html
+    const objectNameSpecialChars="äöüex ®©µÄÆÐÕæŒƕƩǅ 01000000 0x40 \u0040 amȡȹɆple&0a!-_.*'()&$@=;:+,?<>.pdf"
+
+    const objectContents = Buffer.alloc(100 * 1024, 0)
+
+    describe('Without Prefix Test', function () {
+
+      step(`putObject(bucketName, objectName, stream)_bucketName:${bucketNameForSpCharObjects}, _objectName:${objectNameSpecialChars}, stream:100Kib_`, done => {
+        client.putObject(bucketNameForSpCharObjects, objectNameSpecialChars, objectContents)
+          .then(() => {
+            done()
+          })
+          .catch(done)
+      })
+
+      step(`listObjects(bucketName, prefix, recursive)_bucketName:${bucketNameForSpCharObjects}, prefix:"", true`, done => {
+        const listStream = client.listObjects(bucketNameForSpCharObjects, "", true )
+        let listedObject = null
+        listStream.on('data', function (obj) {
+          listedObject =obj
+        })
+        listStream.on('end',()=>{
+          if(listedObject.name === objectNameSpecialChars){
+            done()
+          }else{
+            return done(new Error(`Expected object Name: ${objectNameSpecialChars}: received:${listedObject.name}`))
+          }
+        })
+        listStream.on('error', function (e) {
+          done(e)
+        })
+      })
+
+      step(`listObjectsV2(bucketName, prefix, recursive)_bucketName:${bucketNameForSpCharObjects}, prefix:"", true`, done => {
+        const listStream = client.listObjectsV2(bucketNameForSpCharObjects, "", true )
+        let listedObject = null
+        listStream.on('data', function (obj) {
+          listedObject =obj
+        })
+        listStream.on('end',()=>{
+          if(listedObject.name === objectNameSpecialChars){
+            done()
+          }else{
+            return done(new Error(`Expected object Name: ${objectNameSpecialChars}: received:${listedObject.name}`))
+          }
+        })
+
+        listStream.on('error', function (e) {
+          done(e)
+        })
+      })
+      step(`extensions.listObjectsV2WithMetadata(bucketName, prefix, recursive)_bucketName:${bucketNameForSpCharObjects}, prefix:"", true`, done => {
+        const listStream = client.extensions.listObjectsV2WithMetadata(bucketNameForSpCharObjects, "", true )
+        let listedObject = null
+        listStream.on('data', function (obj) {
+          listedObject =obj
+        })
+        listStream.on('end',()=>{
+          if(listedObject.name === objectNameSpecialChars){
+            done()
+          }else{
+            return done(new Error(`Expected object Name: ${objectNameSpecialChars}: received:${listedObject.name}`))
+          }
+        })
+
+        listStream.on('error', function (e) {
+          done(e)
+        })
+      })
+
+      step(`getObject(bucketName, objectName)_bucketName:${bucketNameForSpCharObjects}, _objectName:${objectNameSpecialChars}`, done => {
+        client.getObject(bucketNameForSpCharObjects, objectNameSpecialChars)
+          .then(stream => {
+            stream.on('data', function() {})
+            stream.on('end', done)
+          })
+          .catch(done)
+      })
+
+      step(`statObject(bucketName, objectName, cb)_bucketName:${bucketNameForSpCharObjects}, _objectName:${objectNameSpecialChars}`, done => {
+        client.statObject(bucketNameForSpCharObjects, objectNameSpecialChars, (e) => {
+          if (e) return done(e)
+          done()
+        })
+      })
+
+      step(`removeObject(bucketName, objectName)_bucketName:${bucketNameForSpCharObjects}, _objectName:${objectNameSpecialChars}`, done => {
+        client.removeObject(bucketNameForSpCharObjects, objectNameSpecialChars)
+          .then(() => done())
+          .catch(done)
+      })
+    })
+
+
+  })
+  describe('Object Name special characters test with a Prefix', ()=> {
+    // Isolate the bucket/object for easy debugging and tracking.
+    const bucketNameForSpCharObjects = "minio-js-test-obj-spnpre-" + uuid.v4()
+    before((done) => client.makeBucket(bucketNameForSpCharObjects, '', done))
+    after((done) => client.removeBucket(bucketNameForSpCharObjects, done))
+
+    // Reference:: https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html
+    const objectNameSpecialChars="äöüex ®©µÄÆÐÕæŒƕƩǅ 01000000 0x40 \u0040 amȡȹɆple&0a!-_.*'()&$@=;:+,?<>.pdf"
+    const prefix="test"
+    const objectNameWithPrefixForSpecialChars = `${prefix}/${objectNameSpecialChars}`
+
+    const objectContents = Buffer.alloc(100 * 1024, 0)
+
+
+    describe('With Prefix Test', function () {
+
+      step(`putObject(bucketName, objectName, stream)_bucketName:${bucketNameForSpCharObjects}, _objectName:${objectNameWithPrefixForSpecialChars}, stream:100Kib`, done => {
+        client.putObject(bucketNameForSpCharObjects, objectNameWithPrefixForSpecialChars, objectContents)
+          .then(() => {
+            done()
+          })
+          .catch(done)
+      })
+
+      step(`listObjects(bucketName, prefix, recursive)_bucketName:${bucketNameForSpCharObjects}, prefix:${prefix}, recursive:true`, done => {
+        const listStream = client.listObjects(bucketNameForSpCharObjects, prefix, true )
+        let listedObject = null
+        listStream.on('data', function (obj) {
+          listedObject =obj
+
+        })
+        listStream.on('end',()=>{
+          if(listedObject.name === objectNameWithPrefixForSpecialChars){
+            done()
+          }else{
+            return done(new Error(`Expected object Name: ${objectNameWithPrefixForSpecialChars}: received:${listedObject.name}`))
+          }
+        })
+        listStream.on('error', function (e) {
+          done(e)
+        })
+      })
+
+      step(`listObjectsV2(bucketName, prefix, recursive)_bucketName:${bucketNameForSpCharObjects}, prefix:${prefix}, recursive:true`, done => {
+        const listStream = client.listObjectsV2(bucketNameForSpCharObjects, prefix, true )
+        let listedObject = null
+        listStream.on('data', function (obj) {
+          listedObject =obj
+        })
+        listStream.on('end',()=>{
+          if(listedObject.name === objectNameWithPrefixForSpecialChars){
+            done()
+          }else{
+            return done(new Error(`Expected object Name: ${objectNameWithPrefixForSpecialChars}: received:${listedObject.name}`))
+          }
+        })
+        listStream.on('error', function (e) {
+          done(e)
+        })
+      })
+
+      step(`extensions.listObjectsV2WithMetadata(bucketName, prefix, recursive)_bucketName:${bucketNameForSpCharObjects}, prefix:${prefix}, recursive:true`, done => {
+        const listStream = client.extensions.listObjectsV2WithMetadata(bucketNameForSpCharObjects, prefix, true )
+        let listedObject = null
+        listStream.on('data', function (obj) {
+          listedObject =obj
+        })
+        listStream.on('end',()=>{
+          if(listedObject.name === objectNameWithPrefixForSpecialChars){
+            done()
+          }else{
+            return done(new Error(`Expected object Name: ${objectNameWithPrefixForSpecialChars}: received:${listedObject.name}`))
+          }
+        })
+        listStream.on('error', function (e) {
+          done(e)
+        })
+      })
+
+      step(`getObject(bucketName, objectName)_bucketName:${bucketNameForSpCharObjects}, _objectName_:${objectNameWithPrefixForSpecialChars}`, done => {
+        client.getObject(bucketNameForSpCharObjects, objectNameWithPrefixForSpecialChars)
+          .then(stream => {
+            stream.on('data', function() {})
+            stream.on('end', done)
+          })
+          .catch(done)
+      })
+
+      step(`statObject(bucketName, objectName, cb)_bucketName:${bucketNameForSpCharObjects}, _objectName:${objectNameWithPrefixForSpecialChars}`, done => {
+        client.statObject(bucketNameForSpCharObjects, objectNameWithPrefixForSpecialChars, (e) => {
+          if (e) return done(e)
+          done()
+        })
+      })
+
+      step(`removeObject(bucketName, objectName)_bucketName:${bucketNameForSpCharObjects}, _objectName:${objectNameWithPrefixForSpecialChars}`, done => {
+        client.removeObject(bucketNameForSpCharObjects, objectNameWithPrefixForSpecialChars)
+          .then(() => done())
+          .catch(done)
+      })
+    })
+
+  })
 })