api:put_object not saving user metadata without x-amz-meta- prefix (#655)

Prefix user-defined metadata with x-amz-meta- if not already prefixed.

Author: poornas

minio/api.py

@@ -69,7 +69,8 @@
                       is_valid_bucket_name, PartMetadata, read_full,
                       is_valid_bucket_notification_config,
                       get_s3_region_from_endpoint,
-                      mkdir_p, dump_http)
+                      mkdir_p, dump_http, amzprefix_user_metadata,
+                      is_supported_header,is_amz_header)
 from .helpers import (MAX_MULTIPART_OBJECT_SIZE,
                       MAX_POOL_SIZE,
                       MIN_PART_SIZE)
@@ -732,9 +733,10 @@ def put_object(self, bucket_name, object_name, data, length,
         if not metadata:
             metadata = {}
 
+        metadata = amzprefix_user_metadata(metadata)
+
         metadata['Content-Type'] = 'application/octet-stream' if \
             not content_type else content_type
-
         if length > MIN_PART_SIZE:
             return self._stream_put_object(bucket_name, object_name,
                                            data, length, metadata=metadata)
@@ -903,18 +905,10 @@ def stat_object(self, bucket_name, object_name):
         content_type = response.headers.get('content-type', '')
         last_modified = response.headers.get('last-modified')
 
-        ## Supported headers for object.
-        supported_headers = [
-            'cache-control',
-            'content-encoding',
-            'content-disposition',
-            ## Add more supported headers here.
-        ]
-
         ## Capture only custom metadata.
         custom_metadata = dict()
         for k in response.headers:
-            if k in supported_headers or k.lower().startswith('x-amz-meta-'):
+            if is_supported_header(k) or is_amz_header(k):
                 custom_metadata[k] = response.headers.get(k)
 
         if last_modified:

minio/helpers.py

@@ -603,3 +603,36 @@ def optimal_part_info(length):
     # Last part size.
     last_part_size = length - int(total_parts_count-1)*part_size
     return total_parts_count, part_size, last_part_size
+
+# return a new metadata dictionary where user defined metadata keys
+# are prefixed by "x-amz-meta-"
+def amzprefix_user_metadata(metadata):
+    m = dict()
+    for k,v in metadata.items():
+       if is_amz_header(k) or is_supported_header(k) or is_storageclass_header(k):
+            m[k] = v
+       else:
+            m["X-Amz-Meta-" + k] = v
+    return m
+
+# returns true if amz s3 system defined metadata
+def is_amz_header(key):
+    key = key.lower()
+    return key.startswith("x-amz-meta") or key == "x-amz-acl" or key.startswith("x-amz-server-side-encryption")
+
+# returns true if a standard supported header
+def is_supported_header(key):
+    ## Supported headers for object.
+    supported_headers = [
+	   "cache-control",
+	   "content-encoding",
+	   "content-disposition",
+	   "content-language",
+	   "x-amz-website-redirect-location",
+            ## Add more supported headers here.
+        ]
+    return key.lower() in supported_headers
+
+# returns true if header is a storage class header
+def is_storageclass_header(key):
+    return key.lower() == "x-amz-storage-class"

tests/functional/tests.py

@@ -582,7 +582,7 @@ def test_put_object(client, log_output):
         log_output.args['length'] = MB_11 = 11*1024*1024 # 11MiB.
         MB_11_reader = LimitedRandomReader(MB_11)
         log_output.args['data'] = 'LimitedRandomReader(MB_11)'
-        log_output.args['metadata'] = metadata = {'x-amz-meta-testing': 'value'}
+        log_output.args['metadata'] = metadata = {'x-amz-meta-testing': 'value','test-key':'value2'}
         log_output.args['content_type'] = content_type='application/octet-stream'
         client.put_object(bucket_name,
                           object_name+'-metadata',
@@ -599,6 +599,8 @@ def test_put_object(client, log_output):
         if value != 'value':
             raise ValueError('Metadata key has unexpected'
                              ' value {0}'.format(value))
+        if 'X-Amz-Meta-Test-Key' not in st_obj.metadata:
+            raise ValueError("Metadata key 'x-amz-meta-test-key' not found")
     except Exception as err:
         raise Exception(err)
     finally:

tests/unit/header_value_test.py

@@ -0,0 +1,59 @@
+# -*- coding: utf-8 -*-
+# Minio Python Library for Amazon S3 Compatible Cloud Storage,
+# (C) 2018 Minio, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from nose.tools import eq_, raises
+from unittest import TestCase
+
+from minio.helpers import is_storageclass_header,is_amz_header,is_supported_header,amzprefix_user_metadata
+
+class HeaderTests(TestCase):
+    def test_is_supported_header(self):
+        eq_(is_supported_header("content-type"),False)
+        eq_(is_supported_header("Content-Type"),False)
+        eq_(is_supported_header("cOntent-TypE"),False)
+        eq_(is_supported_header("x-amz-meta-me"),False)
+        eq_(is_supported_header("Cache-Control"),True)
+        eq_(is_supported_header("content-encoding"),True)
+        eq_(is_supported_header("content-disposition"),True)
+        eq_(is_supported_header("content-language"),True)
+        eq_(is_supported_header("x-amz-website-redirect-location"),True)
+    def test_is_amz_header(self):
+        eq_(is_amz_header("x-amz-meta-status-code"),True)
+        eq_(is_amz_header("X-Amz-Meta-status-code"),True)
+        eq_(is_amz_header("X_AMZ_META-VALUE"),False)
+        eq_(is_amz_header("content-type"),False)
+        eq_(is_amz_header("x-amz-server-side-encryption"),True)
+    def test_is_storageclass_header(self):
+        eq_(is_storageclass_header("x-amz-storage-classs"),False)
+        eq_(is_storageclass_header("x-amz-storage-class"),True)
+    def test_amzprefix_user_metadata(self):
+        metadata = {
+                  'x-amz-meta-testing': 'values',
+                  'x-amz-meta-setting': 'zombies',
+                  'amz-meta-setting': 'zombiesddd',
+                  'hhh':34,
+                  'u_u': 'dd',
+                  'y-fu-bar': 'zoo',
+                  'Content-Type': 'application/csv',
+                  'x-amz-storage-class': 'REDUCED_REDUNDANCY',
+                  'content-language':'fr'
+                  }
+        m = amzprefix_user_metadata(metadata)
+        self.assertTrue('X-Amz-Meta-hhh',m)
+        self.assertTrue('Content-Type',m)
+        self.assertTrue('x-amz-storage-class',m)
+        self.assertTrue('content-language',m)
+        self.assertTrue('X-Amz-Meta-amz-meta-setting',m)