Merge pull request #270 from Azure/dev

Final Release.

Author: mirobers

BreakingChanges.md

@@ -2,6 +2,17 @@
 
 > See the [Change Log](ChangeLog.md) for a summary of storage library changes.
 
+## Version 0.34.0:
+
+### All:
+- Several error messages have been clarified or made more specific.
+
+### Blob:
+- If-None-Match: * will now fail when reading a blob. Previously this header was ignored for blob reads.
+
+### Queue:
+- For put_message a QueueMessage will be returned.  This message will have pop receipt, insertion/expiration time, and message ID populated.
+
 ## Version 0.33.0:
 - Remove with_filter from service client in favor of the newer callback functions.
 - Remove max_retries and retry_wait from the blob and file create and get functions in favor of the new client-level retry policies.

ChangeLog.md

@@ -2,6 +2,33 @@
 
 > See [BreakingChanges](BreakingChanges.md) for a detailed list of API breaks.
 
+## Version 0.34.0:
+
+### All:
+- All: Support for 2016-05-31 REST version. Please see our REST API documentation and blogs for information about the related added features. If you are using the Storage Emulator, please update to Emulator version 4.6
+- All: Several error messages have been clarified or made more specific.
+
+### Blob:
+- Added support for server-side encryption headers.
+- Properly return connections to pool when checking for non-existent blobs.
+- Fixed a bug with parallel uploads for PageBlobs and BlockBlobs where chunks were being buffered and queued faster than can be processed, potentially causing out-of-memory issues.
+- Added large block blob upload support. Blocks can now support sizes up to 100 MB and thus the maximum size of a BlockBlob is now 5,000,000 MB (~4.75 TB).
+- Added streaming upload support for the put_block method and a new memory optimized upload algorithm for create_blob_from_stream and create_blob_from_file APIs. (BlockBlobService)
+- The new upload strategy will no longer fully buffer seekable streams unless Encryption is enabled. See 'use_byte_buffer' parameter documentation on the 'create_blob_from_stream' method for more details.
+- Fixed a deserialization bug with get_block_list() where calling it with anything but the 'all' block_list_type would cause an error.
+- Using If-None-Match: * will now fail when reading a blob. Previously this header was ignored for blob reads.
+- Populate public access when listing blob containers.
+- The public access setting on a blob container is now a container property returned from downloadProperties.
+- Populate content MD5 for range gets on blobs.
+- Added support for incremental copy on page blobs. The source must be a snapshot of a page blob and include a SAS token.
+
+### File:
+- Prefix support for listing files and directories.
+- Populate content MD5 for range gets on files.
+
+### Queue:
+-  put_message now returns a QueueMessage with the PopReceipt, Id, NextVisibleTime, InsertionTime, and ExpirationTime properties populated along with the content.
+
 ## Version 0.33.0:
 
 ### All:

azure/storage/_common_conversion.py

@@ -18,7 +18,12 @@
 import hmac
 import sys
 from dateutil.tz import tzutc
+from io import (IOBase, SEEK_SET)
 
+from ._error import (
+    _ERROR_VALUE_SHOULD_BE_BYTES_OR_STREAM,
+    _ERROR_VALUE_SHOULD_BE_SEEKABLE_STREAM,
+)
 from .models import (
     _unicode_type,
 )
@@ -98,8 +103,24 @@ def _sign_string(key, string_to_sign, key_is_base64=True):
 
 def _get_content_md5(data):
     md5 = hashlib.md5()
-    md5.update(data)
-    return base64.b64encode(md5.digest()).decode('utf-8') 
+    if isinstance(data, bytes):
+        md5.update(data)
+    elif hasattr(data, 'read'):
+        pos = 0
+        try:
+            pos = data.tell()
+        except:
+            pass
+        for chunk in iter(lambda: data.read(4096), b""):
+            md5.update(chunk)
+        try:
+            data.seek(pos, SEEK_SET)
+        except (AttributeError, IOError):
+            raise ValueError(_ERROR_VALUE_SHOULD_BE_SEEKABLE_STREAM.format('data'))
+    else:
+        raise ValueError(_ERROR_VALUE_SHOULD_BE_BYTES_OR_STREAM.format('data'))
+
+    return base64.b64encode(md5.digest()).decode('utf-8')
 
 def _lower(text):
     return text.lower()

azure/storage/_constants.py

@@ -15,10 +15,10 @@
 import platform
 
 __author__ = 'Microsoft Corp. <[email protected]>'
-__version__ = '0.33.0'
+__version__ = '0.34.0'
 
 # x-ms-version for storage service.
-X_MS_VERSION = '2015-07-08'
+X_MS_VERSION = '2016-05-31'
 
 # UserAgent string sample: 'Azure-Storage/0.32.0 (Python CPython 3.4.2; Windows 8)'
 USER_AGENT_STRING = 'Azure-Storage/{} (Python {} {}; {} {})'.format(__version__, platform.python_implementation(), platform.python_version(), platform.system(), platform.release())

azure/storage/_deserialization.py

@@ -34,6 +34,9 @@
 def _int_to_str(value):
     return value if value is None else int(value)
 
+def _bool(value):
+    return value.lower() == 'true'
+
 def _get_download_size(start_range, end_range, resource_size):
     if start_range is not None:
         end_range = end_range if end_range else (resource_size if resource_size else None)
@@ -53,6 +56,7 @@ def _get_download_size(start_range, end_range, resource_size):
     'x-ms-blob-sequence-number': (None, 'page_blob_sequence_number', _int_to_str),
     'x-ms-blob-committed-block-count': (None, 'append_blob_committed_block_count', _int_to_str),
     'x-ms-share-quota': (None, 'quota', _int_to_str),
+    'x-ms-server-encrypted': (None, 'server_encrypted', _bool),
     'content-type': ('content_settings', 'content_type', _to_str),
     'cache-control': ('content_settings', 'cache_control', _to_str),
     'content-encoding': ('content_settings', 'content_encoding', _to_str),
@@ -67,6 +71,7 @@ def _get_download_size(start_range, end_range, resource_size):
     'x-ms-copy-status': ('copy', 'status', _to_str),
     'x-ms-copy-progress': ('copy', 'progress', _to_str),
     'x-ms-copy-completion-time': ('copy', 'completion_time', parser.parse),
+    'x-ms-copy-destination-snapshot': ('copy', 'destination_snapshot_time', _to_str),
     'x-ms-copy-status-description': ('copy', 'status_description', _to_str),
 }
 
@@ -324,8 +329,4 @@ def _convert_xml_to_retention_policy(xml, retention_policy):
     # Days
     days_element =  xml.find('Days')
     if days_element is not None:
-        retention_policy.days = int(days_element.text)
-
-
-def _bool(value):
-    return value.lower() == 'true'
+        retention_policy.days = int(days_element.text)

azure/storage/_error.py

@@ -12,7 +12,20 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #--------------------------------------------------------------------------
-from ._common_conversion import _to_str
+from sys import version_info
+from io import IOBase
+if version_info < (3,):
+    def _str(value):
+        if isinstance(value, unicode):
+            return value.encode('utf-8')
+
+        return str(value)
+else:
+    _str = str
+
+def _to_str(value):
+    return _str(value) if value is not None else None
+
 from azure.common import (
     AzureHttpError,
     AzureConflictHttpError,
@@ -34,6 +47,9 @@
     'instance'
 _ERROR_PARALLEL_NOT_SEEKABLE = 'Parallel operations require a seekable stream.'
 _ERROR_VALUE_SHOULD_BE_BYTES = '{0} should be of type bytes.'
+_ERROR_VALUE_SHOULD_BE_BYTES_OR_STREAM = '{0} should be of type bytes or a readable file-like/io.IOBase stream object.'
+_ERROR_VALUE_SHOULD_BE_SEEKABLE_STREAM = '{0} should be a seekable file-like/io.IOBase type stream object.'
+_ERROR_VALUE_SHOULD_BE_STREAM = '{0} should be a file-like/io.IOBase type stream object with a read method.'
 _ERROR_VALUE_NONE = '{0} should not be None.'
 _ERROR_VALUE_NONE_OR_EMPTY = '{0} should not be None or empty.'
 _ERROR_VALUE_NEGATIVE = '{0} should not be negative.'
@@ -102,6 +118,10 @@ def _validate_type_bytes(param_name, param):
     if not isinstance(param, bytes):
         raise TypeError(_ERROR_VALUE_SHOULD_BE_BYTES.format(param_name))
 
+def _validate_type_bytes_or_stream(param_name, param):
+    if not (isinstance(param, bytes) or hasattr(param, 'read')):
+        raise TypeError(_ERROR_VALUE_SHOULD_BE_BYTES_OR_STREAM.format(param_name))
+
 def _validate_not_none(param_name, param):
     if param is None:
         raise ValueError(_ERROR_VALUE_NONE.format(param_name))

azure/storage/_http/httpclient.py

@@ -19,19 +19,19 @@
     from httplib import (
         HTTP_PORT,
         HTTPS_PORT,
-        )
+    )
     from urllib2 import quote as url_quote
 else:
     from http.client import (
         HTTP_PORT,
         HTTPS_PORT,
-        )
+    )
     from urllib.parse import quote as url_quote
 
 from . import HTTPError, HTTPResponse
+from .._serialization import _get_data_bytes_or_stream_only
 
 class _HTTPClient(object):
-
     '''
     Takes the request and sends it to cloud service and returns the response.
     '''
@@ -64,9 +64,9 @@ def set_proxy(self, host, port, user, password):
         '''
         Sets the proxy server host and port for the HTTP CONNECT Tunnelling.
 
-        Note that we set the proxies directly on the request later on rather than 
-        using the session object as requests has a bug where session proxy is ignored 
-        in favor of environment proxy. So, auth will not work unless it is passed 
+        Note that we set the proxies directly on the request later on rather than
+        using the session object as requests has a bug where session proxy is ignored
+        in favor of environment proxy. So, auth will not work unless it is passed
         directly when making the request as this overrides both.
 
         :param str host:
@@ -95,11 +95,11 @@ def perform_request(self, request):
         :param HTTPRequest request:
             The request to serialize and send.
         :return: An HTTPResponse containing the parsed HTTP response.
-        :rtype: :class:`~azure.storage._http.HTTPResponse`             
+        :rtype: :class:`~azure.storage._http.HTTPResponse`
         '''
-        # Verify the body is in bytes
+        # Verify the body is in bytes or either a file-like/stream object
         if request.body:
-            assert isinstance(request.body, bytes)
+            request.body = _get_data_bytes_or_stream_only('request.body', request.body)
 
         # Construct the URI
         uri = self.protocol.lower() + '://' + request.host + request.path
@@ -119,4 +119,7 @@ def perform_request(self, request):
         for key, name in response.headers.items():
             respheaders[key.lower()] = name
 
-        return HTTPResponse(status, response.reason, respheaders, response.content)
+        wrap = HTTPResponse(status, response.reason, respheaders, response.content)
+        response.close()
+
+        return wrap

azure/storage/_serialization.py

@@ -18,13 +18,13 @@
 from dateutil.tz import tzutc
 from time import time
 from wsgiref.handlers import format_date_time
+from os import fstat
+from io import (BytesIO, IOBase, SEEK_SET, SEEK_END, UnsupportedOperation)
 
 if sys.version_info >= (3,):
-    from io import BytesIO
     from urllib.parse import quote as url_quote
 else:
-    from cStringIO import StringIO as BytesIO
-    from urllib2 import quote as url_quote   
+    from urllib2 import quote as url_quote
 
 try:
     from xml.etree import cElementTree as ETree
@@ -33,6 +33,8 @@
 
 from ._error import (
     _ERROR_VALUE_SHOULD_BE_BYTES,
+    _ERROR_VALUE_SHOULD_BE_BYTES_OR_STREAM,
+    _ERROR_VALUE_SHOULD_BE_SEEKABLE_STREAM
 )
 from ._constants import (
     X_MS_VERSION,
@@ -56,11 +58,16 @@ def _to_utc_datetime(value):
 def _update_request(request):
     # Verify body
     if request.body:
-        assert isinstance(request.body, bytes)
+        request.body = _get_data_bytes_or_stream_only('request.body', request.body)
+        length = _len_plus(request.body)
 
-    # if it is PUT, POST, MERGE, DELETE, need to add content-length to header.
-    if request.method in ['PUT', 'POST', 'MERGE', 'DELETE']:
-        request.headers['Content-Length'] = str(len(request.body))
+        # only scenario where this case is plausible is if the stream object is not seekable.
+        if length is None:
+            raise ValueError(_ERROR_VALUE_SHOULD_BE_SEEKABLE_STREAM)
+
+        # if it is PUT, POST, MERGE, DELETE, need to add content-length to header.
+        if request.method in ['PUT', 'POST', 'MERGE', 'DELETE']:
+            request.headers['Content-Length'] = str(length)
 
     # append addtional headers based on the service
     request.headers['x-ms-version'] = X_MS_VERSION
@@ -99,6 +106,18 @@ def _get_data_bytes_only(param_name, param_value):
     raise TypeError(_ERROR_VALUE_SHOULD_BE_BYTES.format(param_name))
 
 
+def _get_data_bytes_or_stream_only(param_name, param_value):
+    '''Validates the request body passed in is a stream/file-like or bytes
+    object.'''
+    if param_value is None:
+        return b''
+
+    if isinstance(param_value, bytes) or hasattr(param_value, 'read'):
+        return param_value
+
+    raise TypeError(_ERROR_VALUE_SHOULD_BE_BYTES_OR_STREAM.format(param_name))
+
+
 def _get_request_body(request_body):
     '''Converts an object into a request body.  If it's None
     we'll return an empty string, if it's one of our objects it'll
@@ -107,7 +126,7 @@ def _get_request_body(request_body):
     if request_body is None:
         return b''
 
-    if isinstance(request_body, bytes):
+    if isinstance(request_body, bytes) or isinstance(request_body, IOBase):
         return request_body
 
     if isinstance(request_body, _unicode_type):
@@ -145,7 +164,7 @@ def _convert_signed_identifiers_to_xml(signed_identifiers):
             if isinstance(access_policy.expiry, date):
                 expiry = _to_utc_datetime(expiry)
             ETree.SubElement(policy, 'Expiry').text = expiry
-        
+
         if access_policy.permission:
             ETree.SubElement(policy, 'Permission').text = _str(access_policy.permission)
 
@@ -243,7 +262,6 @@ def _convert_service_properties_to_xml(logging, hour_metrics, minute_metrics, co
     if target_version:
         ETree.SubElement(service_properties_element, 'DefaultServiceVersion').text = target_version
 
-
     # Add xml declaration and serialize
     try:
         stream = BytesIO()
@@ -291,3 +309,32 @@ def _convert_retention_policy_to_xml(retention_policy, root):
     # Days
     if retention_policy.enabled and retention_policy.days:
         ETree.SubElement(root, 'Days').text = str(retention_policy.days)
+
+def _len_plus(data):
+    length = None
+    # Check if object implements the __len__ method, covers most input cases such as bytearray.
+    try:
+        length = len(data)
+    except:
+        pass
+
+    if not length:
+        # Check if the stream is a file-like stream object.
+        # If so, calculate the size using the file descriptor.
+        try:
+            fileno = data.fileno()
+        except (AttributeError, UnsupportedOperation):
+            pass
+        else:
+            return fstat(fileno).st_size
+
+        # If the stream is seekable and tell() is implemented, calculate the stream size.
+        try:
+            currentPosition = data.tell()
+            data.seek(0, SEEK_END)
+            length = data.tell() - currentPosition
+            data.seek(currentPosition, SEEK_SET)
+        except (AttributeError, UnsupportedOperation):
+            pass
+
+    return length