Merge pull request #172 from Azure/dev

Version 0.32.0

Author: emgerner-msft

BreakingChanges.md

@@ -2,6 +2,16 @@
 
 > See the [Change Log](ChangeLog.md) for a summary of storage library changes.
 
+## Version 0.32.0:
+
+### Blob:
+- get_blob_to_* will do an initial get request of size 32 MB. If it then finds the blob is larger than this size, it will parallelize by default.
+- Block blob and page blob create_blob_from_* methods will parallelize by default.
+
+### File:
+- get_file_to_* will do an initial get request of size 32 MB. If it then finds the file is larger than this size, it will parallelize by default.
+- create_file_from_* methods will parallelize by default.
+
 ## Version 0.30.0
 
 ### All:

ChangeLog.md

@@ -2,6 +2,25 @@
 
 > See [BreakingChanges](BreakingChanges.md) for a detailed list of API breaks.
 
+## Version 0.32.0:
+
+### All:
+- request_callback and response_callback functions may be set on the service clients. These callbacks will be run before the request is executed and after the response is received, respectively. They maybe used to add custom headers to the request and for logging, among other purposes.
+- A client request id is added to requests by default.
+
+### Blob:
+- Get requests taking the start_range parameter incorrectly sent an x-ms-range header when start_range was not specified.
+- get_blob_to_* will do an initial get request of size 32 MB. If it then finds the blob is larger than this size, it will parallelize by default.
+- Block blob and page blob create_blob_from_* methods will parallelize by default.
+- The validate_content option on get_blob_to_* and on methods which put blob data will compute and validate an md5 hash of the content if set to True. This is primarily valuable for detecting bitflips on the wire if using http instead of https as https (the default) will already validate.
+- Fixed a bug where lease_id was not specified if given by the user for each chunk on parallel get requests.
+
+### File:
+- Get requests taking the start_range parameter incorrectly sent an x-ms-range header when start_range was not specified.
+- get_file_to_* will do an initial get request of size 32 MB. If it then finds the file is larger than this size, it will parallelize by default.
+- create_file_from_* methods will parallelize by default.
+- The validate_content option on get_file_to_* and create_file_from_* will compute and validate an md5 hash of the content if set to True. This is primarily valuable for detecting bitflips on the wire if using http instead of https as https (the default) will already validate.
+
 ## Version 0.31.0:
 
 ### All:

azure-storage-python.pyproj

@@ -21,13 +21,15 @@
     <PtvsTargetsFile>$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)\Python Tools\Microsoft.PythonTools.targets</PtvsTargetsFile>
   </PropertyGroup>
   <ItemGroup>
+    <Compile Include="azure\storage\blob\_upload_chunking.py" />
+    <Compile Include="azure\storage\file\_upload_chunking.py" />
     <Compile Include="azure\storage\_auth.py" />
     <Compile Include="azure\storage\blob\appendblobservice.py" />
     <Compile Include="azure\storage\blob\blockblobservice.py" />
     <Compile Include="azure\storage\blob\models.py" />
     <Compile Include="azure\storage\blob\pageblobservice.py" />
     <Compile Include="azure\storage\blob\baseblobservice.py" />
-    <Compile Include="azure\storage\blob\_chunking.py" />
+    <Compile Include="azure\storage\blob\_download_chunking.py" />
     <Compile Include="azure\storage\blob\_deserialization.py" />
     <Compile Include="azure\storage\blob\_error.py" />
     <Compile Include="azure\storage\blob\_serialization.py" />
@@ -37,7 +39,7 @@
     <Compile Include="azure\storage\_constants.py" />
     <Compile Include="azure\storage\file\fileservice.py" />
     <Compile Include="azure\storage\file\models.py" />
-    <Compile Include="azure\storage\file\_chunking.py" />
+    <Compile Include="azure\storage\file\_download_chunking.py" />
     <Compile Include="azure\storage\file\_deserialization.py" />
     <Compile Include="azure\storage\file\_serialization.py" />
     <Compile Include="azure\storage\file\__init__.py" />
@@ -62,13 +64,15 @@
     <Compile Include="azure\storage\_error.py" />
     <Compile Include="azure\storage\_deserialization.py" />
     <Compile Include="azure\storage\_http\httpclient.py" />
-    <Compile Include="azure\storage\_http\requestsclient.py" />
     <Compile Include="azure\storage\_http\__init__.py" />
     <Compile Include="azure\storage\_serialization.py" />
     <Compile Include="azure\storage\__init__.py" />
     <Compile Include="azure\__init__.py" />
     <Compile Include="doc\conf.py" />
     <Compile Include="doc\__init__.py" />
+    <Compile Include="samples\advanced\client.py" />
+    <Compile Include="samples\advanced\authentication.py" />
+    <Compile Include="samples\advanced\__init__.py" />
     <Compile Include="samples\blob\append_blob_usage.py" />
     <Compile Include="samples\blob\block_blob_usage.py" />
     <Compile Include="samples\blob\page_blob_usage.py" />
@@ -97,6 +101,7 @@
     <Compile Include="tests\test_client.py" />
     <Compile Include="tests\test_account.py" />
     <Compile Include="tests\test_directory.py" />
+    <Compile Include="tests\test_get_file.py" />
     <Compile Include="tests\test_service_stats.py" />
     <Compile Include="tests\test_share.py" />
     <Compile Include="tests\test_get_blob.py" />
@@ -128,6 +133,7 @@
     <Folder Include="samples\" />
     <Folder Include="samples\file\" />
     <Folder Include="samples\blob\" />
+    <Folder Include="samples\advanced\" />
     <Folder Include="samples\table\" />
     <Folder Include="samples\queue\" />
     <Folder Include="tests" />

azure/storage/_auth.py

@@ -23,7 +23,7 @@ def __init__(self, account_name, account_key):
         self.account_key = account_key
 
     def _get_headers(self, request, headers_to_sign):
-        headers = dict((name.lower(), value) for name, value in request.headers if value)
+        headers = dict((name.lower(), value) for name, value in request.headers.items() if value)
         if 'content-length' in headers and headers['content-length'] == '0':
             del headers['content-length']
         return '\n'.join(headers.get(x, '') for x in headers_to_sign) + '\n'
@@ -38,7 +38,7 @@ def _get_canonicalized_resource(self, request):
     def _get_canonicalized_headers(self, request):
         string_to_sign = ''
         x_ms_headers = []
-        for name, value in request.headers:
+        for name, value in request.headers.items():
             if name.startswith('x-ms-'):
                 x_ms_headers.append((name.lower(), value))
         x_ms_headers.sort()
@@ -50,7 +50,7 @@ def _get_canonicalized_headers(self, request):
     def _add_authorization_header(self, request, string_to_sign):
         signature = _sign_string(self.account_key, string_to_sign)
         auth_string = 'SharedKey ' + self.account_name + ':' + signature
-        request.headers.append(('Authorization', auth_string))
+        request.headers['Authorization'] = auth_string
 
 
 class _StorageSharedKeyAuthentication(_StorageSharedKeyAuthentication):
@@ -72,10 +72,11 @@ def sign_request(self, request):
         self._add_authorization_header(request, string_to_sign)
 
     def _get_canonicalized_resource_query(self, request):
-        request.query.sort()
+        sorted_queries = [(name, value) for name, value in request.query.items()]
+        sorted_queries.sort()
 
         string_to_sign = ''
-        for name, value in request.query:
+        for name, value in sorted_queries:
             if value:
                 string_to_sign += '\n' + name.lower() + ':' + value
 
@@ -96,7 +97,7 @@ def sign_request(self, request):
         self._add_authorization_header(request, string_to_sign)
 
     def _get_canonicalized_resource_query(self, request):
-        for name, value in request.query:
+        for name, value in request.query.items():
             if name == 'comp':
                 return '?comp=' + value
         return ''

azure/storage/_common_conversion.py

@@ -98,6 +98,10 @@ def _sign_string(key, string_to_sign, key_is_base64=True):
     encoded_digest = _encode_base64(digest)
     return encoded_digest
 
+def _get_content_md5(data):
+    md5 = hashlib.md5()
+    md5.update(data)
+    return base64.b64encode(md5.digest()).decode('utf-8') 
 
 def _lower(text):
     return text.lower()

azure/storage/_constants.py

@@ -15,12 +15,12 @@
 import platform
 
 __author__ = 'Microsoft Corp. <[email protected]>'
-__version__ = '0.31.0'
+__version__ = '0.32.0'
 
 # x-ms-version for storage service.
 X_MS_VERSION = '2015-07-08'
 
-# UserAgent string sample: 'Azure-Storage/0.31.0 (Python CPython 3.4.2; Windows 8)'
+# UserAgent string sample: 'Azure-Storage/0.32.0 (Python CPython 3.4.2; Windows 8)'
 _USER_AGENT_STRING = 'Azure-Storage/{} (Python {} {}; {} {})'.format(__version__, platform.python_implementation(), platform.python_version(), platform.system(), platform.release())
 
 # Live ServiceClient URLs

azure/storage/_deserialization.py

@@ -49,6 +49,7 @@ def _get_download_size(start_range, end_range, resource_size):
     'etag': (None, 'etag', _to_str),
     'x-ms-blob-type': (None, 'blob_type', _to_str),
     'content-length': (None, 'content_length', _int_to_str),
+    'content-range': (None, 'content_range', _to_str),
     'x-ms-blob-sequence-number': (None, 'page_blob_sequence_number', _int_to_str),
     'x-ms-blob-committed-block-count': (None, 'append_blob_committed_block_count', _int_to_str),
     'x-ms-share-quota': (None, 'quota', _int_to_str),
@@ -78,7 +79,7 @@ def _parse_metadata(response):
         return None
 
     metadata = _dict()
-    for key, value in response.headers:
+    for key, value in response.headers.items():
         if key.startswith('x-ms-meta-'):
             metadata[key[10:]] = _to_str(value)
 
@@ -94,7 +95,7 @@ def _parse_properties(response, result_class):
         return None
 
     props = result_class()
-    for key, value in response.headers:
+    for key, value in response.headers.items():
         info = GET_PROPERTIES_ATTRIBUTE_MAP.get(key)
         if info:
             if info[0] is None:
@@ -105,21 +106,17 @@ def _parse_properties(response, result_class):
 
     return props
 
-def _parse_response_for_dict(response):
-    ''' Extracts name-values from response header. Filter out the standard
-    http headers.'''
-
-    if response is None:
+def _parse_length_from_content_range(content_range):
+    '''
+    Parses the blob length from the content range header: bytes 1-3/65537
+    '''   
+    if content_range is None:
         return None
-    http_headers = ['server', 'date', 'location', 'host',
-                    'via', 'proxy-connection', 'connection']
-    return_dict = _HeaderDict()
-    if response.headers:
-        for name, value in response.headers:
-            if not name.lower() in http_headers:
-                return_dict[name] = value
-
-    return return_dict
+
+    # First, split in space and take the second half: '1-3/65537'
+    # Next, split on slash and take the second half: '65537'
+    # Finally, convert to an int: 65537
+    return int(content_range.split(' ', 1)[1].split('/', 1)[1])
 
 def _convert_xml_to_signed_identifiers(xml):
     '''

azure/storage/_error.py

@@ -12,11 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #--------------------------------------------------------------------------
-
+from ._common_conversion import _to_str
 from azure.common import (
     AzureHttpError,
     AzureConflictHttpError,
     AzureMissingResourceHttpError,
+    AzureException,
 )
 
 _ERROR_CONFLICT = 'Conflict ({0})'
@@ -44,6 +45,8 @@
 _ERROR_RANGE_TOO_LARGE_FOR_MD5 = \
     'Getting content MD5 for a range greater than 4MB ' + \
     'is not supported.'
+_ERROR_MD5_MISMATCH = \
+    'MD5 mismatch. Expected value is \'{0}\', computed value is \'{1}\'.'
 
 def _dont_fail_on_exist(error):
     ''' don't throw exception if the resource exists.
@@ -78,4 +81,8 @@ def _validate_type_bytes(param_name, param):
 
 def _validate_not_none(param_name, param):
     if param is None:
-        raise ValueError(_ERROR_VALUE_NONE.format(param_name))
+        raise ValueError(_ERROR_VALUE_NONE.format(param_name))
+
+def _validate_content_match(server_md5, computed_md5):
+    if server_md5 != computed_md5:
+        raise AzureException(_ERROR_MD5_MISMATCH.format(server_md5, computed_md5))

azure/storage/_http/__init__.py

@@ -15,11 +15,20 @@
 
 class HTTPError(Exception):
 
-    ''' HTTP Exception when response status code >= 300 '''
+    '''
+    Represents an HTTP Exception when response status code >= 300.
+
+    :ivar int status:
+        the status code of the response
+    :ivar str message:
+        the message
+    :ivar list headers:
+        the returned headers, as a list of (name, value) pairs
+    :ivar bytes body:
+        the body of the response
+    '''
 
     def __init__(self, status, message, respheader, respbody):
-        '''Creates a new HTTPError with the specified status, message,
-        response headers and body'''
         self.status = status
         self.respheader = respheader
         self.respbody = respbody
@@ -28,18 +37,18 @@ def __init__(self, status, message, respheader, respbody):
 
 class HTTPResponse(object):
 
-    """Represents a response from an HTTP request.  An HTTPResponse has the
-    following attributes:
+    '''
+    Represents a response from an HTTP request.
     
-    status:
+    :ivar int status:
         the status code of the response
-    message:
+    :ivar str message:
         the message
-    headers:
-        the returned headers, as a list of (name, value) pairs
-    body:
+    :ivar dict headers:
+        the returned headers
+    :ivar bytes body:
         the body of the response
-    """
+    '''
 
     def __init__(self, status, message, headers, body):
         self.status = status
@@ -50,30 +59,27 @@ def __init__(self, status, message, headers, body):
 
 class HTTPRequest(object):
 
-    '''Represents an HTTP Request.  An HTTP Request consists of the following
-    attributes:
-    host:
+    '''
+    Represents an HTTP Request.
+
+    :ivar str host:
         the host name to connect to
-    method:
+    :ivar str method:
         the method to use to connect (string such as GET, POST, PUT, etc.)
-    path:
+    :ivar str path:
         the uri fragment
-    query:
-        query parameters specified as a list of (name, value) pairs
-    headers:
-        header values specified as (name, value) pairs
-    body:
+    :ivar dict query:
+        query parameters
+    :ivar dict headers:
+        header values
+    :ivar bytes body:
         the body of the request.
-    protocol_override:
-        specify to use this protocol instead of the global one stored in
-        _HTTPClient.
     '''
 
     def __init__(self):
         self.host = ''
         self.method = ''
         self.path = ''
-        self.query = []      # list of (name, value)
-        self.headers = []    # list of (header name, header value)
+        self.query = {}      # list of (name, value)
+        self.headers = {}    # list of (header name, header value)
         self.body = ''
-        self.protocol_override = None