Wastore master to Azure master

ChangeLog.md

@@ -2,6 +2,23 @@
 
 > See [BreakingChanges](BreakingChanges.md) for a detailed list of API breaks.
 
+## Version 0.35.0:
+
+### All:
+- Support for 2017-04-17 REST version. Please see our REST API documentation and blogs for information about the related added features. If you are using the Storage Emulator, please update to Emulator version 5.2.
+- Fixed a bug where deserialization of service stats throws a TypeError when the service is unavailable.
+
+### Blob:
+- For Premium Accounts only, added support for getting and setting the tier on a page blob. The tier can also be set when creating or copying from an existing page blob.
+- create_from_* and and append_blob_from_* methods will return response_properties which contains the etag and last modified time.
+
+### Table:
+- Fixed syntax error in _convert_json_response_to_entities.
+- Fixed a bug where the urls are not correctly formed when making commit_batch to the emulator.
+
+### File:
+- The `server_encrypted` file property will now be populated when calling 'get_directory_properties', 'get_file', and 'get_file_properties'. This value is set to True if the file data (for files) and application metadata are completely encrypted.
+
 ## Version 0.34.3:
 - All: Made the socket timeout configurable. Increased the default socket timeout to 20 seconds.
 - All: Fixed a bug where SAS tokens were being duplicated on retries
@@ -179,4 +196,4 @@
 - Client-side validation added for ranges used in APIs.
 - Metadata returned for shares, directories, and files will be returned without the 'x-ms-meta' prefix on the keys. Namely, metadata will be returned as it is received.
 - get_share_properties, get_directory_properties, and get_file_properties return parsed Share, Directory, and File objects, respectively, instead of string header dictionaries.
-- copy_file returns a parsed CopyProperties object instead of a string header dictionary.
+- copy_file returns a parsed CopyProperties object instead of a string header dictionary.

azure/storage/_connection.py

@@ -55,6 +55,7 @@ def __init__(self, service, account_name=None, account_key=None, sas_token=None,
         self.account_key = account_key
         self.sas_token = sas_token
         self.protocol = protocol or DEFAULT_PROTOCOL
+        self.is_emulated = is_emulated
 
         if is_emulated:
             self.account_name = DEV_ACCOUNT_NAME

azure/storage/_constants.py

@@ -15,10 +15,10 @@
 import platform
 
 __author__ = 'Microsoft Corp. <[email protected]>'
-__version__ = '0.34.3'
+__version__ = '0.35.0'
 
 # x-ms-version for storage service.
-X_MS_VERSION = '2016-05-31'
+X_MS_VERSION = '2017-04-17'
 
 # UserAgent string sample: 'Azure-Storage/0.32.0 (Python CPython 3.4.2; Windows 8)'
 USER_AGENT_STRING = 'Azure-Storage/{} (Python {} {}; {} {})'.format(__version__, platform.python_implementation(), platform.python_version(), platform.system(), platform.release())

azure/storage/_deserialization.py

@@ -37,6 +37,9 @@ def _int_to_str(value):
 def _bool(value):
     return value.lower() == 'true'
 
+def _to_upper_str(value):
+    return _to_str(value).upper() if value is not None else None
+
 def _get_download_size(start_range, end_range, resource_size):
     if start_range is not None:
         end_range = end_range if end_range else (resource_size if resource_size else None)
@@ -55,6 +58,8 @@ def _get_download_size(start_range, end_range, resource_size):
     'content-range': (None, 'content_range', _to_str),
     'x-ms-blob-sequence-number': (None, 'page_blob_sequence_number', _int_to_str),
     'x-ms-blob-committed-block-count': (None, 'append_blob_committed_block_count', _int_to_str),
+    'x-ms-access-tier': (None, 'blob_tier', _to_upper_str),
+    'x-ms-access-tier-inferred': (None, 'blob_tier_inferred', _bool),
     'x-ms-share-quota': (None, 'quota', _int_to_str),
     'x-ms-server-encrypted': (None, 'server_encrypted', _bool),
     'content-type': ('content_settings', 'content_type', _to_str),
@@ -184,7 +189,8 @@ def _convert_xml_to_service_stats(response):
 
     geo_replication = GeoReplication()
     geo_replication.status = geo_replication_element.find('Status').text
-    geo_replication.last_sync_time = parser.parse(geo_replication_element.find('LastSyncTime').text)
+    last_sync_time = geo_replication_element.find('LastSyncTime').text
+    geo_replication.last_sync_time = parser.parse(last_sync_time) if last_sync_time else None
 
     service_stats = ServiceStats()
     service_stats.geo_replication = geo_replication

azure/storage/blob/_deserialization.py

@@ -233,6 +233,7 @@ def _convert_xml_to_containers(response):
     'CopyProgress': ('copy', 'progress', _to_str),
     'CopyCompletionTime': ('copy', 'completion_time', _to_str),
     'CopyStatusDescription': ('copy', 'status_description', _to_str),
+    'AccessTier': (None, 'blob_tier', _to_str)
 }
 
 def _convert_xml_to_blob_list(response):
@@ -267,6 +268,7 @@ def _convert_xml_to_blob_list(response):
             <CopyProgress>bytes copied/bytes total</CopyProgress>
             <CopyCompletionTime>datetime</CopyCompletionTime>
             <CopyStatusDescription>error string</CopyStatusDescription>
+            <AccessTier>P4 | P6 | P10 | P20 | P30 | P40 | P50 | P60</AccessTier>
           </Properties>
           <Metadata>   
             <Name>value</Name>

azure/storage/blob/_upload_chunking.py

@@ -15,6 +15,7 @@
 import sys
 from threading import Lock
 from time import sleep
+
 from cryptography.hazmat.primitives.padding import PKCS7
 from .._common_conversion import _encode_base64
 from .._serialization import (
@@ -37,7 +38,7 @@ def _upload_blob_chunks(blob_service, container_name, blob_name,
                         blob_size, block_size, stream, max_connections,
                         progress_callback, validate_content, lease_id, uploader_class, 
                         maxsize_condition=None, if_match=None, timeout=None,
-                        content_encryption_key=None, initialization_vector=None):
+                        content_encryption_key=None, initialization_vector=None, resource_properties=None):
 
     encryptor, padder = _get_blob_encryptor_and_padder(content_encryption_key, initialization_vector,
                                                        uploader_class is not _PageBlobChunkUploader)
@@ -104,6 +105,10 @@ def _upload_blob_chunks(blob_service, container_name, blob_name,
     else:
         range_ids = [uploader.process_chunk(result) for result in uploader.get_chunk_streams()]
 
+    if resource_properties:
+        resource_properties.last_modified = uploader.last_modified
+        resource_properties.etag = uploader.etag
+
     return range_ids
 
 def _upload_blob_substream_blocks(blob_service, container_name, blob_name,
@@ -249,6 +254,10 @@ def _upload_substream_block_with_progress(self, block_id, block_stream):
         self._update_progress(len(block_stream))
         return range_id
 
+    def set_response_properties(self, resp):
+        self.etag = resp.etag
+        self.last_modified = resp.last_modified
+
 class _BlockBlobChunkUploader(_BlobChunkUploader):
     def _upload_chunk(self, chunk_offset, chunk_data):
         block_id = url_quote(_encode_base64('{0:032d}'.format(chunk_offset)))
@@ -297,6 +306,8 @@ def _upload_chunk(self, chunk_start, chunk_data):
         if not self.parallel:
             self.if_match = resp.etag
 
+        self.set_response_properties(resp)
+
 class _AppendBlobChunkUploader(_BlobChunkUploader):
     def _upload_chunk(self, chunk_offset, chunk_data):
         if not hasattr(self, 'current_length'):
@@ -323,6 +334,8 @@ def _upload_chunk(self, chunk_offset, chunk_data):
                 timeout=self.timeout,
             )
 
+        self.set_response_properties(resp)
+
 class _SubStream(IOBase):
     def __init__(self, wrapped_stream, stream_begin_index, length, lockObj):
         # Python 2.7: file-like objects created with open() typically support seek(), but are not

azure/storage/blob/appendblobservice.py

@@ -33,7 +33,10 @@
     _AppendBlobChunkUploader,
     _upload_blob_chunks,
 )
-from .models import _BlobTypes
+from .models import (
+    _BlobTypes,
+    ResourceProperties
+)
 from .._constants import (
     SERVICE_HOST_BASE,
     DEFAULT_PROTOCOL,
@@ -325,6 +328,8 @@ def append_blob_from_path(
             The timeout parameter is expressed in seconds. This method may make 
             multiple calls to the Azure service and the timeout will apply to 
             each call individually.
+        :return: ETag and last modified properties for the Append Blob
+        :rtype: :class:`~azure.storage.blob.models.ResourceProperties`
         '''
         _validate_not_none('container_name', container_name)
         _validate_not_none('blob_name', blob_name)
@@ -333,16 +338,16 @@ def append_blob_from_path(
 
         count = path.getsize(file_path)
         with open(file_path, 'rb') as stream:
-            self.append_blob_from_stream(
-                container_name,
-                blob_name,
-                stream,
-                count=count,
-                validate_content=validate_content,
-                maxsize_condition=maxsize_condition,
-                progress_callback=progress_callback,
-                lease_id=lease_id,
-                timeout=timeout)
+            return self.append_blob_from_stream(
+                    container_name,
+                    blob_name,
+                    stream,
+                    count=count,
+                    validate_content=validate_content,
+                    maxsize_condition=maxsize_condition,
+                    progress_callback=progress_callback,
+                    lease_id=lease_id,
+                    timeout=timeout)
 
     def append_blob_from_bytes(
         self, container_name, blob_name, blob, index=0, count=None,
@@ -387,6 +392,8 @@ def append_blob_from_bytes(
             The timeout parameter is expressed in seconds. This method may make 
             multiple calls to the Azure service and the timeout will apply to 
             each call individually.
+        :return: ETag and last modified properties for the Append Blob
+        :rtype: :class:`~azure.storage.blob.models.ResourceProperties`
         '''
         _validate_not_none('container_name', container_name)
         _validate_not_none('blob_name', blob_name)
@@ -404,16 +411,16 @@ def append_blob_from_bytes(
         stream = BytesIO(blob)
         stream.seek(index)
 
-        self.append_blob_from_stream(
-            container_name,
-            blob_name,
-            stream,
-            count=count,
-            validate_content=validate_content,
-            maxsize_condition=maxsize_condition,
-            lease_id=lease_id,
-            progress_callback=progress_callback,
-            timeout=timeout)
+        return self.append_blob_from_stream(
+                container_name,
+                blob_name,
+                stream,
+                count=count,
+                validate_content=validate_content,
+                maxsize_condition=maxsize_condition,
+                lease_id=lease_id,
+                progress_callback=progress_callback,
+                timeout=timeout)
 
     def append_blob_from_text(
         self, container_name, blob_name, text, encoding='utf-8',
@@ -455,6 +462,8 @@ def append_blob_from_text(
             The timeout parameter is expressed in seconds. This method may make 
             multiple calls to the Azure service and the timeout will apply to 
             each call individually.
+        :return: ETag and last modified properties for the Append Blob
+        :rtype: :class:`~azure.storage.blob.models.ResourceProperties`
         '''
         _validate_not_none('container_name', container_name)
         _validate_not_none('blob_name', blob_name)
@@ -465,17 +474,17 @@ def append_blob_from_text(
             _validate_not_none('encoding', encoding)
             text = text.encode(encoding)
 
-        self.append_blob_from_bytes(
-            container_name,
-            blob_name,
-            text,
-            index=0,
-            count=len(text),
-            validate_content=validate_content,
-            maxsize_condition=maxsize_condition,
-            lease_id=lease_id,
-            progress_callback=progress_callback,
-            timeout=timeout)
+        return self.append_blob_from_bytes(
+                container_name,
+                blob_name,
+                text,
+                index=0,
+                count=len(text),
+                validate_content=validate_content,
+                maxsize_condition=maxsize_condition,
+                lease_id=lease_id,
+                progress_callback=progress_callback,
+                timeout=timeout)
 
     def append_blob_from_stream(
         self, container_name, blob_name, stream, count=None,
@@ -518,12 +527,18 @@ def append_blob_from_stream(
             The timeout parameter is expressed in seconds. This method may make 
             multiple calls to the Azure service and the timeout will apply to 
             each call individually.
+        :return: ETag and last modified properties for the Append Blob
+        :rtype: :class:`~azure.storage.blob.models.ResourceProperties`
         '''
         _validate_not_none('container_name', container_name)
         _validate_not_none('blob_name', blob_name)
         _validate_not_none('stream', stream)
         _validate_encryption_unsupported(self.require_encryption, self.key_encryption_key)
 
+        # _upload_blob_chunks returns the block ids for block blobs so resource_properties
+        # is passed as a parameter to get the last_modified and etag for page and append blobs.
+        # this info is not needed for block_blobs since _put_block_list is called after which gets this info
+        resource_properties = ResourceProperties()
         _upload_blob_chunks(
             blob_service=self,
             container_name=container_name,
@@ -537,5 +552,8 @@ def append_blob_from_stream(
             lease_id=lease_id,
             uploader_class=_AppendBlobChunkUploader,
             maxsize_condition=maxsize_condition,
-            timeout=timeout
-        )
+            timeout=timeout,
+            resource_properties=resource_properties
+        )
+
+        return resource_properties

azure/storage/blob/baseblobservice.py

@@ -2981,6 +2981,7 @@ def copy_blob(self, container_name, blob_name, copy_source,
         '''
         return self._copy_blob(container_name, blob_name, copy_source,
                           metadata,
+                          None,
                           source_if_modified_since, source_if_unmodified_since,
                           source_if_match, source_if_none_match,
                           destination_if_modified_since,
@@ -2993,6 +2994,7 @@ def copy_blob(self, container_name, blob_name, copy_source,
 
     def _copy_blob(self, container_name, blob_name, copy_source,
                   metadata=None,
+                  premium_page_blob_tier=None,
                   source_if_modified_since=None,
                   source_if_unmodified_since=None,
                   source_if_match=None, source_if_none_match=None,
@@ -3053,7 +3055,8 @@ def _copy_blob(self, container_name, blob_name, copy_source,
             'If-Match': _to_str(destination_if_match),
             'If-None-Match': _to_str(destination_if_none_match),
             'x-ms-lease-id': _to_str(destination_lease_id),
-            'x-ms-source-lease-id': _to_str(source_lease_id)
+            'x-ms-source-lease-id': _to_str(source_lease_id),
+            'x-ms-access-tier': _to_str(premium_page_blob_tier)
         }
         _add_metadata_headers(metadata, request)