Merge pull request #592 from Azure/dev

Release 2.0.0 for Blob, File, and Queue

Author: zezha-msft

azure-storage-blob/ChangeLog.md

@@ -2,6 +2,13 @@
 
 > See [BreakingChanges](BreakingChanges.md) for a detailed list of API breaks.
 
+## Version 2.0.0:
+- Support for 2018-11-09 REST version. Please see our REST API documentation and blog for information about the related added features.
+- Added support for append block from URL(synchronously) for append blobs.
+- Added support for update page from URL(synchronously) for page blobs.
+- Added support for generating and using blob snapshot SAS tokens.
+- Added support for generating user delegation SAS tokens.
+
 ## Version 1.5.0:
 
 - Added new method list_blob_names to efficiently list only blob names in an efficient way.

azure-storage-blob/azure/storage/blob/_constants.py

@@ -5,10 +5,10 @@
 # --------------------------------------------------------------------------
 
 __author__ = 'Microsoft Corp. <[email protected]>'
-__version__ = '1.5.0'
+__version__ = '2.0.0'
 
 # x-ms-version for storage service.
-X_MS_VERSION = '2018-03-28'
+X_MS_VERSION = '2018-11-09'
 
 # internal configurations, should not be changed
 _LARGE_BLOB_UPLOAD_MAX_READ_BUFFER_SIZE = 4 * 1024 * 1024

azure-storage-blob/azure/storage/blob/_deserialization.py

@@ -36,6 +36,7 @@
     ResourceProperties,
     BlobPrefix,
     AccountInformation,
+    UserDelegationKey,
 )
 from ._encryption import _decrypt_blob
 from azure.storage.common.models import _list
@@ -520,3 +521,36 @@ def _parse_account_information(response):
     account_info.account_kind = response.headers['x-ms-account-kind']
 
     return account_info
+
+
+def _convert_xml_to_user_delegation_key(response):
+    """
+    <?xml version="1.0" encoding="utf-8"?>
+    <UserDelegationKey>
+        <SignedOid> Guid </SignedOid>
+        <SignedTid> Guid </SignedTid>
+        <SignedStart> String, formatted ISO Date </SignedStart>
+        <SignedExpiry> String, formatted ISO Date </SignedExpiry>
+        <SignedService>b</SignedService>
+        <SignedVersion> String, rest api version used to create delegation key </SignedVersion>
+        <Value>Ovg+o0K/0/2V8upg7AwlyAPCriEcOSXKuBu2Gv/PU70Y7aWDW3C2ZRmw6kYWqPWBaM1GosLkcSZkgsobAlT+Sw==</value>
+    </UserDelegationKey >
+
+    Converts xml response to UserDelegationKey class.
+    """
+
+    if response is None or response.body is None:
+        return None
+
+    delegation_key = UserDelegationKey()
+
+    key_element = ETree.fromstring(response.body)
+    delegation_key.signed_oid = key_element.findtext('SignedOid')
+    delegation_key.signed_tid = key_element.findtext('SignedTid')
+    delegation_key.signed_start = key_element.findtext('SignedStart')
+    delegation_key.signed_expiry = key_element.findtext('SignedExpiry')
+    delegation_key.signed_service = key_element.findtext('SignedService')
+    delegation_key.signed_version = key_element.findtext('SignedVersion')
+    delegation_key.value = key_element.findtext('Value')
+
+    return delegation_key

azure-storage-blob/azure/storage/blob/_serialization.py

@@ -4,7 +4,7 @@
 # license information.
 # --------------------------------------------------------------------------
 from xml.sax.saxutils import escape as xml_escape
-
+from datetime import date
 try:
     from xml.etree import cElementTree as ETree
 except ImportError:
@@ -13,6 +13,9 @@
     _encode_base64,
     _str,
 )
+from azure.storage.common._serialization import (
+    _to_utc_datetime,
+)
 from azure.storage.common._error import (
     _validate_not_none,
     _ERROR_START_END_NEEDED_FOR_MD5,
@@ -46,7 +49,8 @@ def _get_path(container_name=None, blob_name=None):
 
 
 def _validate_and_format_range_headers(request, start_range, end_range, start_range_required=True,
-                                       end_range_required=True, check_content_md5=False, align_to_page=False):
+                                       end_range_required=True, check_content_md5=False, align_to_page=False,
+                                       range_header_name='x-ms-range'):
     # If end range is provided, start range must be provided
     if start_range_required or end_range is not None:
         _validate_not_none('start_range', start_range)
@@ -63,9 +67,9 @@ def _validate_and_format_range_headers(request, start_range, end_range, start_ra
     # Format based on whether end_range is present
     request.headers = request.headers or {}
     if end_range is not None:
-        request.headers['x-ms-range'] = 'bytes={0}-{1}'.format(start_range, end_range)
+        request.headers[range_header_name] = 'bytes={0}-{1}'.format(start_range, end_range)
     elif start_range is not None:
-        request.headers['x-ms-range'] = "bytes={0}-".format(start_range)
+        request.headers[range_header_name] = "bytes={0}-".format(start_range)
 
     # Content MD5 can only be provided for a complete range less than 4MB in size
     if check_content_md5:
@@ -116,3 +120,34 @@ def _convert_block_list_to_xml(block_id_list):
 
     # return xml value
     return output
+
+
+def _convert_delegation_key_info_to_xml(start_time, expiry_time):
+    """
+    <?xml version="1.0" encoding="utf-8"?>
+    <KeyInfo>
+        <Start> String, formatted ISO Date </Start>
+        <Expiry> String, formatted ISO Date </Expiry>
+    </KeyInfo>
+
+    Convert key info to xml to send.
+    """
+    if start_time is None or expiry_time is None:
+        raise ValueError("delegation key start/end times are required")
+
+    key_info_element = ETree.Element('KeyInfo')
+    ETree.SubElement(key_info_element, 'Start').text = \
+        _to_utc_datetime(start_time) if isinstance(start_time, date) else start_time
+    ETree.SubElement(key_info_element, 'Expiry').text = \
+        _to_utc_datetime(expiry_time) if isinstance(expiry_time, date) else expiry_time
+
+    # Add xml declaration and serialize
+    try:
+        stream = BytesIO()
+        ETree.ElementTree(key_info_element).write(stream, xml_declaration=True, encoding='utf-8', method='xml')
+    finally:
+        output = stream.getvalue()
+        stream.close()
+
+    # return xml value
+    return output

azure-storage-blob/azure/storage/blob/appendblobservice.py

@@ -33,6 +33,7 @@
 )
 from ._serialization import (
     _get_path,
+    _validate_and_format_range_headers,
 )
 from ._upload_chunking import (
     _AppendBlobChunkUploader,
@@ -286,6 +287,125 @@ def append_block(self, container_name, blob_name, block,
 
         return self._perform_request(request, _parse_append_block)
 
+    def append_block_from_url(self, container_name, blob_name, copy_source_url, source_range_start=None,
+                              source_range_end=None, source_content_md5=None, source_if_modified_since=None,
+                              source_if_unmodified_since=None, source_if_match=None,
+                              source_if_none_match=None, maxsize_condition=None,
+                              appendpos_condition=None, lease_id=None, if_modified_since=None,
+                              if_unmodified_since=None, if_match=None,
+                              if_none_match=None, timeout=None):
+        """
+        Creates a new block to be committed as part of a blob, where the contents are read from a source url.
+
+        :param str container_name:
+            Name of existing container.
+        :param str blob_name:
+            Name of blob.
+        :param str copy_source_url:
+            The URL of the source data. It can point to any Azure Blob or File, that is either public or has a
+            shared access signature attached.
+        :param int source_range_start:
+            This indicates the start of the range of bytes(inclusive) that has to be taken from the copy source.
+        :param int source_range_end:
+            This indicates the end of the range of bytes(inclusive) that has to be taken from the copy source.
+        :param str source_content_md5:
+            If given, the service will calculate the MD5 hash of the block content and compare against this value.
+        :param datetime source_if_modified_since:
+            A DateTime value. Azure expects the date value passed in to be UTC.
+            If timezone is included, any non-UTC datetimes will be converted to UTC.
+            If a date is passed in without timezone info, it is assumed to be UTC.
+            Specify this header to perform the operation only
+            if the source resource has been modified since the specified time.
+        :param datetime source_if_unmodified_since:
+            A DateTime value. Azure expects the date value passed in to be UTC.
+            If timezone is included, any non-UTC datetimes will be converted to UTC.
+            If a date is passed in without timezone info, it is assumed to be UTC.
+            Specify this header to perform the operation only if
+            the source resource has not been modified since the specified date/time.
+        :param str source_if_match:
+            An ETag value, or the wildcard character (*). Specify this header to perform
+            the operation only if the source resource's ETag matches the value specified.
+        :param str source_if_none_match:
+            An ETag value, or the wildcard character (*). Specify this header
+            to perform the operation only if the source resource's ETag does not match
+            the value specified. Specify the wildcard character (*) to perform
+            the operation only if the source resource does not exist, and fail the
+            operation if it does exist.
+        :param int maxsize_condition:
+            Optional conditional header. The max length in bytes permitted for
+            the append blob. If the Append Block operation would cause the blob
+            to exceed that limit or if the blob size is already greater than the
+            value specified in this header, the request will fail with
+            MaxBlobSizeConditionNotMet error (HTTP status code 412 - Precondition Failed).
+        :param int appendpos_condition:
+            Optional conditional header, used only for the Append Block operation.
+            A number indicating the byte offset to compare. Append Block will
+            succeed only if the append position is equal to this number. If it
+            is not, the request will fail with the
+            AppendPositionConditionNotMet error
+            (HTTP status code 412 - Precondition Failed).
+        :param str lease_id:
+            Required if the blob has an active lease.
+        :param datetime if_modified_since:
+            A DateTime value. Azure expects the date value passed in to be UTC.
+            If timezone is included, any non-UTC datetimes will be converted to UTC.
+            If a date is passed in without timezone info, it is assumed to be UTC.
+            Specify this header to perform the operation only
+            if the resource has been modified since the specified time.
+        :param datetime if_unmodified_since:
+            A DateTime value. Azure expects the date value passed in to be UTC.
+            If timezone is included, any non-UTC datetimes will be converted to UTC.
+            If a date is passed in without timezone info, it is assumed to be UTC.
+            Specify this header to perform the operation only if
+            the resource has not been modified since the specified date/time.
+        :param str if_match:
+            An ETag value, or the wildcard character (*). Specify this header to perform
+            the operation only if the resource's ETag matches the value specified.
+        :param str if_none_match:
+            An ETag value, or the wildcard character (*). Specify this header
+            to perform the operation only if the resource's ETag does not match
+            the value specified. Specify the wildcard character (*) to perform
+            the operation only if the resource does not exist, and fail the
+            operation if it does exist.
+        :param int timeout:
+            The timeout parameter is expressed in seconds.
+        """
+        _validate_encryption_unsupported(self.require_encryption, self.key_encryption_key)
+        _validate_not_none('container_name', container_name)
+        _validate_not_none('blob_name', blob_name)
+        _validate_not_none('copy_source_url', copy_source_url)
+
+        request = HTTPRequest()
+        request.method = 'PUT'
+        request.host_locations = self._get_host_locations()
+        request.path = _get_path(container_name, blob_name)
+        request.query = {
+            'comp': 'appendblock',
+            'timeout': _int_to_str(timeout),
+        }
+        request.headers = {
+            'x-ms-copy-source': copy_source_url,
+            'x-ms-source-content-md5': source_content_md5,
+            'x-ms-source-if-Modified-Since': _datetime_to_utc_string(source_if_modified_since),
+            'x-ms-source-if-Unmodified-Since': _datetime_to_utc_string(source_if_unmodified_since),
+            'x-ms-source-if-Match': _to_str(source_if_match),
+            'x-ms-source-if-None-Match': _to_str(source_if_none_match),
+            'x-ms-blob-condition-maxsize': _to_str(maxsize_condition),
+            'x-ms-blob-condition-appendpos': _to_str(appendpos_condition),
+            'x-ms-lease-id': _to_str(lease_id),
+            'If-Modified-Since': _datetime_to_utc_string(if_modified_since),
+            'If-Unmodified-Since': _datetime_to_utc_string(if_unmodified_since),
+            'If-Match': _to_str(if_match),
+            'If-None-Match': _to_str(if_none_match)
+        }
+
+        _validate_and_format_range_headers(request, source_range_start, source_range_end,
+                                           start_range_required=False,
+                                           end_range_required=False,
+                                           range_header_name="x-ms-source-range")
+
+        return self._perform_request(request, _parse_append_block)
+
     # ----Convenience APIs----------------------------------------------
 
     def append_blob_from_path(