Skip to content

Generate copy part #3951

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Aug 18, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions generator/.DevConfigs/fc91462d-3430-4b70-bc03-4363f2696619.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
{
"services": [
{
"serviceName": "S3",
"type": "patch",
"changeLogMessages": [
"Generate CopyPart.",
"[Breaking Change] Amazon.S3.Model.CopyPartResponse's BucketKeyEnabled will now return bool? instead of bool, in line with other S3 nullable types."
]
}
]
}
15 changes: 0 additions & 15 deletions generator/ServiceClientGeneratorLib/Customizations.cs
Original file line number Diff line number Diff line change
Expand Up @@ -1066,7 +1066,6 @@ public ShapeModifier(JsonData data)
_shapeModifierXmlNamespace = ParseXmlNamespace(data);
_predicateListUnmarshallers = ParsePredicateListUnmarshallers(data);
_excludedUnmarshallingProperties = ParseExcludedUnmarshallingProperties(data);
Validate(data);
}


Expand All @@ -1087,20 +1086,6 @@ public ShapeModifier(JsonData data)
// }
// },
// Above is an example of how to specify the originalMember on the injected member.
private void Validate(JsonData data)
Copy link
Contributor Author

@peterrsongg peterrsongg Aug 12, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I added this when I was working on splitting GetACL PutACL into their respective GetObjectACL and GetBucketACL a while back because I thought if we were excluding a member we would always be injecting another member and referencing the original member that the injected member is referencing. However, as I started to generate more S3 operations I realize that this is not always the case. Sometimes we exclude a member and inject a member but that member doesn't reference an "originalMember" at all.

{
// if a property was excluded, then the injected member must reference the original member that it is replacing
// unless the originalMember isn't part of the members array and is from a different shape.
if (_excludedProperties.Count == 0 || _injectedProperties.Count == 0)
return;
int injectedPropertyOriginalMemberCount = _injectedProperties.Values
.Count(jsonData => jsonData[CustomizationsModel.OriginalMemberKey] != null || jsonData[ShapeModifier.OriginalMemberIsOutsideContainingShapeKey] != null);
if (_excludedProperties.Count != injectedPropertyOriginalMemberCount)
{
throw new InvalidDataException($"The customization excludes and injects members without specifying the originalMember. If you are excluding a member and injecting a different member, make sure to specify the original member that" +
$" was excluded on the injected member in the customizations.json file.");
}
}

#region Property Exclusion

Expand Down

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
Expand Up @@ -445,6 +445,8 @@ using Amazon.Runtime.Internal.Util;
if (dataTypeOverride.IsFlattened)
dataTypeSwapIsFlattened = true;
}
if (this.Config.ServiceModel.Customizations.FlattenShapes(member.OwningShape.Name).Contains(member.ModeledName))
return;
string unmarshalledVariable = isStructure ? "unmarshalledObject" : "response";
if (member.Shape.IsList || isDataTypeSwapList)
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -298,6 +298,19 @@ public override AmazonWebServiceResponse Unmarshall(XmlUnmarshallerContext conte
#line 135 "C:\Dev\Repos\aws-sdk-net-staging\generator\ServiceClientGeneratorLib\Generators\Marshallers\RestXmlResponseUnmarshaller.tt"

}
if (payload != null && this.Config.ServiceModel.Customizations.FlattenShapes(this.Structure.Name).Contains(payload.ModeledName))
{


#line default
#line hidden
this.Write(" // payload has been flattened so that members of the result object ar" +
"e directly unmarshalled instead of unmarshalled\r\n // as part of the c" +
"ontainer class, so look one level deeper\r\n targetDepth += 1;\r\n");

#line 143 "C:\Dev\Repos\aws-sdk-net-staging\generator\ServiceClientGeneratorLib\Generators\Marshallers\RestXmlResponseUnmarshaller.tt"

}
}


Expand All @@ -307,7 +320,7 @@ public override AmazonWebServiceResponse Unmarshall(XmlUnmarshallerContext conte
"\n }\r\n while (context.Read())\r\n {\r\n\t\t\t\tif (conte" +
"xt.IsStartElement || context.IsAttribute)\r\n {\r\n");

#line 147 "C:\Dev\Repos\aws-sdk-net-staging\generator\ServiceClientGeneratorLib\Generators\Marshallers\RestXmlResponseUnmarshaller.tt"
#line 155 "C:\Dev\Repos\aws-sdk-net-staging\generator\ServiceClientGeneratorLib\Generators\Marshallers\RestXmlResponseUnmarshaller.tt"

foreach (var member in this.Operation.ResponseBodyMembers)
{
Expand All @@ -326,7 +339,7 @@ public override AmazonWebServiceResponse Unmarshall(XmlUnmarshallerContext conte
"\r\n {\r\n return;\r\n }\r\n " +
" }\r\n return;\r\n }\r\n");

#line 166 "C:\Dev\Repos\aws-sdk-net-staging\generator\ServiceClientGeneratorLib\Generators\Marshallers\RestXmlResponseUnmarshaller.tt"
#line 174 "C:\Dev\Repos\aws-sdk-net-staging\generator\ServiceClientGeneratorLib\Generators\Marshallers\RestXmlResponseUnmarshaller.tt"

}

Expand All @@ -346,7 +359,7 @@ public override AmazonServiceException UnmarshallException(XmlUnmarshallerContex
{
");

#line 179 "C:\Dev\Repos\aws-sdk-net-staging\generator\ServiceClientGeneratorLib\Generators\Marshallers\RestXmlResponseUnmarshaller.tt"
#line 187 "C:\Dev\Repos\aws-sdk-net-staging\generator\ServiceClientGeneratorLib\Generators\Marshallers\RestXmlResponseUnmarshaller.tt"

if (this.Config.ServiceId == "S3")
{
Expand All @@ -357,7 +370,7 @@ public override AmazonServiceException UnmarshallException(XmlUnmarshallerContex
this.Write(" S3ErrorResponse errorResponse = S3ErrorResponseUnmarshaller.Instance." +
"Unmarshall(context);\r\n");

#line 184 "C:\Dev\Repos\aws-sdk-net-staging\generator\ServiceClientGeneratorLib\Generators\Marshallers\RestXmlResponseUnmarshaller.tt"
#line 192 "C:\Dev\Repos\aws-sdk-net-staging\generator\ServiceClientGeneratorLib\Generators\Marshallers\RestXmlResponseUnmarshaller.tt"

}
else
Expand All @@ -369,7 +382,7 @@ public override AmazonServiceException UnmarshallException(XmlUnmarshallerContex
this.Write(" ErrorResponse errorResponse = XmlErrorResponseUnmarshaller.GetInstanc" +
"e().Unmarshall(context);\r\n");

#line 190 "C:\Dev\Repos\aws-sdk-net-staging\generator\ServiceClientGeneratorLib\Generators\Marshallers\RestXmlResponseUnmarshaller.tt"
#line 198 "C:\Dev\Repos\aws-sdk-net-staging\generator\ServiceClientGeneratorLib\Generators\Marshallers\RestXmlResponseUnmarshaller.tt"

}

Expand All @@ -386,7 +399,7 @@ public override AmazonServiceException UnmarshallException(XmlUnmarshallerContex
{
");

#line 201 "C:\Dev\Repos\aws-sdk-net-staging\generator\ServiceClientGeneratorLib\Generators\Marshallers\RestXmlResponseUnmarshaller.tt"
#line 209 "C:\Dev\Repos\aws-sdk-net-staging\generator\ServiceClientGeneratorLib\Generators\Marshallers\RestXmlResponseUnmarshaller.tt"

foreach (var exception in this.Operation.Exceptions)
{
Expand All @@ -396,22 +409,22 @@ public override AmazonServiceException UnmarshallException(XmlUnmarshallerContex
#line hidden
this.Write(" if (errorResponse.Code != null && errorResponse.Code.Equals(\"");

#line 205 "C:\Dev\Repos\aws-sdk-net-staging\generator\ServiceClientGeneratorLib\Generators\Marshallers\RestXmlResponseUnmarshaller.tt"
#line 213 "C:\Dev\Repos\aws-sdk-net-staging\generator\ServiceClientGeneratorLib\Generators\Marshallers\RestXmlResponseUnmarshaller.tt"
this.Write(this.ToStringHelper.ToStringWithCulture(exception.Code));

#line default
#line hidden
this.Write("\"))\r\n {\r\n return ");

#line 207 "C:\Dev\Repos\aws-sdk-net-staging\generator\ServiceClientGeneratorLib\Generators\Marshallers\RestXmlResponseUnmarshaller.tt"
#line 215 "C:\Dev\Repos\aws-sdk-net-staging\generator\ServiceClientGeneratorLib\Generators\Marshallers\RestXmlResponseUnmarshaller.tt"
this.Write(this.ToStringHelper.ToStringWithCulture(exception.Name));

#line default
#line hidden
this.Write("Unmarshaller.Instance.Unmarshall(contextCopy, errorResponse);\r\n }\r" +
"\n");

#line 209 "C:\Dev\Repos\aws-sdk-net-staging\generator\ServiceClientGeneratorLib\Generators\Marshallers\RestXmlResponseUnmarshaller.tt"
#line 217 "C:\Dev\Repos\aws-sdk-net-staging\generator\ServiceClientGeneratorLib\Generators\Marshallers\RestXmlResponseUnmarshaller.tt"

}

Expand All @@ -420,7 +433,7 @@ public override AmazonServiceException UnmarshallException(XmlUnmarshallerContex
#line hidden
this.Write(" }\r\n");

#line 213 "C:\Dev\Repos\aws-sdk-net-staging\generator\ServiceClientGeneratorLib\Generators\Marshallers\RestXmlResponseUnmarshaller.tt"
#line 221 "C:\Dev\Repos\aws-sdk-net-staging\generator\ServiceClientGeneratorLib\Generators\Marshallers\RestXmlResponseUnmarshaller.tt"

if (this.Config.ServiceId == "S3")
{
Expand All @@ -431,7 +444,7 @@ public override AmazonServiceException UnmarshallException(XmlUnmarshallerContex
this.Write(" return base.ConstructS3Exception(context, errorResponse, innerExcepti" +
"on, statusCode);\r\n");

#line 218 "C:\Dev\Repos\aws-sdk-net-staging\generator\ServiceClientGeneratorLib\Generators\Marshallers\RestXmlResponseUnmarshaller.tt"
#line 226 "C:\Dev\Repos\aws-sdk-net-staging\generator\ServiceClientGeneratorLib\Generators\Marshallers\RestXmlResponseUnmarshaller.tt"

}
else
Expand All @@ -442,15 +455,15 @@ public override AmazonServiceException UnmarshallException(XmlUnmarshallerContex
#line hidden
this.Write(" return new ");

#line 223 "C:\Dev\Repos\aws-sdk-net-staging\generator\ServiceClientGeneratorLib\Generators\Marshallers\RestXmlResponseUnmarshaller.tt"
#line 231 "C:\Dev\Repos\aws-sdk-net-staging\generator\ServiceClientGeneratorLib\Generators\Marshallers\RestXmlResponseUnmarshaller.tt"
this.Write(this.ToStringHelper.ToStringWithCulture(this.BaseException));

#line default
#line hidden
this.Write("(errorResponse.Message, innerException, errorResponse.Type, errorResponse.Code, e" +
"rrorResponse.RequestId, statusCode);\r\n");

#line 224 "C:\Dev\Repos\aws-sdk-net-staging\generator\ServiceClientGeneratorLib\Generators\Marshallers\RestXmlResponseUnmarshaller.tt"
#line 232 "C:\Dev\Repos\aws-sdk-net-staging\generator\ServiceClientGeneratorLib\Generators\Marshallers\RestXmlResponseUnmarshaller.tt"

}

Expand All @@ -460,14 +473,14 @@ public override AmazonServiceException UnmarshallException(XmlUnmarshallerContex
this.Write(" }\r\n\r\n partial void PostUnmarshallCustomization(XmlUnmarshallerCont" +
"ext context, ");

#line 229 "C:\Dev\Repos\aws-sdk-net-staging\generator\ServiceClientGeneratorLib\Generators\Marshallers\RestXmlResponseUnmarshaller.tt"
#line 237 "C:\Dev\Repos\aws-sdk-net-staging\generator\ServiceClientGeneratorLib\Generators\Marshallers\RestXmlResponseUnmarshaller.tt"
this.Write(this.ToStringHelper.ToStringWithCulture(this.Operation.Name));

#line default
#line hidden
this.Write("Response response);\r\n\r\n");

#line 231 "C:\Dev\Repos\aws-sdk-net-staging\generator\ServiceClientGeneratorLib\Generators\Marshallers\RestXmlResponseUnmarshaller.tt"
#line 239 "C:\Dev\Repos\aws-sdk-net-staging\generator\ServiceClientGeneratorLib\Generators\Marshallers\RestXmlResponseUnmarshaller.tt"

this.AddResponseSingletonMethod();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,14 @@ namespace <#=this.Config.Namespace #>.Model.Internal.MarshallTransformations
#>
if (context.IsStartOfDocument)
targetDepth += 1;
<#
}
if (payload != null && this.Config.ServiceModel.Customizations.FlattenShapes(this.Structure.Name).Contains(payload.ModeledName))
{
#>
// payload has been flattened so that members of the result object are directly unmarshalled instead of unmarshalled
// as part of the container class, so look one level deeper
targetDepth += 1;
<#
}
}
Expand Down
2 changes: 1 addition & 1 deletion generator/ServiceClientGeneratorLib/ServiceModel.cs
Original file line number Diff line number Diff line change
Expand Up @@ -546,7 +546,7 @@ public List<Operation> S3AllowListOperations
new Operation(this, "PutBucketLifecycleConfiguration", DocumentRoot[OperationsKey]["PutBucketLifecycleConfiguration"]),
new Operation(this, "PutBucketNotificationConfiguration", DocumentRoot[OperationsKey]["PutBucketNotificationConfiguration"]),
new Operation(this, "PutObjectAcl", DocumentRoot[OperationsKey]["PutObjectAcl"]),
//////new Operation(this, "UploadPartCopy", DocumentRoot[OperationsKey]["UploadPartCopy"]),
new Operation(this, "UploadPartCopy", DocumentRoot[OperationsKey]["UploadPartCopy"]),
new Operation(this, "ListObjectsV2", DocumentRoot[OperationsKey]["ListObjectsV2"]),

};
Expand Down
101 changes: 89 additions & 12 deletions generator/ServiceModels/s3/s3.customizations.json
Original file line number Diff line number Diff line change
Expand Up @@ -388,41 +388,82 @@
"UploadPartCopyRequest": {
"modify": [
{
"Bucket": { "emitPropertyName": "DestinationBucket" }
"CopySourceIfModifiedSince":{"emitPropertyName":"ModifiedSinceDate"}
},
{
"CopySourceIfUnmodifiedSince":{"emitPropertyName":"UnmodifiedSinceDate"}
},
{
"SSECustomerAlgorithm" : {"emitPropertyName": "ServerSideEncryptionCustomerMethod"}
},
{
"SSECustomerKey" : {"emitPropertyName" : "ServerSideEncryptionCustomerProvidedKey"}
},
{
"SSECustomerKeyMD5" : {"emitPropertyName":"ServerSideEncryptionCustomerProvidedKeyMD5"}
},
{
"CopySourceSSECustomerAlgorithm" : {"emitPropertyName" : "CopySourceServerSideEncryptionCustomerMethod"}
},
{
"CopySourceSSECustomerKey" : {"emitPropertyName" : "CopySourceServerSideEncryptionCustomerProvidedKey"}
},
{
"CopySourceSSECustomerKeyMD5" : {"emitPropertyName":"CopySourceServerSideEncryptionCustomerProvidedKeyMD5"}
},
{
"CopySourceIfMatch" : {"emitPropertyName" : "ETagToMatch"}
},
{
"CopySourceIfNoneMatch" : {"emitPropertyName" : "ETagsToNotMatch"}
},
{
"Bucket" : {"emitPropertyName" : "DestinationBucket"}
}
],
"exclude": [
"CopySource",
"Key"
"Key",
"CopySourceRange"
],
Copy link
Preview

Copilot AI Aug 12, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The CopySourceRange property is excluded from generation but there's no corresponding inject or modify entry to handle the FirstByte/LastByte range logic. This could result in missing functionality for byte range copy operations.

Suggested change
],
],
"modify": [
{
"CopySourceRange": {
"emitPropertyName": "CopySourceRange",
"emitAsHeader": "x-amz-copy-source-range",
"transform": "bytes={FirstByte}-{LastByte}"
}
}
],

Copilot uses AI. Check for mistakes.

Copy link
Contributor

@GarrettBeatty GarrettBeatty Aug 14, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

does this need any changes? not really 100% sure what its saying but please confirm @peterrsongg (see copilots other comment)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nope, the ai is partially right here. I left FirstByte and LastByte in the custom file and added it to the custom partial. There is no need to "inject" it because FirstByte and LastByte are not modeled, so there is nothing to inject

"inject": [
{
"SourceBucket": {
"shape": "BucketName"
"shape": "BucketName",
"location":"header"
}
},
{
"SourceKey": {
"shape": "ObjectKey",
"originalMember":"CopySource"
"originalMember":"CopySource",
"location":"header"
}
},
{
"SourceVersionId": {
"shape": "CopySourceVersionId"
}
},
{
"DestinationBucket": {
"shape": "BucketName"
"shape": "CopySourceVersionId",
"location":"header"
}
},
{
"DestinationKey": {
"shape": "ObjectKey",
"originalMember": "Key"
"originalMember": "Key",
"location":"header"
}
}
],
"excludeFromMarshalling":[
"ServerSideEncryptionCustomerProvidedKey",
"ServerSideEncryptionCustomerProvidedKeyMD5",
"CopySourceServerSideEncryptionCustomerProvidedKey",
"CopySourceServerSideEncryptionCustomerProvidedKeyMD5",
"SourceKey",
"SourceBucket",
"DestinationKey",
"DestinationBucket",
"SourceVersionId"
]
},
"S3Grantee":{
Expand Down Expand Up @@ -636,6 +677,16 @@
"EncodingType" : {"emitPropertyName" : "Encoding"}
}
]
},
"UploadPartCopyOutput" :{
"modify" : [
{
"SSEKMSKeyId" : {"emitPropertyName" : "ServerSideEncryptionKeyManagementServiceKeyId"}
},
{
"ServerSideEncryption" : {"emitPropertyName" : "ServerSideEncryptionMethod"}
}
]
}
},
"operationModifiers": {
Expand Down Expand Up @@ -896,6 +947,28 @@
"Marshaller": "StringUtils.FromString",
"Unmarshaller" : "CommonPrefixesItemUnmarshaller"
}
},
"UploadPartCopyRequest":{
"ServerSideEncryptionCustomerMethod" : {
"Type": "ServerSideEncryptionCustomerMethod",
"Marshaller": "StringUtils.FromString",
"Unmarshaller": "StringUnmarshaller"
},
"CopySourceServerSideEncryptionCustomerMethod":{
"Type": "ServerSideEncryptionCustomerMethod",
"Marshaller": "StringUtils.FromString",
"Unmarshaller": "StringUnmarshaller"
},
"ETagToMatch":{
"Type" : "List<string>",
"Marshaller": "Amazon.Util.AWSSDKUtils.Join",
"Unmarshaller": "StringUnmarshaller"
},
"ETagsToNotMatch" : {
"Type" : "List<string>",
"Marshaller" : "Amazon.Util.AWSSDKUtils.Join",
"Unmarshaller" : "StringUnmarshaller"
}
}
},
"excludeMembers":{
Expand Down Expand Up @@ -971,10 +1044,14 @@
"flattenShapes" : {
"PutBucketNotificationConfigurationRequest" : [
"NotificationConfiguration"
],
"UploadPartCopyOutput" : [
"CopyPartResult"
]
},
"excludeShapes":[
"NotificationConfiguration"
"NotificationConfiguration",
"CopyPartResult"
],
"implementsVisitorPattern":{
"LifecycleFilter":{
Expand Down
Loading