/// <summary>
/// Asynchronously abort a multipart Upload operation.
/// </summary>
/// <param name="existingBucketName"></param>
/// <param name="keyName"></param>
/// <param name="uploadID"></param>
/// <param name="token"></param>
/// <returns></returns>
public async Task AbortMultiPartUploadAsync(string existingBucketName, string keyName, string uploadId, CancellationToken token)
{
token.ThrowIfCancellationRequested();
_log.Debug("Called AbortMultiPartUploadAsync with parameters keyName = \"" + keyName +
"\", UploadId = \"" + uploadId + "\".");
AbortMultipartUploadRequest request = new AbortMultipartUploadRequest()
{
BucketName = existingBucketName,
Key = keyName,
UploadId = uploadId
};
try
{
await s3Client.AbortMultipartUploadAsync(request, token).ConfigureAwait(false);
}
catch (Exception e)
{
if (!(e is TaskCanceledException || e is OperationCanceledException))
{
string messagePart = " with parameters keyName = \"" + keyName +
"\", UploadId = \"" + uploadId + "\"";
this.LogAmazonException(messagePart, e);
}
throw;
}
}
private static async Task UploadObjectAsync()
{
// Create list to store upload part responses.
List <UploadPartResponse> uploadResponses = new List <UploadPartResponse>();
// Setup information required to initiate the multipart upload.
InitiateMultipartUploadRequest initiateRequest = new InitiateMultipartUploadRequest
{
BucketName = bucketName,
Key = keyName
};
// Initiate the upload.
InitiateMultipartUploadResponse initResponse =
await s3Client.InitiateMultipartUploadAsync(initiateRequest);
// Upload parts.
long contentLength = new FileInfo(filePath).Length;
long partSize = 5 * (long)Math.Pow(2, 20); // 5 MB
try
{
Console.WriteLine("Uploading parts");
long filePosition = 0;
for (int i = 1; filePosition < contentLength; i++)
{
UploadPartRequest uploadRequest = new UploadPartRequest
{
BucketName = bucketName,
Key = keyName,
UploadId = initResponse.UploadId,
PartNumber = i,
PartSize = partSize,
FilePosition = filePosition,
FilePath = filePath
};
// Track upload progress.
uploadRequest.StreamTransferProgress +=
new EventHandler <StreamTransferProgressArgs>(UploadPartProgressEventCallback);
// Upload a part and add the response to our list.
uploadResponses.Add(await s3Client.UploadPartAsync(uploadRequest));
filePosition += partSize;
}
// Setup to complete the upload.
CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest
{
BucketName = bucketName,
Key = keyName,
UploadId = initResponse.UploadId
};
completeRequest.AddPartETags(uploadResponses);
// Complete the upload.
CompleteMultipartUploadResponse completeUploadResponse =
await s3Client.CompleteMultipartUploadAsync(completeRequest);
}
catch (Exception exception)
{
Console.WriteLine("An AmazonS3Exception was thrown: { 0}", exception.Message);
// Abort the upload.
AbortMultipartUploadRequest abortMPURequest = new AbortMultipartUploadRequest
{
BucketName = bucketName,
Key = keyName,
UploadId = initResponse.UploadId
};
await s3Client.AbortMultipartUploadAsync(abortMPURequest);
}
}
/// <summary>
/// Provides the actual implementation to move or copy an S3 object
/// </summary>
/// <param name="client"></param>
/// <param name="request"></param>
/// <param name="partSize"></param>
/// <param name="deleteSource"></param>
/// <returns></returns>
private static async Task <CopyObjectRequestResponse> CopyOrMoveObjectAsync(this IAmazonS3 client, CopyObjectRequest request, long partSize, bool deleteSource, Func <long, long, bool> useMulitpart)
{
/// Handle operation cancelled exceptions
ExponentialBackoffAndRetryClient backoffClient = new ExponentialBackoffAndRetryClient(4, 100, 1000)
{
ExceptionHandlingLogic = (ex) =>
{
if (ex is OperationCanceledException)
{
return(true);
}
else
{
return(false);
}
}
};
try
{
ParameterTests.NonNull(request, "request");
ParameterTests.OutOfRange(partSize >= Constants.MINIMUM_MULTIPART_PART_SIZE, "partSize", $"The part size must be at least {Constants.MINIMUM_MULTIPART_PART_SIZE} bytes.");
ParameterTests.OutOfRange(partSize <= Constants.MAXIMUM_MULTIPART_PART_SIZE, "partSize", $"The part size cannot exceed {Constants.MAXIMUM_MULTIPART_PART_SIZE} bytes.");
if (request.SourceKey == request.DestinationKey &&
request.SourceBucket != null &&
request.SourceBucket.Equals(request.DestinationBucket, StringComparison.OrdinalIgnoreCase))
{
throw new SourceDestinationSameException("The source and destination of the copy operation cannot be the same.", new CopyObjectRequest[] { request });
}
// Get the size of the object.
GetObjectMetadataRequest metadataRequest = new GetObjectMetadataRequest
{
BucketName = request.SourceBucket,
Key = request.SourceKey
};
long objectSize;
GetObjectMetadataResponse metadataResponse;
try
{
metadataResponse = await backoffClient.RunAsync(() => client.GetObjectMetadataAsync(metadataRequest));
objectSize = metadataResponse.ContentLength; // Length in bytes.
}
catch (Exception e)
{
throw e;
}
CopyObjectResponse response = null;
if (UseMultipart(objectSize, partSize))
{
// If it takes more than a 5 GiB part to make 10000 or less parts, than this operation
// isn't supported for an object this size
if (objectSize / partSize > Constants.MAXIMUM_PARTS)
{
throw new NotSupportedException($"The object size, {objectSize}, cannot be broken into fewer than {Constants.MAXIMUM_PARTS} parts using a part size of {partSize} bytes.");
}
List <Task <CopyPartResponse> > copyResponses = new List <Task <CopyPartResponse> >();
// This property has a nullable backing private field that when set to
// anything non-null causes the x-amz-object-lock-retain-until-date
// header to be sent which in turn results in an exception being thrown
// that the Bucket is missing ObjectLockConfiguration
InitiateMultipartUploadRequest initiateRequest = request.ConvertTo <InitiateMultipartUploadRequest>("ObjectLockRetainUntilDate");
initiateRequest.BucketName = request.DestinationBucket;
initiateRequest.Key = request.DestinationKey;
InitiateMultipartUploadResponse initiateResponse = await backoffClient.RunAsync(() => client.InitiateMultipartUploadAsync(initiateRequest));
try
{
long bytePosition = 0;
int counter = 1;
// Launch all of the copy parts
while (bytePosition < objectSize)
{
CopyPartRequest copyRequest = request.ConvertTo <CopyPartRequest>("ObjectLockRetainUntilDate");
copyRequest.UploadId = initiateResponse.UploadId;
copyRequest.FirstByte = bytePosition;
// If we're on the last part, the last byte is the object size minus 1, otherwise the last byte is the part size minus one
// added to the current byte position
copyRequest.LastByte = ((bytePosition + partSize - 1) >= objectSize) ? objectSize - 1 : bytePosition + partSize - 1;
copyRequest.PartNumber = counter++;
copyResponses.Add(backoffClient.RunAsync(() => client.CopyPartAsync(copyRequest)));
bytePosition += partSize;
}
IEnumerable <CopyPartResponse> responses = (await Task.WhenAll(copyResponses)).OrderBy(x => x.PartNumber);
// Set up to complete the copy.
CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest
{
BucketName = request.DestinationBucket,
Key = request.DestinationKey,
UploadId = initiateResponse.UploadId
};
completeRequest.AddPartETags(responses);
// Complete the copy.
CompleteMultipartUploadResponse completeUploadResponse = await backoffClient.RunAsync(() => client.CompleteMultipartUploadAsync(completeRequest));
response = completeUploadResponse.CopyProperties <CopyObjectResponse>();
response.SourceVersionId = metadataResponse.VersionId;
}
catch (AmazonS3Exception e)
{
AbortMultipartUploadRequest abortRequest = new AbortMultipartUploadRequest()
{
BucketName = request.DestinationBucket,
Key = request.DestinationKey,
UploadId = initiateResponse.UploadId
};
await backoffClient.RunAsync(() => client.AbortMultipartUploadAsync(abortRequest));
throw e;
}
}
else
{
response = await backoffClient.RunAsync(() => client.CopyObjectAsync(request));
}
if (response.HttpStatusCode != HttpStatusCode.OK)
{
throw new AmazonS3Exception($"Could not copy object from s3://{request.SourceBucket}/{request.SourceKey} to s3://{request.DestinationBucket}/{request.DestinationKey}. Received response : {(int)response.HttpStatusCode}");
}
else
{
// We already checked to make sure the source and destination weren't the same
// and it's safe to delete the source object
if (deleteSource)
{
DeleteObjectRequest deleteRequest = new DeleteObjectRequest()
{
BucketName = request.SourceBucket,
Key = request.SourceKey
};
DeleteObjectResponse deleteResponse = await backoffClient.RunAsync(() => client.DeleteObjectAsync(deleteRequest));
if (deleteResponse.HttpStatusCode != HttpStatusCode.NoContent)
{
throw new AmazonS3Exception($"Could not delete s3://{request.SourceBucket}/{request.SourceKey}. Received response : {(int)deleteResponse.HttpStatusCode}");
}
}
return(new CopyObjectRequestResponse(request, response));
}
}
catch (Exception e)
{
return(null);
}
}
public async Task <AbortMultipartUploadResponse> AbortMultipartUploadAsync(string bucketName, string key, string uploadId) => await _amazonS3.AbortMultipartUploadAsync(bucketName, key, uploadId);
private async Task MultiPartUploadAsync(Stream sourceStream, string sourceMD5, string bucketName, string keyName, CancellationToken cancellationToken)
{
// Create list to store upload part responses.
var uploadResponses = new List <UploadPartResponse>();
// Setup information required to initiate the multipart upload.
var initiateRequest = new InitiateMultipartUploadRequest
{
BucketName = bucketName,
Key = keyName
};
var validateMD5 = !string.IsNullOrEmpty(sourceMD5);
if (validateMD5)
{
initiateRequest.Metadata.Add("Content-MD5", sourceMD5);
}
// Initiate the upload.
var initResponse = await _amazonS3.InitiateMultipartUploadAsync(initiateRequest, cancellationToken);
try
{
var blobHasher = validateMD5 ? CopyUtils.GetMD5Hasher() : null;
var partSize = 10 * 1024 * 1024; // todo: config
var partNo = 1;
await CopyUtils.CopyAsync(
(buffer, ct) => CopyUtils.ReadStreamMaxBufferAsync(buffer, sourceStream, ct),
async (buffer, count, cancellationToken2) =>
{
var blockMD5Hash = CopyUtils.GetMD5HashString(buffer, 0, count);
if (validateMD5)
{
CopyUtils.AppendMDHasherData(blobHasher, buffer, 0, count);
}
using (var ms = new MemoryStream(buffer, 0, count))
{
ms.Position = 0;
partNo++;
var uploadRequest = new UploadPartRequest
{
BucketName = bucketName,
Key = keyName,
UploadId = initResponse.UploadId,
PartNumber = partNo,
PartSize = count,
InputStream = ms,
MD5Digest = blockMD5Hash
};
// Upload a part and add the response to our list.
var uploadResponse = await _amazonS3.UploadPartAsync(uploadRequest, cancellationToken);
uploadResponses.Add(uploadResponse);
}
},
partSize,
cancellationToken
);
if (validateMD5)
{
var blobHash = CopyUtils.GetMD5HashString(blobHasher);
if ((!string.IsNullOrEmpty(sourceMD5)) && (sourceMD5 != blobHash))
{
throw new Exception("Invalid destination MD5");
}
}
// Setup to complete the upload.
var completeRequest = new CompleteMultipartUploadRequest
{
BucketName = bucketName,
Key = keyName,
UploadId = initResponse.UploadId
};
completeRequest.AddPartETags(uploadResponses);
// Complete the upload.
var completeUploadResponse = await _amazonS3.CompleteMultipartUploadAsync(completeRequest, cancellationToken);
}
catch (Exception exception)
{
Console.WriteLine("An AmazonS3Exception was thrown: { 0}", exception.Message);
// Abort the upload.
var abortMPURequest = new AbortMultipartUploadRequest
{
BucketName = bucketName,
Key = keyName,
UploadId = initResponse.UploadId
};
await _amazonS3.AbortMultipartUploadAsync(abortMPURequest, cancellationToken);
}
}
private static async Task CreateSampleObjUsingClientEncryptionKeyAsync(string base64Key, IAmazonS3 s3Client)
{
// List to store upload part responses.
List <UploadPartResponse> uploadResponses = new List <UploadPartResponse>();
// 1. Initialize.
InitiateMultipartUploadRequest initiateRequest = new InitiateMultipartUploadRequest
{
BucketName = existingBucketName,
Key = sourceKeyName,
ServerSideEncryptionCustomerMethod = ServerSideEncryptionCustomerMethod.AES256,
ServerSideEncryptionCustomerProvidedKey = base64Key
};
InitiateMultipartUploadResponse initResponse =
await s3Client.InitiateMultipartUploadAsync(initiateRequest);
// 2. Upload Parts.
long contentLength = new FileInfo(filePath).Length;
long partSize = 5 * (long)Math.Pow(2, 20); // 5 MB
try
{
long filePosition = 0;
for (int i = 1; filePosition < contentLength; i++)
{
UploadPartRequest uploadRequest = new UploadPartRequest
{
BucketName = existingBucketName,
Key = sourceKeyName,
UploadId = initResponse.UploadId,
PartNumber = i,
PartSize = partSize,
FilePosition = filePosition,
FilePath = filePath,
ServerSideEncryptionCustomerMethod = ServerSideEncryptionCustomerMethod.AES256,
ServerSideEncryptionCustomerProvidedKey = base64Key
};
// Upload part and add response to our list.
uploadResponses.Add(await s3Client.UploadPartAsync(uploadRequest));
filePosition += partSize;
}
// Step 3: complete.
CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest
{
BucketName = existingBucketName,
Key = sourceKeyName,
UploadId = initResponse.UploadId,
//PartETags = new List<PartETag>(uploadResponses)
};
completeRequest.AddPartETags(uploadResponses);
CompleteMultipartUploadResponse completeUploadResponse =
await s3Client.CompleteMultipartUploadAsync(completeRequest);
}
catch (Exception exception)
{
Console.WriteLine("Exception occurred: {0}", exception.Message);
AbortMultipartUploadRequest abortMPURequest = new AbortMultipartUploadRequest
{
BucketName = existingBucketName,
Key = sourceKeyName,
UploadId = initResponse.UploadId
};
await s3Client.AbortMultipartUploadAsync(abortMPURequest);
}
}
/// <summary>
/// Creates and uploads an object using a multi-part upload.
/// </summary>
/// <param name="client">The initialized Amazon S3 object used to
/// initialize and perform the multi-part upload.</param>
/// <param name="existingBucketName">The name of the bucket to which
/// the object will be uploaded.</param>
/// <param name="sourceKeyName">The source object name.</param>
/// <param name="filePath">The location of the source object.</param>
/// <param name="base64Key">The encryption key to use with the upload.</param>
public static async Task CreateSampleObjUsingClientEncryptionKeyAsync(
IAmazonS3 client,
string existingBucketName,
string sourceKeyName,
string filePath,
string base64Key)
{
List <UploadPartResponse> uploadResponses = new List <UploadPartResponse>();
InitiateMultipartUploadRequest initiateRequest = new InitiateMultipartUploadRequest
{
BucketName = existingBucketName,
Key = sourceKeyName,
ServerSideEncryptionCustomerMethod = ServerSideEncryptionCustomerMethod.AES256,
ServerSideEncryptionCustomerProvidedKey = base64Key,
};
InitiateMultipartUploadResponse initResponse =
await client.InitiateMultipartUploadAsync(initiateRequest);
long contentLength = new FileInfo(filePath).Length;
long partSize = 5 * (long)Math.Pow(2, 20); // 5 MB
try
{
long filePosition = 0;
for (int i = 1; filePosition < contentLength; i++)
{
UploadPartRequest uploadRequest = new UploadPartRequest
{
BucketName = existingBucketName,
Key = sourceKeyName,
UploadId = initResponse.UploadId,
PartNumber = i,
PartSize = partSize,
FilePosition = filePosition,
FilePath = filePath,
ServerSideEncryptionCustomerMethod = ServerSideEncryptionCustomerMethod.AES256,
ServerSideEncryptionCustomerProvidedKey = base64Key,
};
// Upload part and add response to our list.
uploadResponses.Add(await client.UploadPartAsync(uploadRequest));
filePosition += partSize;
}
CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest
{
BucketName = existingBucketName,
Key = sourceKeyName,
UploadId = initResponse.UploadId,
};
completeRequest.AddPartETags(uploadResponses);
CompleteMultipartUploadResponse completeUploadResponse =
await client.CompleteMultipartUploadAsync(completeRequest);
}
catch (Exception exception)
{
Console.WriteLine($"Exception occurred: {exception.Message}");
// If there was an error, abort the multi-part upload.
AbortMultipartUploadRequest abortMPURequest = new AbortMultipartUploadRequest
{
BucketName = existingBucketName,
Key = sourceKeyName,
UploadId = initResponse.UploadId,
};
await client.AbortMultipartUploadAsync(abortMPURequest);
}
}
public async Task <string> UploadFileAsync(Stream stream, Action <UploadEvent> callback)
{
var objectKey = $"{_appOptions.UploadBucketPrefix}/{Guid.NewGuid()}";
_logger.LogInformation($"Start uploading to {objectKey}");
var initateResponse = await _s3Client.InitiateMultipartUploadAsync(new InitiateMultipartUploadRequest
{
BucketName = _appOptions.PhotoStorageBucket,
Key = objectKey
});
_logger.LogInformation($"Initiated multi part upload with id {initateResponse.UploadId}");
try
{
using var inputStream = stream;
var partETags = new List <PartETag>();
var readBuffer = ArrayPool <byte> .Shared.Rent(READ_BUFFER_SIZE);
var partBuffer = ArrayPool <byte> .Shared.Rent(PART_SIZE + READ_BUFFER_SIZE * 3);
var callbackEvent = new UploadEvent();
var nextUploadBuffer = new MemoryStream(partBuffer);
try
{
var partNumber = 1;
int readCount;
while ((readCount = await inputStream.ReadAsync(readBuffer, 0, readBuffer.Length)) != 0)
{
callbackEvent.UploadBytes += readCount;
callback?.Invoke(callbackEvent);
await nextUploadBuffer.WriteAsync(readBuffer, 0, readCount);
if (PART_SIZE < nextUploadBuffer.Position)
{
var isLastPart = readCount == READ_BUFFER_SIZE;
var partSize = nextUploadBuffer.Position;
nextUploadBuffer.Position = 0;
var partResponse = await _s3Client.UploadPartAsync(new UploadPartRequest
{
BucketName = _appOptions.PhotoStorageBucket,
Key = objectKey,
UploadId = initateResponse.UploadId,
InputStream = nextUploadBuffer,
PartSize = partSize,
PartNumber = partNumber,
IsLastPart = isLastPart
});
_logger.LogInformation(
$"Uploaded part {partNumber}. (Last part = {isLastPart}, Part size = {partSize}, Upload Id: {initateResponse.UploadId}");
partETags.Add(new PartETag
{
PartNumber = partResponse.PartNumber, ETag = partResponse.ETag
});
partNumber++;
nextUploadBuffer = new MemoryStream(partBuffer);
callbackEvent.UploadParts++;
callback?.Invoke(callbackEvent);
}
}
if (nextUploadBuffer.Position != 0)
{
var partSize = nextUploadBuffer.Position;
nextUploadBuffer.Position = 0;
var partResponse = await _s3Client.UploadPartAsync(new UploadPartRequest
{
BucketName = _appOptions.PhotoStorageBucket,
Key = objectKey,
UploadId = initateResponse.UploadId,
InputStream = nextUploadBuffer,
PartSize = partSize,
PartNumber = partNumber,
IsLastPart = true
});
_logger.LogInformation(
$"Uploaded final part. (Part size = {partSize}, Upload Id: {initateResponse.UploadId})");
partETags.Add(new PartETag {
PartNumber = partResponse.PartNumber, ETag = partResponse.ETag
});
callbackEvent.UploadParts++;
callback?.Invoke(callbackEvent);
}
}
finally
{
ArrayPool <byte> .Shared.Return(partBuffer);
ArrayPool <byte> .Shared.Return(readBuffer);
}
await _s3Client.CompleteMultipartUploadAsync(new CompleteMultipartUploadRequest
{
BucketName = _appOptions.PhotoStorageBucket,
Key = objectKey,
UploadId = initateResponse.UploadId,
PartETags = partETags
});
_logger.LogInformation(
$"Completed multi part upload. (Part count: {partETags.Count}, Upload Id: {initateResponse.UploadId})");
}
catch (Exception e)
{
await _s3Client.AbortMultipartUploadAsync(new AbortMultipartUploadRequest
{
BucketName = _appOptions.PhotoStorageBucket,
Key = objectKey,
UploadId = initateResponse.UploadId
});
_logger.LogError($"Error uploading to S3 with error: {e.Message}");
throw;
}
return(_s3Client.GetPreSignedURL(new GetPreSignedUrlRequest
{
BucketName = _appOptions.PhotoStorageBucket,
Key = objectKey,
Verb = HttpVerb.GET,
Expires = DateTime.UtcNow.AddDays(1)
}));
}
static async Task S3BucketIOWorker(MemoryStream inStream, string fileName)
{
List <UploadPartResponse> uploadResponses = new List <UploadPartResponse>();
InitiateMultipartUploadRequest initiateRequest = new InitiateMultipartUploadRequest
{
BucketName = inBucketName,
Key = fileName
};
InitiateMultipartUploadResponse initResponse =
await s3Client.InitiateMultipartUploadAsync(initiateRequest);
try
{
const long chunkSize = 3145728;
long sLen = inStream.Length;
long partSize;
if (sLen > 0)
{
for (int iii = 1; sLen > (iii - 1) * chunkSize; iii++)
{
if (sLen <= iii * chunkSize)
{
partSize = sLen % (long)chunkSize;
if (partSize == 0)
{
break;
}
}
else
{
partSize = chunkSize;
}
UploadPartRequest uploadRequest = new UploadPartRequest
{
BucketName = inBucketName,
Key = fileName,
UploadId = initResponse.UploadId,
PartNumber = iii,
PartSize = partSize,
InputStream = inStream,
};
retStr += partSize.ToString() + "\n";
uploadResponses.Add(await s3Client.UploadPartAsync(uploadRequest));
}
}
CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest
{
BucketName = inBucketName,
Key = fileName,
UploadId = initResponse.UploadId,
};
completeRequest.AddPartETags(uploadResponses);
CompleteMultipartUploadResponse completeUploadResponse =
await s3Client.CompleteMultipartUploadAsync(completeRequest);
retStr += completeUploadResponse.HttpStatusCode.ToString();
}
catch (Exception exception)
{
retStr += "An AmazonS3Exception was thrown: " + exception.Message;
// Abort the upload.
AbortMultipartUploadRequest abortMPURequest = new AbortMultipartUploadRequest
{
BucketName = inBucketName,
Key = fileName,
UploadId = initResponse.UploadId
};
await s3Client.AbortMultipartUploadAsync(abortMPURequest);
}
//PutObjectRequest putObjReq = new PutObjectRequest
//{
// BucketName = inBucketName,
// Key = fileName,
// InputStream = outStream,
// ContentType = "text/plain",
//};
//using (StreamWriter writer = new StreamWriter(outStream))
//{
// string line;
// PutObjectResponse objResp=(await s3Client.PutObjectAsync(putObjReq));
// objResp.HttpStatusCode = System.Net.HttpStatusCode.Processing;
// while ((line = reader.ReadLine()) != null)
// {
// writer.WriteLine(line);
// }
// objResp.HttpStatusCode = System.Net.HttpStatusCode.OK;
//}
}