private async Task CopyStreamToBlobAsync(Stream sourceStream, BlobContainerClient container, string blobName, int bufferSize, string sourceMD5, CancellationToken cancellationToken)
{
var blob = container.GetBlockBlobClient(blobName);
var blobHasher = CopyUtils.GetMD5Hasher();
var blockIDs = new List <string>();
var blockNumber = 0;
await CopyUtils.CopyAsync(
(buffer, ct) => CopyUtils.ReadStreamMaxBufferAsync(buffer, sourceStream, ct),
async (buffer, count, cancellationToken2) =>
{
var blockId = GetBlockId(blockNumber);
blockIDs.Add(blockId);
var blockMD5Hash = CopyUtils.GetMD5Hash(buffer, 0, count);
CopyUtils.AppendMDHasherData(blobHasher, buffer, 0, count);
using (var ms = new MemoryStream(buffer, 0, count))
{
await blob.StageBlockAsync(blockId, ms, blockMD5Hash, null, null, cancellationToken);
}
blockNumber++;
},
bufferSize,
cancellationToken
);
var blobHash = CopyUtils.GetMD5Hash(blobHasher);
if ((!string.IsNullOrEmpty(sourceMD5)) && (sourceMD5 != CopyUtils.GetMD5HashString(blobHash)))
{
throw new Exception("Invalid destination MD5");
}
var headers = new global::Azure.Storage.Blobs.Models.BlobHttpHeaders()
{
ContentHash = blobHash
};
await blob.CommitBlockListAsync(blockIDs, headers, null, null, null, cancellationToken);
}
private async Task MultiPartUploadAsync(Stream sourceStream, string sourceMD5, string bucketName, string keyName, CancellationToken cancellationToken)
{
// Create list to store upload part responses.
var uploadResponses = new List <UploadPartResponse>();
// Setup information required to initiate the multipart upload.
var initiateRequest = new InitiateMultipartUploadRequest
{
BucketName = bucketName,
Key = keyName
};
var validateMD5 = !string.IsNullOrEmpty(sourceMD5);
if (validateMD5)
{
initiateRequest.Metadata.Add("Content-MD5", sourceMD5);
}
// Initiate the upload.
var initResponse = await _amazonS3.InitiateMultipartUploadAsync(initiateRequest, cancellationToken);
try
{
var blobHasher = validateMD5 ? CopyUtils.GetMD5Hasher() : null;
var partSize = 10 * 1024 * 1024; // todo: config
var partNo = 1;
await CopyUtils.CopyAsync(
(buffer, ct) => CopyUtils.ReadStreamMaxBufferAsync(buffer, sourceStream, ct),
async (buffer, count, cancellationToken2) =>
{
var blockMD5Hash = CopyUtils.GetMD5HashString(buffer, 0, count);
if (validateMD5)
{
CopyUtils.AppendMDHasherData(blobHasher, buffer, 0, count);
}
using (var ms = new MemoryStream(buffer, 0, count))
{
ms.Position = 0;
partNo++;
var uploadRequest = new UploadPartRequest
{
BucketName = bucketName,
Key = keyName,
UploadId = initResponse.UploadId,
PartNumber = partNo,
PartSize = count,
InputStream = ms,
MD5Digest = blockMD5Hash
};
// Upload a part and add the response to our list.
var uploadResponse = await _amazonS3.UploadPartAsync(uploadRequest, cancellationToken);
uploadResponses.Add(uploadResponse);
}
},
partSize,
cancellationToken
);
if (validateMD5)
{
var blobHash = CopyUtils.GetMD5HashString(blobHasher);
if ((!string.IsNullOrEmpty(sourceMD5)) && (sourceMD5 != blobHash))
{
throw new Exception("Invalid destination MD5");
}
}
// Setup to complete the upload.
var completeRequest = new CompleteMultipartUploadRequest
{
BucketName = bucketName,
Key = keyName,
UploadId = initResponse.UploadId
};
completeRequest.AddPartETags(uploadResponses);
// Complete the upload.
var completeUploadResponse = await _amazonS3.CompleteMultipartUploadAsync(completeRequest, cancellationToken);
}
catch (Exception exception)
{
Console.WriteLine("An AmazonS3Exception was thrown: { 0}", exception.Message);
// Abort the upload.
var abortMPURequest = new AbortMultipartUploadRequest
{
BucketName = bucketName,
Key = keyName,
UploadId = initResponse.UploadId
};
await _amazonS3.AbortMultipartUploadAsync(abortMPURequest, cancellationToken);
}
}
