private async Task CopyStreamToBlobAsync(Stream sourceStream, BlobContainerClient container, string blobName, int bufferSize, string sourceMD5, CancellationToken cancellationToken) { var blob = container.GetBlockBlobClient(blobName); var blobHasher = CopyUtils.GetMD5Hasher(); var blockIDs = new List <string>(); var blockNumber = 0; await CopyUtils.CopyAsync( (buffer, ct) => CopyUtils.ReadStreamMaxBufferAsync(buffer, sourceStream, ct), async (buffer, count, cancellationToken2) => { var blockId = GetBlockId(blockNumber); blockIDs.Add(blockId); var blockMD5Hash = CopyUtils.GetMD5Hash(buffer, 0, count); CopyUtils.AppendMDHasherData(blobHasher, buffer, 0, count); using (var ms = new MemoryStream(buffer, 0, count)) { await blob.StageBlockAsync(blockId, ms, blockMD5Hash, null, null, cancellationToken); } blockNumber++; }, bufferSize, cancellationToken ); var blobHash = CopyUtils.GetMD5Hash(blobHasher); if ((!string.IsNullOrEmpty(sourceMD5)) && (sourceMD5 != CopyUtils.GetMD5HashString(blobHash))) { throw new Exception("Invalid destination MD5"); } var headers = new global::Azure.Storage.Blobs.Models.BlobHttpHeaders() { ContentHash = blobHash }; await blob.CommitBlockListAsync(blockIDs, headers, null, null, null, cancellationToken); }
private async Task MultiPartUploadAsync(Stream sourceStream, string sourceMD5, string bucketName, string keyName, CancellationToken cancellationToken) { // Create list to store upload part responses. var uploadResponses = new List <UploadPartResponse>(); // Setup information required to initiate the multipart upload. var initiateRequest = new InitiateMultipartUploadRequest { BucketName = bucketName, Key = keyName }; var validateMD5 = !string.IsNullOrEmpty(sourceMD5); if (validateMD5) { initiateRequest.Metadata.Add("Content-MD5", sourceMD5); } // Initiate the upload. var initResponse = await _amazonS3.InitiateMultipartUploadAsync(initiateRequest, cancellationToken); try { var blobHasher = validateMD5 ? CopyUtils.GetMD5Hasher() : null; var partSize = 10 * 1024 * 1024; // todo: config var partNo = 1; await CopyUtils.CopyAsync( (buffer, ct) => CopyUtils.ReadStreamMaxBufferAsync(buffer, sourceStream, ct), async (buffer, count, cancellationToken2) => { var blockMD5Hash = CopyUtils.GetMD5HashString(buffer, 0, count); if (validateMD5) { CopyUtils.AppendMDHasherData(blobHasher, buffer, 0, count); } using (var ms = new MemoryStream(buffer, 0, count)) { ms.Position = 0; partNo++; var uploadRequest = new UploadPartRequest { BucketName = bucketName, Key = keyName, UploadId = initResponse.UploadId, PartNumber = partNo, PartSize = count, InputStream = ms, MD5Digest = blockMD5Hash }; // Upload a part and add the response to our list. var uploadResponse = await _amazonS3.UploadPartAsync(uploadRequest, cancellationToken); uploadResponses.Add(uploadResponse); } }, partSize, cancellationToken ); if (validateMD5) { var blobHash = CopyUtils.GetMD5HashString(blobHasher); if ((!string.IsNullOrEmpty(sourceMD5)) && (sourceMD5 != blobHash)) { throw new Exception("Invalid destination MD5"); } } // Setup to complete the upload. var completeRequest = new CompleteMultipartUploadRequest { BucketName = bucketName, Key = keyName, UploadId = initResponse.UploadId }; completeRequest.AddPartETags(uploadResponses); // Complete the upload. var completeUploadResponse = await _amazonS3.CompleteMultipartUploadAsync(completeRequest, cancellationToken); } catch (Exception exception) { Console.WriteLine("An AmazonS3Exception was thrown: { 0}", exception.Message); // Abort the upload. var abortMPURequest = new AbortMultipartUploadRequest { BucketName = bucketName, Key = keyName, UploadId = initResponse.UploadId }; await _amazonS3.AbortMultipartUploadAsync(abortMPURequest, cancellationToken); } }