private async Task CopyStreamToBlobAsync(Stream sourceStream, BlobContainerClient container, string blobName, int bufferSize, string sourceMD5, CancellationToken cancellationToken) { var blob = container.GetBlockBlobClient(blobName); var blobHasher = CopyUtils.GetMD5Hasher(); var blockIDs = new List <string>(); var blockNumber = 0; await CopyUtils.CopyAsync( (buffer, ct) => CopyUtils.ReadStreamMaxBufferAsync(buffer, sourceStream, ct), async (buffer, count, cancellationToken2) => { var blockId = GetBlockId(blockNumber); blockIDs.Add(blockId); var blockMD5Hash = CopyUtils.GetMD5Hash(buffer, 0, count); CopyUtils.AppendMDHasherData(blobHasher, buffer, 0, count); using (var ms = new MemoryStream(buffer, 0, count)) { await blob.StageBlockAsync(blockId, ms, blockMD5Hash, null, null, cancellationToken); } blockNumber++; }, bufferSize, cancellationToken ); var blobHash = CopyUtils.GetMD5Hash(blobHasher); if ((!string.IsNullOrEmpty(sourceMD5)) && (sourceMD5 != CopyUtils.GetMD5HashString(blobHash))) { throw new Exception("Invalid destination MD5"); } var headers = new global::Azure.Storage.Blobs.Models.BlobHttpHeaders() { ContentHash = blobHash }; await blob.CommitBlockListAsync(blockIDs, headers, null, null, null, cancellationToken); }
public async Task CollectAsync(string collectUrl, IEnumerable <KeyValuePair <string, string> > collectHeaders, IIdentityServiceClientInfo collectIdentityServiceClientInfo, string dataCollectionName, string fileName, TimeSpan timeout, bool finishWait, int tryNo, CancellationToken cancellationToken) { int bufferSize = 1 * 1024 * 1024; using (var response = await HttpUtils.SendAsync(collectUrl, HttpMethod.Get, collectUrl, null, collectHeaders, null, timeout, null, HttpCompletionOption.ResponseHeadersRead, cancellationToken)) { var sourceMD5 = response.ContentMD5(); var fullFileName = GetFullFileName(dataCollectionName, fileName); Directory.CreateDirectory(Path.GetDirectoryName(fullFileName)); using (var sourceStream = await response.Content.ReadAsStreamAsync()) { using (var fileStream = new FileStream(fullFileName, FileMode.Create)) { await CopyUtils.CopyAsync( (buffer, ct) => CopyUtils.ReadStreamMaxBufferAsync(buffer, sourceStream, ct), async (buffer, count, cancellationToken2) => await fileStream.WriteAsync(buffer, 0, count, cancellationToken2), bufferSize, cancellationToken ); } } using (var fileStream = new FileStream(fullFileName, FileMode.Open, FileAccess.Read)) { var newMD5 = await CopyUtils.GetMD5HashStringAsync(fileStream, bufferSize, cancellationToken); if ((!string.IsNullOrEmpty(sourceMD5)) && (!string.Equals(newMD5, sourceMD5, StringComparison.InvariantCultureIgnoreCase))) { throw new Exception("Invalid destination MD5"); } File.WriteAllText(GetMD5FileName(fullFileName), newMD5); } } }
private async Task MultiPartUploadAsync(Stream sourceStream, string sourceMD5, string bucketName, string keyName, CancellationToken cancellationToken) { // Create list to store upload part responses. var uploadResponses = new List <UploadPartResponse>(); // Setup information required to initiate the multipart upload. var initiateRequest = new InitiateMultipartUploadRequest { BucketName = bucketName, Key = keyName }; var validateMD5 = !string.IsNullOrEmpty(sourceMD5); if (validateMD5) { initiateRequest.Metadata.Add("Content-MD5", sourceMD5); } // Initiate the upload. var initResponse = await _amazonS3.InitiateMultipartUploadAsync(initiateRequest, cancellationToken); try { var blobHasher = validateMD5 ? CopyUtils.GetMD5Hasher() : null; var partSize = 10 * 1024 * 1024; // todo: config var partNo = 1; await CopyUtils.CopyAsync( (buffer, ct) => CopyUtils.ReadStreamMaxBufferAsync(buffer, sourceStream, ct), async (buffer, count, cancellationToken2) => { var blockMD5Hash = CopyUtils.GetMD5HashString(buffer, 0, count); if (validateMD5) { CopyUtils.AppendMDHasherData(blobHasher, buffer, 0, count); } using (var ms = new MemoryStream(buffer, 0, count)) { ms.Position = 0; partNo++; var uploadRequest = new UploadPartRequest { BucketName = bucketName, Key = keyName, UploadId = initResponse.UploadId, PartNumber = partNo, PartSize = count, InputStream = ms, MD5Digest = blockMD5Hash }; // Upload a part and add the response to our list. var uploadResponse = await _amazonS3.UploadPartAsync(uploadRequest, cancellationToken); uploadResponses.Add(uploadResponse); } }, partSize, cancellationToken ); if (validateMD5) { var blobHash = CopyUtils.GetMD5HashString(blobHasher); if ((!string.IsNullOrEmpty(sourceMD5)) && (sourceMD5 != blobHash)) { throw new Exception("Invalid destination MD5"); } } // Setup to complete the upload. var completeRequest = new CompleteMultipartUploadRequest { BucketName = bucketName, Key = keyName, UploadId = initResponse.UploadId }; completeRequest.AddPartETags(uploadResponses); // Complete the upload. var completeUploadResponse = await _amazonS3.CompleteMultipartUploadAsync(completeRequest, cancellationToken); } catch (Exception exception) { Console.WriteLine("An AmazonS3Exception was thrown: { 0}", exception.Message); // Abort the upload. var abortMPURequest = new AbortMultipartUploadRequest { BucketName = bucketName, Key = keyName, UploadId = initResponse.UploadId }; await _amazonS3.AbortMultipartUploadAsync(abortMPURequest, cancellationToken); } }