public IActionResult GetContentIDs(string containerName, [FromQuery] DateTimeOffset?after, [FromQuery] string maxCount, [FromQuery] string afterContentID, [FromQuery] string auth, [FromHeader] string authenticationKey, CancellationToken cancellationToken) { var intMaxCount = string.IsNullOrEmpty(maxCount) ? default(int?) : int.Parse(maxCount); return(new FileCallbackResult( new MediaTypeHeaderValue("text/plain"), async(outputStream, _) => { using (var contentIDsEnumerator = _contents.GetContentIDs(containerName, after, intMaxCount, afterContentID, auth ?? authenticationKey).GetEnumerator()) { await CopyUtils.CopyAsync( (buffer, ct) => { var byteCount = 0; while ((byteCount < buffer.Length - 1000) && (contentIDsEnumerator.MoveNext())) { var enumBuff = Encoding.UTF8.GetBytes(contentIDsEnumerator.Current + Environment.NewLine); enumBuff.CopyTo(buffer, byteCount); byteCount += enumBuff.Length; } return Task.FromResult(byteCount); }, async(buffer, count, cancellationToken2) => { await outputStream.WriteAsync(buffer, 0, count, cancellationToken2); }, 1_000_000, cancellationToken ); } })); }
private IEnumerable <KeyValuePair <string, string> > MultiPartUpload(IMultiPartIDUploadInfo multiPartIDUploadInfo, Stream stream, long streamSize) { var result = new List <KeyValuePair <string, string> >(); if (multiPartIDUploadInfo.PartSize <= 0) { System.Diagnostics.Debug.Assert(multiPartIDUploadInfo.PartUploadInfos.Count() == 1); UploadPart(multiPartIDUploadInfo.HttpMethod, stream, streamSize, multiPartIDUploadInfo.PartUploadInfos.First(), ""); } else { int writePartIndex = 0; CopyUtils.CopyAsync( async(buffer, cancellationToken) => { return(await CopyUtils.FillBufferAsync(buffer, async(buf, offset, ct) => { return await stream.ReadAsync(buf, offset, (int)multiPartIDUploadInfo.PartSize - offset); }, cancellationToken )); }, (buffer, size, cancellationToken) => { var partUploadInfo = multiPartIDUploadInfo.PartUploadInfos.ElementAt(writePartIndex); var uploadPartResult = UploadPart(multiPartIDUploadInfo.HttpMethod, new MemoryStream(buffer), size, partUploadInfo, multiPartIDUploadInfo.MultiPartUploadResultHeaderName); result.Add(new KeyValuePair <string, string>(partUploadInfo.Identifier, uploadPartResult)); writePartIndex++; return(Task.CompletedTask); }, (int)multiPartIDUploadInfo.PartSize, CancellationToken.None ) .Wait(); //foreach (var partUploadInfo in multiPartIDUploadInfo.PartUploadInfos) //{ // var uploadPartResult = UploadPart(multiPartIDUploadInfo.HttpMethod, stream, multiPartIDUploadInfo.PartSize, partUploadInfo, multiPartIDUploadInfo.MultiPartUploadResultHeaderName); // yield return new KeyValuePair<string, string>(partUploadInfo.Identifier, uploadPartResult); //} } return(result); }
private async Task CopyStreamToBlobAsync(Stream sourceStream, BlobContainerClient container, string blobName, int bufferSize, string sourceMD5, CancellationToken cancellationToken) { var blob = container.GetBlockBlobClient(blobName); var blobHasher = CopyUtils.GetMD5Hasher(); var blockIDs = new List <string>(); var blockNumber = 0; await CopyUtils.CopyAsync( (buffer, ct) => CopyUtils.ReadStreamMaxBufferAsync(buffer, sourceStream, ct), async (buffer, count, cancellationToken2) => { var blockId = GetBlockId(blockNumber); blockIDs.Add(blockId); var blockMD5Hash = CopyUtils.GetMD5Hash(buffer, 0, count); CopyUtils.AppendMDHasherData(blobHasher, buffer, 0, count); using (var ms = new MemoryStream(buffer, 0, count)) { await blob.StageBlockAsync(blockId, ms, blockMD5Hash, null, null, cancellationToken); } blockNumber++; }, bufferSize, cancellationToken ); var blobHash = CopyUtils.GetMD5Hash(blobHasher); if ((!string.IsNullOrEmpty(sourceMD5)) && (sourceMD5 != CopyUtils.GetMD5HashString(blobHash))) { throw new Exception("Invalid destination MD5"); } var headers = new global::Azure.Storage.Blobs.Models.BlobHttpHeaders() { ContentHash = blobHash }; await blob.CommitBlockListAsync(blockIDs, headers, null, null, null, cancellationToken); }
public async Task CollectAsync(string collectUrl, IEnumerable <KeyValuePair <string, string> > collectHeaders, IIdentityServiceClientInfo collectIdentityServiceClientInfo, string dataCollectionName, string fileName, TimeSpan timeout, bool finishWait, int tryNo, CancellationToken cancellationToken) { int bufferSize = 1 * 1024 * 1024; using (var response = await HttpUtils.SendAsync(collectUrl, HttpMethod.Get, collectUrl, null, collectHeaders, null, timeout, null, HttpCompletionOption.ResponseHeadersRead, cancellationToken)) { var sourceMD5 = response.ContentMD5(); var fullFileName = GetFullFileName(dataCollectionName, fileName); Directory.CreateDirectory(Path.GetDirectoryName(fullFileName)); using (var sourceStream = await response.Content.ReadAsStreamAsync()) { using (var fileStream = new FileStream(fullFileName, FileMode.Create)) { await CopyUtils.CopyAsync( (buffer, ct) => CopyUtils.ReadStreamMaxBufferAsync(buffer, sourceStream, ct), async (buffer, count, cancellationToken2) => await fileStream.WriteAsync(buffer, 0, count, cancellationToken2), bufferSize, cancellationToken ); } } using (var fileStream = new FileStream(fullFileName, FileMode.Open, FileAccess.Read)) { var newMD5 = await CopyUtils.GetMD5HashStringAsync(fileStream, bufferSize, cancellationToken); if ((!string.IsNullOrEmpty(sourceMD5)) && (!string.Equals(newMD5, sourceMD5, StringComparison.InvariantCultureIgnoreCase))) { throw new Exception("Invalid destination MD5"); } File.WriteAllText(GetMD5FileName(fullFileName), newMD5); } } }
private async Task MultiPartUploadAsync(Stream sourceStream, string sourceMD5, string bucketName, string keyName, CancellationToken cancellationToken) { // Create list to store upload part responses. var uploadResponses = new List <UploadPartResponse>(); // Setup information required to initiate the multipart upload. var initiateRequest = new InitiateMultipartUploadRequest { BucketName = bucketName, Key = keyName }; var validateMD5 = !string.IsNullOrEmpty(sourceMD5); if (validateMD5) { initiateRequest.Metadata.Add("Content-MD5", sourceMD5); } // Initiate the upload. var initResponse = await _amazonS3.InitiateMultipartUploadAsync(initiateRequest, cancellationToken); try { var blobHasher = validateMD5 ? CopyUtils.GetMD5Hasher() : null; var partSize = 10 * 1024 * 1024; // todo: config var partNo = 1; await CopyUtils.CopyAsync( (buffer, ct) => CopyUtils.ReadStreamMaxBufferAsync(buffer, sourceStream, ct), async (buffer, count, cancellationToken2) => { var blockMD5Hash = CopyUtils.GetMD5HashString(buffer, 0, count); if (validateMD5) { CopyUtils.AppendMDHasherData(blobHasher, buffer, 0, count); } using (var ms = new MemoryStream(buffer, 0, count)) { ms.Position = 0; partNo++; var uploadRequest = new UploadPartRequest { BucketName = bucketName, Key = keyName, UploadId = initResponse.UploadId, PartNumber = partNo, PartSize = count, InputStream = ms, MD5Digest = blockMD5Hash }; // Upload a part and add the response to our list. var uploadResponse = await _amazonS3.UploadPartAsync(uploadRequest, cancellationToken); uploadResponses.Add(uploadResponse); } }, partSize, cancellationToken ); if (validateMD5) { var blobHash = CopyUtils.GetMD5HashString(blobHasher); if ((!string.IsNullOrEmpty(sourceMD5)) && (sourceMD5 != blobHash)) { throw new Exception("Invalid destination MD5"); } } // Setup to complete the upload. var completeRequest = new CompleteMultipartUploadRequest { BucketName = bucketName, Key = keyName, UploadId = initResponse.UploadId }; completeRequest.AddPartETags(uploadResponses); // Complete the upload. var completeUploadResponse = await _amazonS3.CompleteMultipartUploadAsync(completeRequest, cancellationToken); } catch (Exception exception) { Console.WriteLine("An AmazonS3Exception was thrown: { 0}", exception.Message); // Abort the upload. var abortMPURequest = new AbortMultipartUploadRequest { BucketName = bucketName, Key = keyName, UploadId = initResponse.UploadId }; await _amazonS3.AbortMultipartUploadAsync(abortMPURequest, cancellationToken); } }