public IActionResult GetContentIDs(string containerName, [FromQuery] DateTimeOffset?after, [FromQuery] string maxCount, [FromQuery] string afterContentID, [FromQuery] string auth, [FromHeader] string authenticationKey, CancellationToken cancellationToken)
{
var intMaxCount = string.IsNullOrEmpty(maxCount) ? default(int?) : int.Parse(maxCount);
return(new FileCallbackResult(
new MediaTypeHeaderValue("text/plain"),
async(outputStream, _) =>
{
using (var contentIDsEnumerator = _contents.GetContentIDs(containerName, after, intMaxCount, afterContentID, auth ?? authenticationKey).GetEnumerator())
{
await CopyUtils.CopyAsync(
(buffer, ct) =>
{
var byteCount = 0;
while ((byteCount < buffer.Length - 1000) && (contentIDsEnumerator.MoveNext()))
{
var enumBuff = Encoding.UTF8.GetBytes(contentIDsEnumerator.Current + Environment.NewLine);
enumBuff.CopyTo(buffer, byteCount);
byteCount += enumBuff.Length;
}
return Task.FromResult(byteCount);
},
async(buffer, count, cancellationToken2) =>
{
await outputStream.WriteAsync(buffer, 0, count, cancellationToken2);
},
1_000_000,
cancellationToken
);
}
}));
}
private IEnumerable <KeyValuePair <string, string> > MultiPartUpload(IMultiPartIDUploadInfo multiPartIDUploadInfo, Stream stream, long streamSize)
{
var result = new List <KeyValuePair <string, string> >();
if (multiPartIDUploadInfo.PartSize <= 0)
{
System.Diagnostics.Debug.Assert(multiPartIDUploadInfo.PartUploadInfos.Count() == 1);
UploadPart(multiPartIDUploadInfo.HttpMethod, stream, streamSize, multiPartIDUploadInfo.PartUploadInfos.First(), "");
}
else
{
int writePartIndex = 0;
CopyUtils.CopyAsync(
async(buffer, cancellationToken) =>
{
return(await CopyUtils.FillBufferAsync(buffer,
async(buf, offset, ct) =>
{
return await stream.ReadAsync(buf, offset, (int)multiPartIDUploadInfo.PartSize - offset);
},
cancellationToken
));
},
(buffer, size, cancellationToken) =>
{
var partUploadInfo = multiPartIDUploadInfo.PartUploadInfos.ElementAt(writePartIndex);
var uploadPartResult = UploadPart(multiPartIDUploadInfo.HttpMethod, new MemoryStream(buffer), size, partUploadInfo, multiPartIDUploadInfo.MultiPartUploadResultHeaderName);
result.Add(new KeyValuePair <string, string>(partUploadInfo.Identifier, uploadPartResult));
writePartIndex++;
return(Task.CompletedTask);
},
(int)multiPartIDUploadInfo.PartSize,
CancellationToken.None
)
.Wait();
//foreach (var partUploadInfo in multiPartIDUploadInfo.PartUploadInfos)
//{
// var uploadPartResult = UploadPart(multiPartIDUploadInfo.HttpMethod, stream, multiPartIDUploadInfo.PartSize, partUploadInfo, multiPartIDUploadInfo.MultiPartUploadResultHeaderName);
// yield return new KeyValuePair<string, string>(partUploadInfo.Identifier, uploadPartResult);
//}
}
return(result);
}
private async Task CopyStreamToBlobAsync(Stream sourceStream, BlobContainerClient container, string blobName, int bufferSize, string sourceMD5, CancellationToken cancellationToken)
{
var blob = container.GetBlockBlobClient(blobName);
var blobHasher = CopyUtils.GetMD5Hasher();
var blockIDs = new List <string>();
var blockNumber = 0;
await CopyUtils.CopyAsync(
(buffer, ct) => CopyUtils.ReadStreamMaxBufferAsync(buffer, sourceStream, ct),
async (buffer, count, cancellationToken2) =>
{
var blockId = GetBlockId(blockNumber);
blockIDs.Add(blockId);
var blockMD5Hash = CopyUtils.GetMD5Hash(buffer, 0, count);
CopyUtils.AppendMDHasherData(blobHasher, buffer, 0, count);
using (var ms = new MemoryStream(buffer, 0, count))
{
await blob.StageBlockAsync(blockId, ms, blockMD5Hash, null, null, cancellationToken);
}
blockNumber++;
},
bufferSize,
cancellationToken
);
var blobHash = CopyUtils.GetMD5Hash(blobHasher);
if ((!string.IsNullOrEmpty(sourceMD5)) && (sourceMD5 != CopyUtils.GetMD5HashString(blobHash)))
{
throw new Exception("Invalid destination MD5");
}
var headers = new global::Azure.Storage.Blobs.Models.BlobHttpHeaders()
{
ContentHash = blobHash
};
await blob.CommitBlockListAsync(blockIDs, headers, null, null, null, cancellationToken);
}
public async Task CollectAsync(string collectUrl, IEnumerable <KeyValuePair <string, string> > collectHeaders, IIdentityServiceClientInfo collectIdentityServiceClientInfo, string dataCollectionName, string fileName, TimeSpan timeout, bool finishWait, int tryNo, CancellationToken cancellationToken)
{
int bufferSize = 1 * 1024 * 1024;
using (var response = await HttpUtils.SendAsync(collectUrl, HttpMethod.Get, collectUrl, null, collectHeaders, null, timeout, null, HttpCompletionOption.ResponseHeadersRead, cancellationToken))
{
var sourceMD5 = response.ContentMD5();
var fullFileName = GetFullFileName(dataCollectionName, fileName);
Directory.CreateDirectory(Path.GetDirectoryName(fullFileName));
using (var sourceStream = await response.Content.ReadAsStreamAsync())
{
using (var fileStream = new FileStream(fullFileName, FileMode.Create))
{
await CopyUtils.CopyAsync(
(buffer, ct) => CopyUtils.ReadStreamMaxBufferAsync(buffer, sourceStream, ct),
async (buffer, count, cancellationToken2) => await fileStream.WriteAsync(buffer, 0, count, cancellationToken2),
bufferSize,
cancellationToken
);
}
}
using (var fileStream = new FileStream(fullFileName, FileMode.Open, FileAccess.Read))
{
var newMD5 = await CopyUtils.GetMD5HashStringAsync(fileStream, bufferSize, cancellationToken);
if ((!string.IsNullOrEmpty(sourceMD5)) &&
(!string.Equals(newMD5, sourceMD5, StringComparison.InvariantCultureIgnoreCase)))
{
throw new Exception("Invalid destination MD5");
}
File.WriteAllText(GetMD5FileName(fullFileName), newMD5);
}
}
}
private async Task MultiPartUploadAsync(Stream sourceStream, string sourceMD5, string bucketName, string keyName, CancellationToken cancellationToken)
{
// Create list to store upload part responses.
var uploadResponses = new List <UploadPartResponse>();
// Setup information required to initiate the multipart upload.
var initiateRequest = new InitiateMultipartUploadRequest
{
BucketName = bucketName,
Key = keyName
};
var validateMD5 = !string.IsNullOrEmpty(sourceMD5);
if (validateMD5)
{
initiateRequest.Metadata.Add("Content-MD5", sourceMD5);
}
// Initiate the upload.
var initResponse = await _amazonS3.InitiateMultipartUploadAsync(initiateRequest, cancellationToken);
try
{
var blobHasher = validateMD5 ? CopyUtils.GetMD5Hasher() : null;
var partSize = 10 * 1024 * 1024; // todo: config
var partNo = 1;
await CopyUtils.CopyAsync(
(buffer, ct) => CopyUtils.ReadStreamMaxBufferAsync(buffer, sourceStream, ct),
async (buffer, count, cancellationToken2) =>
{
var blockMD5Hash = CopyUtils.GetMD5HashString(buffer, 0, count);
if (validateMD5)
{
CopyUtils.AppendMDHasherData(blobHasher, buffer, 0, count);
}
using (var ms = new MemoryStream(buffer, 0, count))
{
ms.Position = 0;
partNo++;
var uploadRequest = new UploadPartRequest
{
BucketName = bucketName,
Key = keyName,
UploadId = initResponse.UploadId,
PartNumber = partNo,
PartSize = count,
InputStream = ms,
MD5Digest = blockMD5Hash
};
// Upload a part and add the response to our list.
var uploadResponse = await _amazonS3.UploadPartAsync(uploadRequest, cancellationToken);
uploadResponses.Add(uploadResponse);
}
},
partSize,
cancellationToken
);
if (validateMD5)
{
var blobHash = CopyUtils.GetMD5HashString(blobHasher);
if ((!string.IsNullOrEmpty(sourceMD5)) && (sourceMD5 != blobHash))
{
throw new Exception("Invalid destination MD5");
}
}
// Setup to complete the upload.
var completeRequest = new CompleteMultipartUploadRequest
{
BucketName = bucketName,
Key = keyName,
UploadId = initResponse.UploadId
};
completeRequest.AddPartETags(uploadResponses);
// Complete the upload.
var completeUploadResponse = await _amazonS3.CompleteMultipartUploadAsync(completeRequest, cancellationToken);
}
catch (Exception exception)
{
Console.WriteLine("An AmazonS3Exception was thrown: { 0}", exception.Message);
// Abort the upload.
var abortMPURequest = new AbortMultipartUploadRequest
{
BucketName = bucketName,
Key = keyName,
UploadId = initResponse.UploadId
};
await _amazonS3.AbortMultipartUploadAsync(abortMPURequest, cancellationToken);
}
}
