public void StreamChunkVerification()
{
var inputStr = "aaaaaaaaaa";
var sourceStream = GenerateStreamFromString(inputStr);
var chunkCreator = new StreamChunkCreator(sourceStream, 2);
var builder = new StringBuilder();
var i = 0;
foreach (var stream in chunkCreator)
{
builder.Append(GetStringFromStream(stream));
i++;
}
Assert.True(inputStr.Equals(builder.ToString()));
Assert.Equal(inputStr.Length / 2, i);
}
private async Task <UploadResponse> MultipartUpload(UploadRequest uploadRequest, bool isNewUpload = true)
{
MultipartManifest manifest = null;
var putobjectRequest = uploadRequest.PutObjectRequest;
var chunkCreator = new StreamChunkCreator(putobjectRequest.PutObjectBody, _configuration.LengthPerUploadPartInMiB * MultipartUtils.MiB);
// Uses single thread for multipart uploads.
var executor = new SemaphoreSlim(_configuration.ParallelUploadCount);
var assembler = new MultipartObjectAssembler(_osClient,
putobjectRequest.NamespaceName,
putobjectRequest.BucketName,
putobjectRequest.ObjectName,
uploadRequest.AllowOverwrite,
executor,
putobjectRequest.OpcClientRequestId,
_configuration.EnforceMd5MultipartUpload,
uploadRequest.RetryConfiguration);
try
{
if (isNewUpload && !(uploadRequest is ResumeUploadRequest))
{
var putObjectRequest = uploadRequest.PutObjectRequest;
manifest = await assembler.NewRequest(
putobjectRequest.ContentEncoding, putobjectRequest.ContentType,
putobjectRequest.ContentLanguage, putobjectRequest.ContentDisposition,
putobjectRequest.CacheControl, putobjectRequest.OpcMeta);
}
else
{
manifest = await assembler.ResumeUpload((uploadRequest as ResumeUploadRequest).UploadId).ConfigureAwait(false);
}
ProgressTracker progressTracker = null;
if (uploadRequest.OnProgress != null)
{
progressTracker = new ProgressTracker(uploadRequest.OnProgress, putobjectRequest.PutObjectBody.Length);
}
int partNum = 1;
foreach (var stream in chunkCreator)
{
if (!manifest.IsPartExists(partNum))
{
logger.Info($"Creating Chunk: {partNum}");
if (progressTracker != null)
{
var progressStream = new ProgressTrackingInputStream(stream, progressTracker);
await assembler.AddPart(progressStream, stream.Length, partNum).ConfigureAwait(false);
}
else
{
await assembler.AddPart(stream, stream.Length, partNum).ConfigureAwait(false);
}
}
else
{
if (progressTracker != null)
{
progressTracker.OnRead(stream.Length);
}
logger.Info($"Skipping Part {partNum}");
}
partNum++;
}
var response = await assembler.Commit().ConfigureAwait(false);
logger.Info("Upload Complete");
return(new UploadResponse()
{
ETag = response.ETag,
OpcClientRequestId = response.OpcClientRequestId,
OpcRequestId = response.OpcRequestId,
OpcMultipartMd5 = response.OpcMultipartMd5
});
}
catch (Exception e)
{
if (manifest != null)
{
logger.Error($"Failed to upload object using multi-part uploads. Failed part numbers = '{string.Join(" ", manifest.ListFailedParts())}'" +
$". Successful parts = '{string.Join(" ", manifest.ListCompletedParts())}', Upload Id = '{manifest.UploadId}'");
}
if (_configuration.DisableAutoAbort)
{
logger.Info($"Not aborting failed multipart upload {manifest.UploadId} per configuration, client must manually abort it");
}
else
{
logger.Info("Aborting in-progress uploads");
await assembler.Abort();
}
throw e;
}
finally
{
executor.Dispose();
}
}