public void Verify()
{
var failedParts = new List <int> {
1, 2, 3, 4, 5
};
var completedParts = new List <int> {
6, 7, 8, 9, 10
};
var manifest = new MultipartManifest("DummyId");
foreach (var i in Enumerable.Range(1, 10))
{
manifest.RegisterTransfer(i);
}
foreach (var i in completedParts)
{
manifest.RegisterSuccess(i, new Responses.UploadPartResponse());
}
foreach (var i in failedParts)
{
manifest.RegisterFailure(i);
}
Assert.True(!manifest.ListCompletedParts()
.Select(part => part.PartNum.Value)
.Except(completedParts).Any());
Assert.True(!manifest.ListFailedParts()
.Except(failedParts).Any());
}
private async Task <UploadResponse> MultipartUpload(UploadRequest uploadRequest, bool isNewUpload = true)
{
MultipartManifest manifest = null;
var putobjectRequest = uploadRequest.PutObjectRequest;
var chunkCreator = new StreamChunkCreator(putobjectRequest.PutObjectBody, _configuration.LengthPerUploadPartInMiB * MultipartUtils.MiB);
// Uses single thread for multipart uploads.
var executor = new SemaphoreSlim(_configuration.ParallelUploadCount);
var assembler = new MultipartObjectAssembler(_osClient,
putobjectRequest.NamespaceName,
putobjectRequest.BucketName,
putobjectRequest.ObjectName,
uploadRequest.AllowOverwrite,
executor,
putobjectRequest.OpcClientRequestId,
_configuration.EnforceMd5MultipartUpload,
uploadRequest.RetryConfiguration);
try
{
if (isNewUpload && !(uploadRequest is ResumeUploadRequest))
{
var putObjectRequest = uploadRequest.PutObjectRequest;
manifest = await assembler.NewRequest(
putobjectRequest.ContentEncoding, putobjectRequest.ContentType,
putobjectRequest.ContentLanguage, putobjectRequest.ContentDisposition,
putobjectRequest.CacheControl, putobjectRequest.OpcMeta);
}
else
{
manifest = await assembler.ResumeUpload((uploadRequest as ResumeUploadRequest).UploadId).ConfigureAwait(false);
}
ProgressTracker progressTracker = null;
if (uploadRequest.OnProgress != null)
{
progressTracker = new ProgressTracker(uploadRequest.OnProgress, putobjectRequest.PutObjectBody.Length);
}
int partNum = 1;
foreach (var stream in chunkCreator)
{
if (!manifest.IsPartExists(partNum))
{
logger.Info($"Creating Chunk: {partNum}");
if (progressTracker != null)
{
var progressStream = new ProgressTrackingInputStream(stream, progressTracker);
await assembler.AddPart(progressStream, stream.Length, partNum).ConfigureAwait(false);
}
else
{
await assembler.AddPart(stream, stream.Length, partNum).ConfigureAwait(false);
}
}
else
{
if (progressTracker != null)
{
progressTracker.OnRead(stream.Length);
}
logger.Info($"Skipping Part {partNum}");
}
partNum++;
}
var response = await assembler.Commit().ConfigureAwait(false);
logger.Info("Upload Complete");
return(new UploadResponse()
{
ETag = response.ETag,
OpcClientRequestId = response.OpcClientRequestId,
OpcRequestId = response.OpcRequestId,
OpcMultipartMd5 = response.OpcMultipartMd5
});
}
catch (Exception e)
{
if (manifest != null)
{
logger.Error($"Failed to upload object using multi-part uploads. Failed part numbers = '{string.Join(" ", manifest.ListFailedParts())}'" +
$". Successful parts = '{string.Join(" ", manifest.ListCompletedParts())}', Upload Id = '{manifest.UploadId}'");
}
if (_configuration.DisableAutoAbort)
{
logger.Info($"Not aborting failed multipart upload {manifest.UploadId} per configuration, client must manually abort it");
}
else
{
logger.Info("Aborting in-progress uploads");
await assembler.Abort();
}
throw e;
}
finally
{
executor.Dispose();
}
}