/// <summary>
/// Inititate a new Multipart upload request
/// </summary>
/// <param name="putObjectRequest"></param>
/// <returns></returns>
public async Task <MultipartManifest> NewRequest(
string contentEncoding, string contentType,
string contentLanguage, string contentDisposition,
string cacheControl, Dictionary <string, string> metadata)
{
CheckInitialized();
var ifNoneMatch = _allowOverwrite ? null : "*";
var request = new CreateMultipartUploadRequest()
{
BucketName = _bucketName,
NamespaceName = _namespaceName,
IfNoneMatch = ifNoneMatch,
CreateMultipartUploadDetails = new CreateMultipartUploadDetails()
{
Object = _objectName,
ContentEncoding = contentEncoding,
ContentType = contentType,
ContentLanguage = contentLanguage,
Metadata = metadata,
ContentDisposition = contentDisposition,
CacheControl = cacheControl,
},
OpcClientRequestId = CreateClientRequestId("New")
};
var response = await _service.CreateMultipartUpload(request).ConfigureAwait(false);
_multipartManifest = new MultipartManifest(response.MultipartUpload.UploadId);
_transferManager = new MultipartTransferManager(_multipartManifest, _service, _executorServiceToUse, _tokenSource.Token, _retryConfiguration);
_initialized = true;
return(_multipartManifest);
}
public void Verify()
{
var failedParts = new List <int> {
1, 2, 3, 4, 5
};
var completedParts = new List <int> {
6, 7, 8, 9, 10
};
var manifest = new MultipartManifest("DummyId");
foreach (var i in Enumerable.Range(1, 10))
{
manifest.RegisterTransfer(i);
}
foreach (var i in completedParts)
{
manifest.RegisterSuccess(i, new Responses.UploadPartResponse());
}
foreach (var i in failedParts)
{
manifest.RegisterFailure(i);
}
Assert.True(!manifest.ListCompletedParts()
.Select(part => part.PartNum.Value)
.Except(completedParts).Any());
Assert.True(!manifest.ListFailedParts()
.Except(failedParts).Any());
}
public MultipartTransferManager(MultipartManifest manifest,
ObjectStorageClient osClient,
SemaphoreSlim executor,
CancellationToken cancellationToken = default,
RetryConfiguration retryConfiguration = null)
{
this.Manifest = manifest;
this.OSClient = osClient;
this.Executor = executor;
this._cancellationToken = cancellationToken;
this._retryConfiguration = retryConfiguration;
}
/// <summary>
/// Resumes an existing multi-part upload using the upload ID.
/// </summary>
/// <param name="uploadId"></param>
/// <returns>MultipartManifest</returns>
public async Task <MultipartManifest> ResumeUpload(string uploadId)
{
CheckInitialized();
var multipartUpload = FindUpload(uploadId);
if (multipartUpload == null)
{
_logger.Error($"No matching upload found for ${uploadId}");
throw new ArgumentException($"Could not find existing upload with ID: {uploadId} in bucket: {_bucketName} and NamespaceName: {_namespaceName}");
}
var multipartManifest = new MultipartManifest(uploadId);
string nextPageToken = null;
do
{
var partsRequest = new ListMultipartUploadPartsRequest()
{
BucketName = _bucketName,
NamespaceName = _namespaceName,
ObjectName = _objectName,
UploadId = uploadId,
Limit = 100,
Page = nextPageToken,
OpcClientRequestId = CreateClientRequestId("List")
};
var existingParts = await _service.ListMultipartUploadParts(partsRequest, _retryConfiguration).ConfigureAwait(false);
nextPageToken = existingParts.OpcNextPage;
foreach (var part in existingParts.Items)
{
multipartManifest.RegisterExisting(part);
}
} while (nextPageToken != null);
_multipartManifest = multipartManifest;
_transferManager = new MultipartTransferManager(_multipartManifest, _service, _executorServiceToUse, _tokenSource.Token, _retryConfiguration);
_initialized = true;
return(multipartManifest);
}
private async Task <UploadResponse> MultipartUpload(UploadRequest uploadRequest, bool isNewUpload = true)
{
MultipartManifest manifest = null;
var putobjectRequest = uploadRequest.PutObjectRequest;
var chunkCreator = new StreamChunkCreator(putobjectRequest.PutObjectBody, _configuration.LengthPerUploadPartInMiB * MultipartUtils.MiB);
// Uses single thread for multipart uploads.
var executor = new SemaphoreSlim(_configuration.ParallelUploadCount);
var assembler = new MultipartObjectAssembler(_osClient,
putobjectRequest.NamespaceName,
putobjectRequest.BucketName,
putobjectRequest.ObjectName,
uploadRequest.AllowOverwrite,
executor,
putobjectRequest.OpcClientRequestId,
_configuration.EnforceMd5MultipartUpload,
uploadRequest.RetryConfiguration);
try
{
if (isNewUpload && !(uploadRequest is ResumeUploadRequest))
{
var putObjectRequest = uploadRequest.PutObjectRequest;
manifest = await assembler.NewRequest(
putobjectRequest.ContentEncoding, putobjectRequest.ContentType,
putobjectRequest.ContentLanguage, putobjectRequest.ContentDisposition,
putobjectRequest.CacheControl, putobjectRequest.OpcMeta);
}
else
{
manifest = await assembler.ResumeUpload((uploadRequest as ResumeUploadRequest).UploadId).ConfigureAwait(false);
}
ProgressTracker progressTracker = null;
if (uploadRequest.OnProgress != null)
{
progressTracker = new ProgressTracker(uploadRequest.OnProgress, putobjectRequest.PutObjectBody.Length);
}
int partNum = 1;
foreach (var stream in chunkCreator)
{
if (!manifest.IsPartExists(partNum))
{
logger.Info($"Creating Chunk: {partNum}");
if (progressTracker != null)
{
var progressStream = new ProgressTrackingInputStream(stream, progressTracker);
await assembler.AddPart(progressStream, stream.Length, partNum).ConfigureAwait(false);
}
else
{
await assembler.AddPart(stream, stream.Length, partNum).ConfigureAwait(false);
}
}
else
{
if (progressTracker != null)
{
progressTracker.OnRead(stream.Length);
}
logger.Info($"Skipping Part {partNum}");
}
partNum++;
}
var response = await assembler.Commit().ConfigureAwait(false);
logger.Info("Upload Complete");
return(new UploadResponse()
{
ETag = response.ETag,
OpcClientRequestId = response.OpcClientRequestId,
OpcRequestId = response.OpcRequestId,
OpcMultipartMd5 = response.OpcMultipartMd5
});
}
catch (Exception e)
{
if (manifest != null)
{
logger.Error($"Failed to upload object using multi-part uploads. Failed part numbers = '{string.Join(" ", manifest.ListFailedParts())}'" +
$". Successful parts = '{string.Join(" ", manifest.ListCompletedParts())}', Upload Id = '{manifest.UploadId}'");
}
if (_configuration.DisableAutoAbort)
{
logger.Info($"Not aborting failed multipart upload {manifest.UploadId} per configuration, client must manually abort it");
}
else
{
logger.Info("Aborting in-progress uploads");
await assembler.Abort();
}
throw e;
}
finally
{
executor.Dispose();
}
}
