/// <summary>
/// Inititate a new Multipart upload request
/// </summary>
/// <param name="putObjectRequest"></param>
/// <returns></returns>
public async Task <MultipartManifest> NewRequest(
string contentEncoding, string contentType,
string contentLanguage, string contentDisposition,
string cacheControl, Dictionary <string, string> metadata,
StorageTier storageTier)
{
CheckInitialized();
var ifNoneMatch = _allowOverwrite ? null : "*";
var request = new CreateMultipartUploadRequest()
{
BucketName = _bucketName,
NamespaceName = _namespaceName,
IfNoneMatch = ifNoneMatch,
CreateMultipartUploadDetails = new CreateMultipartUploadDetails()
{
Object = _objectName,
ContentEncoding = contentEncoding,
ContentType = contentType,
ContentLanguage = contentLanguage,
Metadata = metadata,
ContentDisposition = contentDisposition,
CacheControl = cacheControl,
StorageTier = storageTier
},
OpcClientRequestId = CreateClientRequestId("New")
};
var response = await _service.CreateMultipartUpload(request).ConfigureAwait(false);
_multipartManifest = new MultipartManifest(response.MultipartUpload.UploadId);
_transferManager = new MultipartTransferManager(_multipartManifest, _service, _executorServiceToUse, _tokenSource.Token, _retryConfiguration);
_initialized = true;
return(_multipartManifest);
}
public async Task MultipartViaExtensions()
{
byte[] data = new byte[100 * 1024 * 1024]; //100 Mb
for (int i = 0; i < data.Length; i++)
{
data[i] = (byte)'A';
}
CreateMultipartUploadRequest createRequest = new CreateMultipartUploadRequest(BucketName, nameof(MultipartViaExtensions));
int count = 0;
using (MemoryStream ms = new MemoryStream(data))
{
await foreach (UploadPartResponse resp in MultipartClient.MultipartOperations.MultipartUploadAsync(createRequest, ms, 10 * 1024 * 1024, 1))
{
count++;
Assert.True(resp.IsSuccess);
}
}
Assert.Equal(10, count);
}
public async Task <CompleteMultipartUploadResponse> MultipartUploadAsync(CreateMultipartUploadRequest req, Stream data, int partSize = 16777216, int numParallelParts = 4, Action <UploadPartResponse>?onPartResponse = null, CancellationToken token = default)
{
Validator.RequireNotNull(req, nameof(req));
Validator.RequireNotNull(data, nameof(data));
foreach (IRequestWrapper wrapper in _requestWrappers)
{
if (wrapper.IsSupported(req))
{
data = wrapper.Wrap(data, req);
}
}
string bucket = req.BucketName;
string objectKey = req.ObjectKey;
byte[]? encryptionKey = null;
try
{
if (req.SseCustomerKey != null)
{
encryptionKey = new byte[req.SseCustomerKey.Length];
Array.Copy(req.SseCustomerKey, 0, encryptionKey, 0, encryptionKey.Length);
}
CreateMultipartUploadResponse initResp = await _multipartOperations.CreateMultipartUploadAsync(req, token).ConfigureAwait(false);
if (token.IsCancellationRequested)
{
return new CompleteMultipartUploadResponse {
BucketName = bucket, ObjectKey = objectKey
}
}
;
if (!initResp.IsSuccess)
{
throw new S3RequestException(initResp, "CreateMultipartUploadRequest was unsuccessful");
}
IEnumerable <ArraySegment <byte> > chunks = ReadChunks(data, partSize);
int partNumber = 0;
IEnumerable <UploadPartResponse> responses = await ParallelHelper.ExecuteAsync(chunks, async (bytes, innerToken) =>
{
Interlocked.Increment(ref partNumber);
using (MemoryStream ms = new MemoryStream(bytes.Array !, 0, bytes.Count))
{
UploadPartResponse resp = await _multipartClient.UploadPartAsync(bucket, objectKey, partNumber, initResp.UploadId, ms, uploadPart =>
{
uploadPart.SseCustomerAlgorithm = req.SseCustomerAlgorithm;
uploadPart.SseCustomerKey = encryptionKey;
uploadPart.SseCustomerKeyMd5 = req.SseCustomerKeyMd5;
}, innerToken).ConfigureAwait(false);
onPartResponse?.Invoke(resp);
return(resp);
}
}, numParallelParts, token);
CompleteMultipartUploadResponse completeResp = await _multipartClient.CompleteMultipartUploadAsync(bucket, objectKey, initResp.UploadId, responses.OrderBy(x => x.PartNumber), null, token).ConfigureAwait(false);
return(completeResp);
}
finally
{
if (encryptionKey != null)
{
Array.Clear(encryptionKey, 0, encryptionKey.Length);
}
}
}
public Task <CompleteMultipartUploadResponse> MultipartUploadAsync(CreateMultipartUploadRequest req, Stream data, int partSize = 16777216, int numParallelParts = 4, Action <UploadPartResponse>?onPartResponse = null, CancellationToken token = default)
{
return(Client.MultipartUploadAsync(req, data, partSize, numParallelParts, onPartResponse, token));
}
public Task <CreateMultipartUploadResponse> CreateMultipartUploadAsync(CreateMultipartUploadRequest request, CancellationToken token = default)
{
return(_requestHandler.SendRequestAsync <CreateMultipartUploadRequest, CreateMultipartUploadResponse>(request, token));
}
public static async IAsyncEnumerable <UploadPartResponse> MultipartUploadAsync(this IMultipartOperations operations, CreateMultipartUploadRequest req, Stream data, int partSize = 16777216, int numParallelParts = 4, [EnumeratorCancellation] CancellationToken token = default)
{
Validator.RequireNotNull(req, nameof(req));
Validator.RequireNotNull(data, nameof(data));
foreach (IRequestWrapper wrapper in operations.RequestWrappers)
{
if (wrapper.IsSupported(req))
{
data = wrapper.Wrap(data, req);
}
}
string bucket = req.BucketName;
string objectKey = req.ObjectKey;
CreateMultipartUploadResponse initResp = await operations.CreateMultipartUploadAsync(req, token).ConfigureAwait(false);
if (token.IsCancellationRequested)
{
yield break;
}
if (!initResp.IsSuccess)
{
throw new S3RequestException(initResp.StatusCode, "CreateMultipartUploadRequest was unsuccessful");
}
Queue <Task <UploadPartResponse> > uploads = new Queue <Task <UploadPartResponse> >();
using (SemaphoreSlim semaphore = new SemaphoreSlim(numParallelParts))
{
long offset = 0;
for (int i = 1; offset < data.Length; i++)
{
await semaphore.WaitAsync(token).ConfigureAwait(false);
if (token.IsCancellationRequested)
{
break;
}
byte[] partData = new byte[partSize];
int read = await data.ReadUpToAsync(partData, 0, partData.Length, token).ConfigureAwait(false);
TaskCompletionSource <UploadPartResponse> completionSource = new TaskCompletionSource <UploadPartResponse>();
uploads.Enqueue(completionSource.Task);
UploadPartAsync(completionSource, operations, bucket, objectKey, partData, read, i, initResp.UploadId, semaphore, token);
offset += partSize;
}
Queue <UploadPartResponse> responses = new Queue <UploadPartResponse>(uploads.Count);
while (uploads.TryDequeue(out Task <UploadPartResponse>?task))
{
if (token.IsCancellationRequested)
{
yield break;
}
UploadPartResponse response = await task !.ConfigureAwait(false);
responses.Enqueue(response);
yield return(response);
}
CompleteMultipartUploadRequest completeReq = new CompleteMultipartUploadRequest(bucket, objectKey, initResp.UploadId, responses);
CompleteMultipartUploadResponse completeResp = await operations.CompleteMultipartUploadAsync(completeReq, token).ConfigureAwait(false);
if (!completeResp.IsSuccess)
{
throw new S3RequestException(completeResp.StatusCode, "CompleteMultipartUploadRequest was unsuccessful");
}
}
}
public static void intercept(CreateMultipartUploadRequest request)
{
Dictionary <string, string> updatedMetadata = AddOpcMetaPrefix(request.CreateMultipartUploadDetails.Metadata);
request.CreateMultipartUploadDetails.Metadata = updatedMetadata;
}
