public Task <CreateMultipartUploadResponse> CreateMultipartUploadAsync(string bucketName, string objectKey, Action <CreateMultipartUploadRequest>?config = null, CancellationToken token = default)
{
CreateMultipartUploadRequest req = new CreateMultipartUploadRequest(bucketName, objectKey);
config?.Invoke(req);
return(_multipartOperations.CreateMultipartUploadAsync(req, token));
}
public async Task <CompleteMultipartUploadResponse> MultipartUploadAsync(CreateMultipartUploadRequest req, Stream data, int partSize = 16777216, int numParallelParts = 4, Action <UploadPartResponse>?onPartResponse = null, CancellationToken token = default)
{
Validator.RequireNotNull(req, nameof(req));
Validator.RequireNotNull(data, nameof(data));
foreach (IRequestWrapper wrapper in _requestWrappers)
{
if (wrapper.IsSupported(req))
{
data = wrapper.Wrap(data, req);
}
}
string bucket = req.BucketName;
string objectKey = req.ObjectKey;
byte[]? encryptionKey = null;
try
{
if (req.SseCustomerKey != null)
{
encryptionKey = new byte[req.SseCustomerKey.Length];
Array.Copy(req.SseCustomerKey, 0, encryptionKey, 0, encryptionKey.Length);
}
CreateMultipartUploadResponse initResp = await _multipartOperations.CreateMultipartUploadAsync(req, token).ConfigureAwait(false);
if (token.IsCancellationRequested)
{
return new CompleteMultipartUploadResponse {
BucketName = bucket, ObjectKey = objectKey
}
}
;
if (!initResp.IsSuccess)
{
throw new S3RequestException(initResp, "CreateMultipartUploadRequest was unsuccessful");
}
IEnumerable <ArraySegment <byte> > chunks = ReadChunks(data, partSize);
int partNumber = 0;
IEnumerable <UploadPartResponse> responses = await ParallelHelper.ExecuteAsync(chunks, async (bytes, innerToken) =>
{
Interlocked.Increment(ref partNumber);
using (MemoryStream ms = new MemoryStream(bytes.Array !, 0, bytes.Count))
{
UploadPartResponse resp = await _multipartClient.UploadPartAsync(bucket, objectKey, partNumber, initResp.UploadId, ms, uploadPart =>
{
uploadPart.SseCustomerAlgorithm = req.SseCustomerAlgorithm;
uploadPart.SseCustomerKey = encryptionKey;
uploadPart.SseCustomerKeyMd5 = req.SseCustomerKeyMd5;
}, innerToken).ConfigureAwait(false);
onPartResponse?.Invoke(resp);
return(resp);
}
}, numParallelParts, token);
CompleteMultipartUploadResponse completeResp = await _multipartClient.CompleteMultipartUploadAsync(bucket, objectKey, initResp.UploadId, responses.OrderBy(x => x.PartNumber), null, token).ConfigureAwait(false);
return(completeResp);
}
finally
{
if (encryptionKey != null)
{
Array.Clear(encryptionKey, 0, encryptionKey.Length);
}
}
}
public static async IAsyncEnumerable <UploadPartResponse> MultipartUploadAsync(this IMultipartOperations operations, CreateMultipartUploadRequest req, Stream data, int partSize = 16777216, int numParallelParts = 4, [EnumeratorCancellation] CancellationToken token = default)
{
Validator.RequireNotNull(req, nameof(req));
Validator.RequireNotNull(data, nameof(data));
foreach (IRequestWrapper wrapper in operations.RequestWrappers)
{
if (wrapper.IsSupported(req))
{
data = wrapper.Wrap(data, req);
}
}
string bucket = req.BucketName;
string objectKey = req.ObjectKey;
CreateMultipartUploadResponse initResp = await operations.CreateMultipartUploadAsync(req, token).ConfigureAwait(false);
if (token.IsCancellationRequested)
{
yield break;
}
if (!initResp.IsSuccess)
{
throw new S3RequestException(initResp.StatusCode, "CreateMultipartUploadRequest was unsuccessful");
}
Queue <Task <UploadPartResponse> > uploads = new Queue <Task <UploadPartResponse> >();
using (SemaphoreSlim semaphore = new SemaphoreSlim(numParallelParts))
{
long offset = 0;
for (int i = 1; offset < data.Length; i++)
{
await semaphore.WaitAsync(token).ConfigureAwait(false);
if (token.IsCancellationRequested)
{
break;
}
byte[] partData = new byte[partSize];
int read = await data.ReadUpToAsync(partData, 0, partData.Length, token).ConfigureAwait(false);
TaskCompletionSource <UploadPartResponse> completionSource = new TaskCompletionSource <UploadPartResponse>();
uploads.Enqueue(completionSource.Task);
UploadPartAsync(completionSource, operations, bucket, objectKey, partData, read, i, initResp.UploadId, semaphore, token);
offset += partSize;
}
Queue <UploadPartResponse> responses = new Queue <UploadPartResponse>(uploads.Count);
while (uploads.TryDequeue(out Task <UploadPartResponse>?task))
{
if (token.IsCancellationRequested)
{
yield break;
}
UploadPartResponse response = await task !.ConfigureAwait(false);
responses.Enqueue(response);
yield return(response);
}
CompleteMultipartUploadRequest completeReq = new CompleteMultipartUploadRequest(bucket, objectKey, initResp.UploadId, responses);
CompleteMultipartUploadResponse completeResp = await operations.CompleteMultipartUploadAsync(completeReq, token).ConfigureAwait(false);
if (!completeResp.IsSuccess)
{
throw new S3RequestException(completeResp.StatusCode, "CompleteMultipartUploadRequest was unsuccessful");
}
}
}
