public async Task ListObjectsMoreThanMaxKeys(S3Provider provider, string _, ISimpleClient client)
{
await CreateTempBucketAsync(provider, client, async tempBucket =>
{
int concurrent = 10;
int count = 11;
await ParallelHelper.ExecuteAsync(Enumerable.Range(0, count), (val, token) => client.PutObjectAsync(tempBucket, val.ToString(), null, null, token), concurrent);
ListObjectVersionsResponse listResp = await client.ListObjectVersionsAsync(tempBucket, r => r.MaxKeys = count - 1).ConfigureAwait(false);
Assert.True(listResp.IsSuccess);
Assert.True(listResp.IsTruncated);
Assert.Equal(10, listResp.MaxKeys);
Assert.Equal(10, listResp.Versions.Count);
ListObjectVersionsResponse listResp2 = await client.ListObjectVersionsAsync(tempBucket, r => r.KeyMarker = listResp.NextKeyMarker).ConfigureAwait(false);
Assert.True(listResp2.IsSuccess);
Assert.False(listResp2.IsTruncated);
if (provider != S3Provider.GoogleCloudStorage)
{
Assert.Equal(1, listResp2.Versions.Count);
}
}).ConfigureAwait(false);
}
public async Task ListObjectsMoreThanMaxKeys(S3Provider provider, string _, ISimpleClient client)
{
await CreateTempBucketAsync(provider, client, async tempBucket =>
{
int concurrent = 10;
int count = 11;
IEnumerable <PutObjectResponse> responses = await ParallelHelper.ExecuteAsync(Enumerable.Range(0, count), (val, token) => client.PutObjectAsync(tempBucket, val.ToString(), null, null, token), concurrent, CancellationToken.None);
foreach (PutObjectResponse putResp in responses)
{
Assert.Equal(200, putResp.StatusCode);
}
ListObjectsResponse listResp = await client.ListObjectsAsync(tempBucket, r => r.MaxKeys = count - 1).ConfigureAwait(false);
Assert.Equal(200, listResp.StatusCode);
Assert.Equal(count - 1, listResp.KeyCount);
Assert.Equal(count - 1, listResp.Objects.Count);
Assert.True(listResp.IsTruncated);
if (provider == S3Provider.AmazonS3)
{
Assert.NotEmpty(listResp.NextContinuationToken);
}
ListObjectsResponse listResp2 = await client.ListObjectsAsync(tempBucket, r => r.ContinuationToken = listResp.NextContinuationToken).ConfigureAwait(false);
Assert.Equal(200, listResp2.StatusCode);
Assert.Equal(1, listResp2.KeyCount);
Assert.Equal(1, listResp2.Objects.Count);
Assert.Equal(listResp.NextContinuationToken, listResp2.ContinuationToken);
Assert.Null(listResp2.NextContinuationToken);
Assert.False(listResp2.IsTruncated);
}).ConfigureAwait(false);
}
public async Task <CompleteMultipartUploadResponse> MultipartUploadAsync(CreateMultipartUploadRequest req, Stream data, int partSize = 16777216, int numParallelParts = 4, Action <UploadPartResponse>?onPartResponse = null, CancellationToken token = default)
{
Validator.RequireNotNull(req, nameof(req));
Validator.RequireNotNull(data, nameof(data));
foreach (IRequestWrapper wrapper in _requestWrappers)
{
if (wrapper.IsSupported(req))
{
data = wrapper.Wrap(data, req);
}
}
string bucket = req.BucketName;
string objectKey = req.ObjectKey;
byte[]? encryptionKey = null;
try
{
if (req.SseCustomerKey != null)
{
encryptionKey = new byte[req.SseCustomerKey.Length];
Array.Copy(req.SseCustomerKey, 0, encryptionKey, 0, encryptionKey.Length);
}
CreateMultipartUploadResponse initResp = await _multipartOperations.CreateMultipartUploadAsync(req, token).ConfigureAwait(false);
if (token.IsCancellationRequested)
{
return new CompleteMultipartUploadResponse {
BucketName = bucket, ObjectKey = objectKey
}
}
;
if (!initResp.IsSuccess)
{
throw new S3RequestException(initResp, "CreateMultipartUploadRequest was unsuccessful");
}
IEnumerable <ArraySegment <byte> > chunks = ReadChunks(data, partSize);
int partNumber = 0;
IEnumerable <UploadPartResponse> responses = await ParallelHelper.ExecuteAsync(chunks, async (bytes, innerToken) =>
{
Interlocked.Increment(ref partNumber);
using (MemoryStream ms = new MemoryStream(bytes.Array !, 0, bytes.Count))
{
UploadPartResponse resp = await _multipartClient.UploadPartAsync(bucket, objectKey, partNumber, initResp.UploadId, ms, uploadPart =>
{
uploadPart.SseCustomerAlgorithm = req.SseCustomerAlgorithm;
uploadPart.SseCustomerKey = encryptionKey;
uploadPart.SseCustomerKeyMd5 = req.SseCustomerKeyMd5;
}, innerToken).ConfigureAwait(false);
onPartResponse?.Invoke(resp);
return(resp);
}
}, numParallelParts, token);
CompleteMultipartUploadResponse completeResp = await _multipartClient.CompleteMultipartUploadAsync(bucket, objectKey, initResp.UploadId, responses.OrderBy(x => x.PartNumber), null, token).ConfigureAwait(false);
return(completeResp);
}
finally
{
if (encryptionKey != null)
{
Array.Clear(encryptionKey, 0, encryptionKey.Length);
}
}
}
