public async Task ListObjectsMoreThanMaxKeys(S3Provider provider, string _, ISimpleClient client) { await CreateTempBucketAsync(provider, client, async tempBucket => { int concurrent = 10; int count = 11; await ParallelHelper.ExecuteAsync(Enumerable.Range(0, count), (val, token) => client.PutObjectAsync(tempBucket, val.ToString(), null, null, token), concurrent); ListObjectVersionsResponse listResp = await client.ListObjectVersionsAsync(tempBucket, r => r.MaxKeys = count - 1).ConfigureAwait(false); Assert.True(listResp.IsSuccess); Assert.True(listResp.IsTruncated); Assert.Equal(10, listResp.MaxKeys); Assert.Equal(10, listResp.Versions.Count); ListObjectVersionsResponse listResp2 = await client.ListObjectVersionsAsync(tempBucket, r => r.KeyMarker = listResp.NextKeyMarker).ConfigureAwait(false); Assert.True(listResp2.IsSuccess); Assert.False(listResp2.IsTruncated); if (provider != S3Provider.GoogleCloudStorage) { Assert.Equal(1, listResp2.Versions.Count); } }).ConfigureAwait(false); }
public async Task ListObjectsMoreThanMaxKeys(S3Provider provider, string _, ISimpleClient client) { await CreateTempBucketAsync(provider, client, async tempBucket => { int concurrent = 10; int count = 11; IEnumerable <PutObjectResponse> responses = await ParallelHelper.ExecuteAsync(Enumerable.Range(0, count), (val, token) => client.PutObjectAsync(tempBucket, val.ToString(), null, null, token), concurrent, CancellationToken.None); foreach (PutObjectResponse putResp in responses) { Assert.Equal(200, putResp.StatusCode); } ListObjectsResponse listResp = await client.ListObjectsAsync(tempBucket, r => r.MaxKeys = count - 1).ConfigureAwait(false); Assert.Equal(200, listResp.StatusCode); Assert.Equal(count - 1, listResp.KeyCount); Assert.Equal(count - 1, listResp.Objects.Count); Assert.True(listResp.IsTruncated); if (provider == S3Provider.AmazonS3) { Assert.NotEmpty(listResp.NextContinuationToken); } ListObjectsResponse listResp2 = await client.ListObjectsAsync(tempBucket, r => r.ContinuationToken = listResp.NextContinuationToken).ConfigureAwait(false); Assert.Equal(200, listResp2.StatusCode); Assert.Equal(1, listResp2.KeyCount); Assert.Equal(1, listResp2.Objects.Count); Assert.Equal(listResp.NextContinuationToken, listResp2.ContinuationToken); Assert.Null(listResp2.NextContinuationToken); Assert.False(listResp2.IsTruncated); }).ConfigureAwait(false); }
public async Task <CompleteMultipartUploadResponse> MultipartUploadAsync(CreateMultipartUploadRequest req, Stream data, int partSize = 16777216, int numParallelParts = 4, Action <UploadPartResponse>?onPartResponse = null, CancellationToken token = default) { Validator.RequireNotNull(req, nameof(req)); Validator.RequireNotNull(data, nameof(data)); foreach (IRequestWrapper wrapper in _requestWrappers) { if (wrapper.IsSupported(req)) { data = wrapper.Wrap(data, req); } } string bucket = req.BucketName; string objectKey = req.ObjectKey; byte[]? encryptionKey = null; try { if (req.SseCustomerKey != null) { encryptionKey = new byte[req.SseCustomerKey.Length]; Array.Copy(req.SseCustomerKey, 0, encryptionKey, 0, encryptionKey.Length); } CreateMultipartUploadResponse initResp = await _multipartOperations.CreateMultipartUploadAsync(req, token).ConfigureAwait(false); if (token.IsCancellationRequested) { return new CompleteMultipartUploadResponse { BucketName = bucket, ObjectKey = objectKey } } ; if (!initResp.IsSuccess) { throw new S3RequestException(initResp, "CreateMultipartUploadRequest was unsuccessful"); } IEnumerable <ArraySegment <byte> > chunks = ReadChunks(data, partSize); int partNumber = 0; IEnumerable <UploadPartResponse> responses = await ParallelHelper.ExecuteAsync(chunks, async (bytes, innerToken) => { Interlocked.Increment(ref partNumber); using (MemoryStream ms = new MemoryStream(bytes.Array !, 0, bytes.Count)) { UploadPartResponse resp = await _multipartClient.UploadPartAsync(bucket, objectKey, partNumber, initResp.UploadId, ms, uploadPart => { uploadPart.SseCustomerAlgorithm = req.SseCustomerAlgorithm; uploadPart.SseCustomerKey = encryptionKey; uploadPart.SseCustomerKeyMd5 = req.SseCustomerKeyMd5; }, innerToken).ConfigureAwait(false); onPartResponse?.Invoke(resp); return(resp); } }, numParallelParts, token); CompleteMultipartUploadResponse completeResp = await _multipartClient.CompleteMultipartUploadAsync(bucket, objectKey, initResp.UploadId, responses.OrderBy(x => x.PartNumber), null, token).ConfigureAwait(false); return(completeResp); } finally { if (encryptionKey != null) { Array.Clear(encryptionKey, 0, encryptionKey.Length); } } }