public async Task TooSmallUpload()
{
string resourceName = nameof(TooSmallUpload);
InitiateMultipartUploadResponse initResp = await ObjectClient.InitiateMultipartUploadAsync(BucketName, resourceName).ConfigureAwait(false);
Assert.Equal(BucketName, initResp.Bucket);
Assert.Equal(resourceName, initResp.Key);
Assert.NotNull(initResp.UploadId);
//4 MB is below the 5 MB limit. See https://docs.aws.amazon.com/AmazonS3/latest/dev/qfacts.html
//Note that if there only is 1 part, then it is technically the last part, and can be of any size. That's why this test has 2 parts.
byte[] file = new byte[1024 * 1024 * 4];
byte[][] chunks = file.Chunk(file.Length / 2).Select(x => x.ToArray()).ToArray();
UploadPartResponse uploadResp1 = await ObjectClient.UploadPartAsync(BucketName, resourceName, 1, initResp.UploadId, new MemoryStream(chunks[0])).ConfigureAwait(false);
Assert.True(uploadResp1.IsSuccess);
Assert.NotNull(uploadResp1.ETag);
UploadPartResponse uploadResp2 = await ObjectClient.UploadPartAsync(BucketName, resourceName, 2, initResp.UploadId, new MemoryStream(chunks[1])).ConfigureAwait(false);
Assert.True(uploadResp2.IsSuccess);
Assert.NotNull(uploadResp2.ETag);
CompleteMultipartUploadResponse completeResp = await ObjectClient.CompleteMultipartUploadAsync(BucketName, resourceName, initResp.UploadId, new[] { uploadResp1, uploadResp2 }).ConfigureAwait(false);
Assert.False(completeResp.IsSuccess);
Assert.Equal(400, completeResp.StatusCode);
}
public async Task RestoreWithSelect()
{
using (StringWriter sw = new StringWriter())
{
sw.WriteLine("name,age,status");
sw.WriteLine("santa,800,missing");
sw.WriteLine("\"donald trump\",7,present");
sw.WriteLine("fantastic fox,31,missing");
await ObjectClient.PutObjectStringAsync(BucketName, nameof(RestoreWithSelect), sw.ToString(), Encoding.UTF8, req => req.StorageClass = StorageClass.Glacier).ConfigureAwait(false);
}
RestoreObjectResponse restoreResp = await ObjectClient.RestoreObjectAsync(BucketName, nameof(RestoreWithSelect), req =>
{
req.RequestType = RestoreRequestType.Select;
req.Description = "This is a description";
req.RequestTier = RetrievalTier.Standard;
S3CsvInputFormat inputFormat = new S3CsvInputFormat();
inputFormat.HeaderUsage = HeaderUsage.Use;
S3CsvOutputFormat outputFormat = new S3CsvOutputFormat();
req.SelectParameters = new S3SelectParameters("SELECT * FROM object WHERE age > 7", inputFormat, outputFormat);
req.OutputLocation = new S3OutputLocation(BucketName, "outputJob");
req.OutputLocation.StorageClass = StorageClass.Standard;
}).ConfigureAwait(false);
Assert.Equal(202, restoreResp.StatusCode);
//TODO: List objects beneath outputJob/* and GET file to determine if format is correct
}
public async Task ManualSinglePartUpload()
{
string resourceName = nameof(ManualSinglePartUpload);
InitiateMultipartUploadResponse initResp = await ObjectClient.InitiateMultipartUploadAsync(BucketName, resourceName, request =>
{
request.SseAlgorithm = SseAlgorithm.Aes256;
request.StorageClass = StorageClass.StandardIa;
}).ConfigureAwait(false);
Assert.True(initResp.IsSuccess);
Assert.Equal(BucketName, initResp.Bucket);
Assert.Equal(resourceName, initResp.Key);
Assert.NotNull(initResp.UploadId);
byte[] file = new byte[1024 * 1024 * 5];
file[0] = (byte)'a';
UploadPartResponse uploadResp = await ObjectClient.UploadPartAsync(BucketName, resourceName, 1, initResp.UploadId, new MemoryStream(file)).ConfigureAwait(false);
Assert.True(uploadResp.IsSuccess);
Assert.NotNull(uploadResp.ETag);
Assert.Equal(SseAlgorithm.Aes256, uploadResp.SseAlgorithm);
Assert.Equal(StorageClass.StandardIa, uploadResp.StorageClass);
CompleteMultipartUploadResponse completeResp = await ObjectClient.CompleteMultipartUploadAsync(BucketName, resourceName, initResp.UploadId, new[] { uploadResp }).ConfigureAwait(false);
Assert.True(completeResp.IsSuccess);
Assert.NotNull(completeResp.ETag);
GetObjectResponse resp = await ObjectClient.GetObjectAsync(BucketName, resourceName).ConfigureAwait(false);
Assert.True(resp.IsSuccess);
Assert.Equal(file, await resp.Content.AsDataAsync().ConfigureAwait(false));
}
public async Task GetWithPrefix()
{
await CreateTempBucketAsync(async bucket =>
{
string tempObjName = "object-" + Guid.NewGuid();
string tempObjName2 = "something-" + Guid.NewGuid();
await ObjectClient.PutObjectStringAsync(bucket, tempObjName, "hello").ConfigureAwait(false);
await ObjectClient.PutObjectStringAsync(bucket, tempObjName2, "world!").ConfigureAwait(false);
GetBucketResponse gResp = await BucketClient.GetBucketAsync(bucket, request => request.Prefix = "object").ConfigureAwait(false);
Assert.True(gResp.IsSuccess);
Assert.Equal(1, gResp.KeyCount);
Assert.Equal(1, gResp.Objects.Count);
Assert.Equal("object", gResp.Prefix);
Assert.Equal(tempObjName, gResp.Objects[0].Name);
Assert.Equal(DateTime.UtcNow, gResp.Objects[0].LastModified.DateTime, TimeSpan.FromSeconds(5));
Assert.Equal("\"5d41402abc4b2a76b9719d911017c592\"", gResp.Objects[0].ETag);
Assert.Equal(5, gResp.Objects[0].Size);
Assert.Equal(StorageClass.Standard, gResp.Objects[0].StorageClass);
}).ConfigureAwait(false);
}
public async Task ListIncompleteParts()
{
await CreateTempBucketAsync(async bucket =>
{
InitiateMultipartUploadResponse initResp = await ObjectClient.InitiateMultipartUploadAsync(bucket, nameof(ListIncompleteParts)).ConfigureAwait(false);
byte[] file = new byte[5 * 1024];
using (MemoryStream ms = new MemoryStream(file))
await ObjectClient.UploadPartAsync(bucket, nameof(ListIncompleteParts), 1, initResp.UploadId, ms).ConfigureAwait(false);
ListMultipartUploadsResponse listResp = await BucketClient.ListMultipartUploadsAsync(bucket).ConfigureAwait(false);
Assert.Equal(bucket, listResp.Bucket);
Assert.Equal("ListIncompleteParts", listResp.NextKeyMarker);
Assert.NotEmpty(listResp.NextUploadIdMarker);
Assert.Equal(1000, listResp.MaxUploads);
Assert.False(listResp.IsTruncated);
S3Upload upload = Assert.Single(listResp.Uploads);
Assert.Equal(listResp.NextKeyMarker, upload.Name);
Assert.Equal(listResp.NextUploadIdMarker, upload.UploadId);
Assert.Equal(TestConstants.TestUsername, upload.Initiator.Name);
Assert.Equal(StorageClass.Standard, upload.StorageClass);
Assert.Equal(DateTime.UtcNow, upload.Initiated.DateTime, TimeSpan.FromSeconds(5));
}).ConfigureAwait(false);
}
public async Task ListObjects()
{
await CreateTempBucketAsync(async bucket =>
{
//Create 3 objects in bucket, including an incomplete multipart upload
await UploadAsync(bucket, "resource1").ConfigureAwait(false);
await UploadAsync(bucket, "resource2").ConfigureAwait(false);
InitiateMultipartUploadResponse initResp = await ObjectClient.InitiateMultipartUploadAsync(bucket, "multipart").ConfigureAwait(false);
//List the objects
List <S3Object> list = await BucketClient.GetBucketRecursiveAsync(bucket, true).ToListAsync().ConfigureAwait(false);
//Only 2 objects should be present, as one of them is only initiated
Assert.Equal(2, list.Count);
Assert.Equal("resource1", list[0].Name);
Assert.Equal("resource2", list[1].Name);
//List multipart transfers
List <S3Upload> uploads = await BucketClient.ListAllMultipartUploadsAsync(bucket).ToListAsync().ConfigureAwait(false);
S3Upload upload = Assert.Single(uploads);
Assert.Equal("multipart", upload.Name);
}).ConfigureAwait(false);
}
public async Task DeleteObjects()
{
List <S3DeleteInfo> resources = new List <S3DeleteInfo>(2);
resources.Add(new S3DeleteInfo(nameof(DeleteObjects) + "1"));
resources.Add(new S3DeleteInfo(nameof(DeleteObjects) + "2", "versionnotfound"));
await UploadAsync(resources[0].Name).ConfigureAwait(false);
await UploadAsync(resources[1].Name).ConfigureAwait(false);
DeleteObjectsResponse resp = await ObjectClient.DeleteObjectsAsync(BucketName, resources, req => req.Quiet = false).ConfigureAwait(false);
S3DeletedObject?delObj = Assert.Single(resp.Deleted);
Assert.Equal(resources[0].Name, delObj.ObjectKey);
Assert.True(delObj.IsDeleteMarker);
Assert.NotEmpty(delObj.DeleteMarkerVersionId);
S3DeleteError?errorObj = Assert.Single(resp.Errors);
Assert.Equal(resources[1].Name, errorObj.ObjectKey);
Assert.Equal(resources[1].VersionId, errorObj.VersionId);
Assert.Equal(ErrorCode.NoSuchVersion, errorObj.Code);
Assert.Equal("The specified version does not exist.", errorObj.Message);
}
public async Task DeleteMarker()
{
await UploadAsync(nameof(DeleteMarker)).ConfigureAwait(false);
DeleteObjectResponse resp = await ObjectClient.DeleteObjectAsync(BucketName, nameof(DeleteMarker)).ConfigureAwait(false);
Assert.True(resp.IsDeleteMarker);
}
public async Task HeadObject()
{
await UploadAsync(nameof(HeadObject)).ConfigureAwait(false);
HeadObjectResponse gResp = await ObjectClient.HeadObjectAsync(BucketName, nameof(HeadObject)).ConfigureAwait(false);
Assert.Equal(200, gResp.StatusCode);
}
public async Task PutObjectInvalidCharacters(string name)
{
//These 2 test cases came after an exhaustive search in the whole UTF-16 character space.
PutObjectResponse resp = await ObjectClient.PutObjectStringAsync(BucketName, name, string.Empty).ConfigureAwait(false);
Assert.False(resp.IsSuccess);
}
public async Task <Object.Object> GetObject(GetRequest request, DateTime?deadline = null, CancellationToken context = default)
{
using var call = ObjectClient.Get(request, deadline: deadline, cancellationToken: context);
var obj = new Object.Object();
var payload = Array.Empty <byte>();
int offset = 0;
while (await call.ResponseStream.MoveNext())
{
var resp = call.ResponseStream.Current;
if (!resp.Verify())
{
throw new InvalidOperationException("invalid object get response");
}
CheckStatus(resp);
switch (resp.Body.ObjectPartCase)
{
case GetResponse.Types.Body.ObjectPartOneofCase.Init:
{
obj.ObjectId = resp.Body.Init.ObjectId;
obj.Signature = resp.Body.Init.Signature;
obj.Header = resp.Body.Init.Header;
payload = new byte[obj.PayloadSize];
break;
}
case GetResponse.Types.Body.ObjectPartOneofCase.Chunk:
{
if (payload.Length == 0)
{
throw new InvalidOperationException("missing init");
}
var chunk = resp.Body.Chunk;
if (obj.PayloadSize < (ulong)(offset + chunk.Length))
{
throw new InvalidOperationException("data exceeds PayloadSize");
}
resp.Body.Chunk.CopyTo(payload, offset);
offset += chunk.Length;
break;
}
case GetResponse.Types.Body.ObjectPartOneofCase.SplitInfo:
{
throw new SplitInfoException(resp.Body.SplitInfo);
}
default:
throw new FormatException("malformed object get reponse");
}
}
if ((ulong)offset < obj.PayloadSize)
{
throw new InvalidOperationException("data is less than PayloadSize");
}
obj.Payload = ByteString.CopyFrom(payload);
return(obj);
}
public async Task Range()
{
await ObjectClient.PutObjectStringAsync(BucketName, nameof(Range), "123456789012345678901234567890123456789012345678901234567890").ConfigureAwait(false);
GetObjectResponse resp = await ObjectClient.GetObjectAsync(BucketName, nameof(Range), request => request.Range.Add(0, 10)).ConfigureAwait(false);
Assert.Equal(206, resp.StatusCode);
Assert.Equal(11, resp.ContentLength);
}
public async Task TestTransientNetworkError_Nonseekable()
{
using NonSeekableStream ms = new NonSeekableStream(new byte[4096]);
PutObjectResponse response = await ObjectClient.PutObjectAsync(BucketName, nameof(TestTransientNetworkError_Nonseekable), ms).ConfigureAwait(false);
Assert.True(response.IsSuccess);
Assert.True(_handler.RequestCounter >= 2);
}
public async Task HeadObjectContentRange()
{
await UploadAsync(nameof(HeadObjectContentRange)).ConfigureAwait(false);
HeadObjectResponse headResp = await ObjectClient.HeadObjectAsync(BucketName, nameof(HeadObjectContentRange), req => req.Range.Add(0, 2)).ConfigureAwait(false);
Assert.Equal("bytes", headResp.AcceptRanges);
Assert.Equal("bytes 0-2/4", headResp.ContentRange);
}
public async Task HeadObjectWebsiteRedirect()
{
await UploadAsync(nameof(HeadObjectWebsiteRedirect), request => request.WebsiteRedirectLocation = "https://google.com").ConfigureAwait(false);
HeadObjectResponse resp1 = await ObjectClient.HeadObjectAsync(BucketName, nameof(HeadObjectWebsiteRedirect)).ConfigureAwait(false);
Assert.Equal("https://google.com", resp1.WebsiteRedirectLocation);
Assert.Equal(200, resp1.StatusCode);
}
public async Task SendNonSeekableStream()
{
byte[] data = new byte[1024 * 10];
Array.Fill(data, (byte)'A');
//We test if it is possible send a non-seekable stream. This should succeed as we use ChunkedStream
PutObjectResponse resp = await ObjectClient.PutObjectAsync(BucketName, nameof(SendNonSeekableStream), new NonSeekableStream(data)).ConfigureAwait(false);
Assert.True(resp.IsSuccess);
}
public async Task PutObjectTooManyTags()
{
await Assert.ThrowsAsync <Exception>(async() => await ObjectClient.PutObjectStringAsync(BucketName, nameof(PutObjectTooManyTags), "data", null, request =>
{
for (int i = 0; i < 51; i++)
{
request.Tags.Add(i.ToString(NumberFormatInfo.InvariantInfo), i.ToString(NumberFormatInfo.InvariantInfo));
}
}).ConfigureAwait(false)).ConfigureAwait(false);
}
public async Task DeleteObjectsRequestPayer()
{
PutObjectResponse putResp2 = await ObjectClient.PutObjectAsync(BucketName, nameof(DeleteObjectsRequestPayer), null, req => req.RequestPayer = Payer.Requester).ConfigureAwait(false);
Assert.True(putResp2.RequestCharged);
DeleteObjectsResponse delResp2 = await ObjectClient.DeleteObjectsAsync(BucketName, new[] { nameof(DeleteObjectsRequestPayer) }, req => req.RequestPayer = Payer.Requester).ConfigureAwait(false);
Assert.True(delResp2.RequestCharged);
}
public async Task PutObjectRequestPayer()
{
PutObjectResponse putResp = await ObjectClient.PutObjectAsync(BucketName, nameof(PutObjectRequestPayer), null, req => req.RequestPayer = Payer.Requester).ConfigureAwait(false);
Assert.True(putResp.RequestCharged);
GetObjectResponse getResp = await ObjectClient.GetObjectAsync(BucketName, nameof(PutObjectRequestPayer), req => req.RequestPayer = Payer.Requester).ConfigureAwait(false);
Assert.True(getResp.RequestCharged);
}
public async Task TestTimeoutError_NonSeekableStream()
{
using NonSeekableStream ms = new NonSeekableStream(new byte[4096]);
PutObjectResponse response = await ObjectClient.PutObjectAsync(BucketName, nameof(TestTimeoutError_NonSeekableStream), ms).ConfigureAwait(false);
// Request should succeed after N tries
Assert.True(response.IsSuccess);
Assert.Equal(2, _handler.RequestCounter);
}
public async Task PutObjectValidCharacters(string name)
{
PutObjectResponse putResp = await ObjectClient.PutObjectStringAsync(BucketName, name, string.Empty).ConfigureAwait(false);
Assert.True(putResp.IsSuccess);
GetObjectResponse getResp = await ObjectClient.GetObjectAsync(BucketName, name).ConfigureAwait(false);
Assert.True(getResp.IsSuccess);
}
private async Task UploadString(string bucketName, string objName, SemaphoreSlim semaphore)
{
try
{
await ObjectClient.PutObjectStringAsync(bucketName, objName, string.Empty).ConfigureAwait(false);
}
finally
{
semaphore.Release();
}
}
public async Task Restore()
{
//Upload an object to glacier
PutObjectResponse putResp = await UploadAsync(nameof(Restore), req => req.StorageClass = StorageClass.Glacier).ConfigureAwait(false);
Assert.Equal(StorageClass.Glacier, putResp.StorageClass);
RestoreObjectResponse restoreResp = await ObjectClient.RestoreObjectAsync(BucketName, nameof(Restore), req => req.Days = 2).ConfigureAwait(false);
Assert.Equal(202, restoreResp.StatusCode);
}
public async Task NameTests(string name)
{
await ObjectClient.PutObjectStringAsync(BucketName, name, name).ConfigureAwait(false);
GetObjectResponse dlResp = await ObjectClient.GetObjectAsync(BucketName, name).ConfigureAwait(false);
Assert.Equal("binary/octet-stream", dlResp.ContentType);
Assert.Equal(name.Length, dlResp.ContentLength);
Assert.Equal(name, Encoding.UTF8.GetString(await dlResp.Content.AsDataAsync().ConfigureAwait(false)));
}
public async Task PutGetObjectAcl()
{
string objectKey = nameof(PutGetObjectAcl);
//Create an object
await UploadAsync(objectKey).ConfigureAwait(false);
//Get the ACL, which should be the default one (owner has ACL)
GetObjectAclResponse getResp = await ObjectClient.GetObjectAclAsync(BucketName, objectKey).ConfigureAwait(false);
Assert.True(getResp.IsSuccess);
S3Grant grant = Assert.Single(getResp.Grants);
Assert.Equal(TestConstants.TestUserId, grant.Id);
Assert.Equal(TestConstants.TestUsername, grant.Name);
Assert.Equal(Permission.FullControl, grant.Permission);
Assert.Equal(GrantType.User, grant.Type);
//Update the object to have another ACL using Canned ACLs
PutObjectAclResponse putResp = await ObjectClient.PutObjectAclAsync(BucketName, objectKey, req => req.Acl = ObjectCannedAcl.PublicReadWrite).ConfigureAwait(false);
Assert.True(putResp.IsSuccess);
GetObjectAclResponse getResp2 = await ObjectClient.GetObjectAclAsync(BucketName, objectKey).ConfigureAwait(false);
Assert.True(getResp2.IsSuccess);
Assert.Equal(TestConstants.TestUserId, getResp2.Owner.Id);
Assert.Equal(TestConstants.TestUsername, getResp2.Owner.Name);
Assert.Equal(3, getResp2.Grants.Count);
//This is the default owner ACL
S3Grant first = getResp2.Grants[0];
Assert.Equal(TestConstants.TestUserId, first.Id);
Assert.Equal(TestConstants.TestUsername, first.Name);
Assert.Equal(Permission.FullControl, first.Permission);
Assert.Equal(GrantType.User, first.Type);
//Next 2 ACLs should be READ + WRITE for AllUsers
S3Grant second = getResp2.Grants[1];
Assert.Equal("http://acs.amazonaws.com/groups/global/AllUsers", second.Uri);
Assert.Equal(Permission.Read, second.Permission);
Assert.Equal(GrantType.Group, second.Type);
S3Grant third = getResp2.Grants[2];
Assert.Equal("http://acs.amazonaws.com/groups/global/AllUsers", third.Uri);
Assert.Equal(Permission.Write, third.Permission);
Assert.Equal(GrantType.Group, third.Type);
}
public async Task GetObjectContentRange()
{
await UploadAsync(nameof(GetObjectContentRange)).ConfigureAwait(false);
GetObjectResponse getResp = await ObjectClient.GetObjectAsync(BucketName, nameof(GetObjectContentRange), req => req.Range.Add(0, 2)).ConfigureAwait(false);
Assert.Equal(206, getResp.StatusCode);
Assert.Equal(3, getResp.ContentLength);
Assert.Equal("bytes", getResp.AcceptRanges);
Assert.Equal("bytes 0-2/4", getResp.ContentRange);
Assert.Equal("tes", await getResp.Content.AsStringAsync().ConfigureAwait(false));
}
public async Task BasicCRDTest()
{
PutObjectResponse pResp = await UploadAsync(nameof(BasicCRDTest)).ConfigureAwait(false);
Assert.Equal(200, pResp.StatusCode);
await AssertAsync(nameof(BasicCRDTest)).ConfigureAwait(false);
DeleteObjectResponse dResp = await ObjectClient.DeleteObjectAsync(BucketName, nameof(BasicCRDTest)).ConfigureAwait(false);
Assert.Equal(204, dResp.StatusCode);
}
public async Task RangeFluid()
{
await ObjectClient.PutObjectStringAsync(BucketName, nameof(RangeFluid), "123456789012345678901234567890123456789012345678901234567890").ConfigureAwait(false);
GetObjectResponse resp = await Transfer.Download(BucketName, nameof(RangeFluid))
.WithRange(0, 10)
.ExecuteAsync()
.ConfigureAwait(false);
Assert.Equal(206, resp.StatusCode);
Assert.Equal(11, resp.ContentLength);
}
public async Task HeadObjectMultipleTags()
{
await UploadAsync(nameof(HeadObjectMultipleTags), request =>
{
request.Tags.Add("mykey1", "myvalue1");
request.Tags.Add("mykey2", "myvalue2");
}).ConfigureAwait(false);
HeadObjectResponse gResp = await ObjectClient.HeadObjectAsync(BucketName, nameof(HeadObjectMultipleTags)).ConfigureAwait(false);
Assert.Equal(2, gResp.TagCount);
}
public async Task HeadObjectLifecycle()
{
PutObjectResponse putResp = await UploadAsync(nameof(HeadObjectLifecycle)).ConfigureAwait(false);
Assert.Equal(200, putResp.StatusCode);
HeadObjectResponse headResp = await ObjectClient.HeadObjectAsync(BucketName, nameof(HeadObjectLifecycle)).ConfigureAwait(false);
//Expiration should work on head too
Assert.Equal(DateTime.UtcNow.AddDays(2).Date, headResp.LifeCycleExpiresOn !.Value.UtcDateTime.Date);
Assert.Equal("ExpireAll", headResp.LifeCycleRuleId);
}
public SystemInfoService(ObjectClient client)
{
Interval = 10000;
//Interval = 9999999;
_client = client;
}