public void GetAvailableChunksReturnsRetryAfterWhenNoNodes()
{
var queryParams = new Dictionary <string, string> {
{ "job", "1a85e743-ec8f-4789-afec-97e587a26936" }
};
var headers = new Dictionary <string, string> {
{ "Retry-After", "300" }
};
var responseContent = ReadResource(EmptyGetAvailableJobChunksResponseResourceName);
var client = MockNetwork
.Expecting(HttpVerb.GET, "/_rest_/job_chunk", queryParams, "")
.Returning(HttpStatusCode.OK, responseContent, headers)
.AsClient;
var response = client.GetAvailableJobChunks(new GetAvailableJobChunksRequest(Guid.Parse("1a85e743-ec8f-4789-afec-97e587a26936")));
TimeSpan?retryValue = null;
response.Match(
(ts, jobResponse) => Assert.Fail(),
() => Assert.Fail(),
retryAfter => retryValue = retryAfter
);
Assert.NotNull(retryValue);
Assert.AreEqual(TimeSpan.FromMinutes(5), retryValue.Value);
}
public void TestGetObject()
{
var stringResponse = "object contents";
using (var memoryStream = new MemoryStream())
{
var jobId = Guid.Parse("6f7a31c1-7ddc-46d3-975d-be26d9878493");
var offset = 10L;
var expectedQueryParams = new Dictionary <string, string>
{
{ "job", "6f7a31c1-7ddc-46d3-975d-be26d9878493" },
{ "offset", "10" }
};
MockNetwork
.Expecting(HttpVerb.GET, "/bucketName/object", expectedQueryParams, "")
.Returning(HttpStatusCode.OK, stringResponse, _emptyHeaders)
.AsClient
.GetObject(new GetObjectRequest("bucketName", "object", jobId, offset, memoryStream));
memoryStream.Position = 0L;
using (var reader = new StreamReader(memoryStream))
{
Assert.AreEqual(stringResponse, reader.ReadToEnd());
}
}
}
public void GetAvailableChunksReturnsJobWhenSuccess()
{
var queryParams = new Dictionary <string, string> {
{ "job", "1a85e743-ec8f-4789-afec-97e587a26936" }
};
var responseContent = ReadResource(GetAvailableJobChunksResponseResourceName);
var client = MockNetwork
.Expecting(HttpVerb.GET, "/_rest_/job_chunk", queryParams, "")
.Returning(HttpStatusCode.OK, responseContent, new Dictionary <string, string> {
{ "retry-after", "123" }
})
.AsClient;
var response = client.GetAvailableJobChunks(new GetAvailableJobChunksRequest(Guid.Parse("1a85e743-ec8f-4789-afec-97e587a26936")));
TimeSpan timeSpan = TimeSpan.MinValue;
JobResponse jobChunks = null;
response.Match(
(ts, jobResponse) => { timeSpan = ts; jobChunks = jobResponse; },
() => Assert.Fail(),
retryAfter => Assert.Fail()
);
Assert.NotNull(jobChunks);
var expectedNodeIds = new[]
{
Guid.Parse("a02053b9-0147-11e4-8d6a-002590c1177c"),
Guid.Parse("95e97010-8e70-4733-926c-aeeb21796848")
};
CollectionAssert.AreEqual(expectedNodeIds, jobChunks.ObjectLists.Select(chunk => chunk.NodeId.Value));
Assert.AreEqual(TimeSpan.FromSeconds(123), timeSpan);
}
public void TestGetService()
{
var responseContent = "<ListAllMyBucketsResult><Owner><ID>ryan</ID><DisplayName>ryan</DisplayName></Owner><Buckets><Bucket><Name>testBucket2</Name><CreationDate>2013-12-11T23:20:09</CreationDate></Bucket><Bucket><Name>bulkTest</Name><CreationDate>2013-12-11T23:20:09</CreationDate></Bucket><Bucket><Name>bulkTest1</Name><CreationDate>2013-12-11T23:20:09</CreationDate></Bucket><Bucket><Name>bulkTest2</Name><CreationDate>2013-12-11T23:20:09</CreationDate></Bucket><Bucket><Name>bulkTest3</Name><CreationDate>2013-12-11T23:20:09</CreationDate></Bucket><Bucket><Name>bulkTest4</Name><CreationDate>2013-12-11T23:20:09</CreationDate></Bucket><Bucket><Name>bulkTest5</Name><CreationDate>2013-12-11T23:20:09</CreationDate></Bucket><Bucket><Name>bulkTest6</Name><CreationDate>2013-12-11T23:20:09</CreationDate></Bucket><Bucket><Name>testBucket3</Name><CreationDate>2013-12-11T23:20:09</CreationDate></Bucket><Bucket><Name>testBucket1</Name><CreationDate>2013-12-11T23:20:09</CreationDate></Bucket><Bucket><Name>testbucket</Name><CreationDate>2013-12-11T23:20:09</CreationDate></Bucket></Buckets></ListAllMyBucketsResult>";
var expectedBuckets = new[] {
new { Key = "testBucket2", CreationDate = "2013-12-11T23:20:09" },
new { Key = "bulkTest", CreationDate = "2013-12-11T23:20:09" },
new { Key = "bulkTest1", CreationDate = "2013-12-11T23:20:09" },
new { Key = "bulkTest2", CreationDate = "2013-12-11T23:20:09" },
new { Key = "bulkTest3", CreationDate = "2013-12-11T23:20:09" },
new { Key = "bulkTest4", CreationDate = "2013-12-11T23:20:09" },
new { Key = "bulkTest5", CreationDate = "2013-12-11T23:20:09" },
new { Key = "bulkTest6", CreationDate = "2013-12-11T23:20:09" },
new { Key = "testBucket3", CreationDate = "2013-12-11T23:20:09" },
new { Key = "testBucket1", CreationDate = "2013-12-11T23:20:09" },
new { Key = "testbucket", CreationDate = "2013-12-11T23:20:09" }
};
var response = MockNetwork
.Expecting(HttpVerb.GET, "/", _emptyQueryParams, "")
.Returning(HttpStatusCode.OK, responseContent, _emptyHeaders)
.AsClient
.GetService(new GetServiceRequest());
Assert.AreEqual("ryan", response.Owner.DisplayName);
Assert.AreEqual("ryan", response.Owner.Id);
Assert.AreEqual(expectedBuckets.Length, response.Buckets.Count);
for (var i = 0; i < expectedBuckets.Length; i++)
{
Assert.AreEqual(expectedBuckets[i].Key, response.Buckets[i].Name);
Assert.AreEqual(expectedBuckets[i].CreationDate, response.Buckets[i].CreationDate);
}
}
public void TestGetWorseService()
{
MockNetwork
.Expecting(HttpVerb.GET, "/", _emptyQueryParams, "")
.Returning(HttpStatusCode.OK, "", _emptyHeaders)
.AsClient
.GetService(new GetServiceRequest());
}
public void TestGetBadBucket()
{
MockNetwork
.Expecting(HttpVerb.GET, "/bucketName", _emptyQueryParams, "")
.Returning(HttpStatusCode.BadRequest, "", _emptyHeaders)
.AsClient
.GetBucket(new GetBucketRequest("bucketName"));
}
public void TestDeleteObject()
{
MockNetwork
.Expecting(HttpVerb.DELETE, "/bucketName/my/file.txt", _emptyQueryParams, "")
.Returning(HttpStatusCode.NoContent, "", _emptyHeaders)
.AsClient
.DeleteObject(new DeleteObjectRequest("bucketName", "my/file.txt"));
}
public void TestPutBucket()
{
MockNetwork
.Expecting(HttpVerb.PUT, "/bucketName", _emptyQueryParams, "")
.Returning(HttpStatusCode.OK, "", _emptyHeaders)
.AsClient
.PutBucket(new PutBucketRequest("bucketName"));
}
public void TestGetJob()
{
var stringResponse = ReadResource(JobResponseResourceName);
var client = MockNetwork
.Expecting(HttpVerb.GET, "/_rest_/job/1a85e743-ec8f-4789-afec-97e587a26936", _emptyQueryParams, "")
.Returning(HttpStatusCode.OK, stringResponse, _emptyHeaders)
.AsClient;
CheckJobResponse(client.GetJob(new GetJobRequest(Guid.Parse("1a85e743-ec8f-4789-afec-97e587a26936"))));
}
public void TestGetJobList()
{
var responseContent = "<Jobs><Job BucketName=\"bucketName\" JobId=\"a4a586a1-cb80-4441-84e2-48974e982d51\" Priority=\"NORMAL\" RequestType=\"PUT\" StartDate=\"2014-05-22T18:24:00.000Z\" Status=\"IN_PROGRESS\"/></Jobs>";
var client = MockNetwork
.Expecting(HttpVerb.GET, "/_rest_/job", new Dictionary <string, string>(), "")
.Returning(HttpStatusCode.OK, responseContent, _emptyHeaders)
.AsClient;
var jobs = client.GetJobList(new GetJobListRequest()).Jobs.ToList();
Assert.AreEqual(1, jobs.Count);
CheckJobInfo(jobs[0]);
}
public static IMockNetworkWithExpectation Expecting(
HttpVerb verb,
string path,
IDictionary<string, string> queryParams,
string requestContent)
{
var mock = new MockNetwork();
mock._verb = verb;
mock._path = path;
mock._queryParams = queryParams;
mock._requestContent = requestContent;
return mock;
}
public static IMockNetworkWithExpectation Expecting(
HttpVerb verb,
string path,
IDictionary <string, string> queryParams,
string requestContent)
{
var mock = new MockNetwork();
mock._verb = verb;
mock._path = path;
mock._queryParams = queryParams;
mock._requestContent = requestContent;
return(mock);
}
public void GetPhysicalPlacementReturnsPartialDetails()
{
var requestString = "<Objects><Object Name=\"o1\" /><Object Name=\"o2\" /><Object Name=\"o3\" /><Object Name=\"o4\" /></Objects>";
var queryParams = new Dictionary <string, string>
{
{ "operation", "get_physical_placement" },
};
var responseContent = ReadResource(GetPhysicalPlacementResponseResourceName);
var client = MockNetwork
.Expecting(HttpVerb.PUT, "/_rest_/bucket/test_bucket", queryParams, requestString)
.Returning(HttpStatusCode.OK, responseContent, _emptyHeaders)
.AsClient;
var result = client.GetAggregatePhysicalPlacement(new GetAggregatePhysicalPlacementRequest("test_bucket", new[] { "o1", "o2", "o3", "o4" }));
CollectionAssert.AreEqual(new[] { _testTape }, result.Tapes.ToArray(), new TapeComparer());
}
public void TestPutObject()
{
var stringRequest = "object content";
var jobId = Guid.Parse("e5d4adce-e170-4915-ba04-595fab30df81");
var offset = 10L;
var expectedQueryParams = new Dictionary <string, string>
{
{ "job", "e5d4adce-e170-4915-ba04-595fab30df81" },
{ "offset", "10" }
};
MockNetwork
.Expecting(HttpVerb.PUT, "/bucketName/object", expectedQueryParams, stringRequest)
.Returning(HttpStatusCode.OK, stringRequest, _emptyHeaders)
.AsClient
.PutObject(new PutObjectRequest("bucketName", "object", jobId, offset, HelpersForTest.StringToStream(stringRequest)));
}
public void AllocateChunkReturnsChunkGoneWhen404()
{
var queryParams = new Dictionary <string, string> {
{ "operation", "allocate" }
};
var client = MockNetwork
.Expecting(HttpVerb.PUT, "/_rest_/job_chunk/f58370c2-2538-4e78-a9f8-e4d2676bdf44", queryParams, "")
.Returning(HttpStatusCode.NotFound, "", _emptyHeaders)
.AsClient;
var response = client.AllocateJobChunk(new AllocateJobChunkRequest(Guid.Parse("f58370c2-2538-4e78-a9f8-e4d2676bdf44")));
var chunkIsGone = false;
response.Match(
chunk => Assert.Fail(),
retryAfter => Assert.Fail(),
() => chunkIsGone = true
);
Assert.IsTrue(chunkIsGone);
}
public void TestGetCompletedJob()
{
var stringResponse = ReadResource(CompletedJobResponseResourceName);
var client = MockNetwork
.Expecting(HttpVerb.GET, "/_rest_/job/350225fb-ec92-4456-a09d-5ffb7c7830bb", _emptyQueryParams, "")
.Returning(HttpStatusCode.OK, stringResponse, _emptyHeaders)
.AsClient;
var jobId = Guid.Parse("350225fb-ec92-4456-a09d-5ffb7c7830bb");
var job = client.GetJob(new GetJobRequest(jobId));
Assert.AreEqual("avid-bucket", job.BucketName);
Assert.AreEqual(ChunkOrdering.InOrder, job.ChunkOrder);
Assert.AreEqual(jobId, job.JobId);
CollectionAssert.AreEqual(Enumerable.Empty <Node>(), job.Nodes);
CollectionAssert.AreEqual(Enumerable.Empty <JobObjectList>(), job.ObjectLists);
Assert.AreEqual("NORMAL", job.Priority);
Assert.AreEqual("PUT", job.RequestType);
Assert.AreEqual(new DateTime(2015, 5, 5, 17, 9, 30, 0, DateTimeKind.Utc), job.StartDate.ToUniversalTime());
Assert.AreEqual(JobStatus.COMPLETED, job.Status);
}
public void GetAvailableChunksReturnsJobGoneWhen404()
{
var queryParams = new Dictionary <string, string> {
{ "job", "1a85e743-ec8f-4789-afec-97e587a26936" }
};
var client = MockNetwork
.Expecting(HttpVerb.GET, "/_rest_/job_chunk", queryParams, "")
.Returning(HttpStatusCode.NotFound, "", _emptyHeaders)
.AsClient;
var response = client.GetAvailableJobChunks(new GetAvailableJobChunksRequest(Guid.Parse("1a85e743-ec8f-4789-afec-97e587a26936")));
var wasJobGone = false;
response.Match(
(ts, jobResponse) => Assert.Fail(),
() => wasJobGone = true,
retryAfter => Assert.Fail()
);
Assert.IsTrue(wasJobGone);
}
private void RunBulkTest(
string operation,
Func <IDs3Client, List <Ds3Object>, JobResponse> makeCall,
IDictionary <string, string> additionalQueryParams)
{
var files = new[] {
new { Key = "client00obj000000-8000000", Size = 8192000000L },
new { Key = "client00obj000001-8000000", Size = 8192000000L },
new { Key = "client00obj000002-8000000", Size = 8192000000L },
new { Key = "client00obj000003-8000000", Size = 8192000000L },
new { Key = "client00obj000004-8000000", Size = 8192000000L },
new { Key = "client00obj000005-8000000", Size = 8192000000L },
new { Key = "client00obj000006-8000000", Size = 8192000000L },
new { Key = "client00obj000007-8000000", Size = 8192000000L },
new { Key = "client00obj000008-8000000", Size = 8192000000L },
new { Key = "client00obj000009-8000000", Size = 8192000000L }
};
var stringRequest = operation == "start_bulk_get"
? "<Objects><Object Name=\"client00obj000000-8000000\" /><Object Name=\"client00obj000001-8000000\" /><Object Name=\"client00obj000002-8000000\" /><Object Name=\"client00obj000003-8000000\" /><Object Name=\"client00obj000004-8000000\" /><Object Name=\"client00obj000005-8000000\" /><Object Name=\"client00obj000006-8000000\" /><Object Name=\"client00obj000007-8000000\" /><Object Name=\"client00obj000008-8000000\" /><Object Name=\"client00obj000009-8000000\" /></Objects>"
: "<Objects><Object Name=\"client00obj000000-8000000\" Size=\"8192000000\" /><Object Name=\"client00obj000001-8000000\" Size=\"8192000000\" /><Object Name=\"client00obj000002-8000000\" Size=\"8192000000\" /><Object Name=\"client00obj000003-8000000\" Size=\"8192000000\" /><Object Name=\"client00obj000004-8000000\" Size=\"8192000000\" /><Object Name=\"client00obj000005-8000000\" Size=\"8192000000\" /><Object Name=\"client00obj000006-8000000\" Size=\"8192000000\" /><Object Name=\"client00obj000007-8000000\" Size=\"8192000000\" /><Object Name=\"client00obj000008-8000000\" Size=\"8192000000\" /><Object Name=\"client00obj000009-8000000\" Size=\"8192000000\" /></Objects>";
var stringResponse = ReadResource(JobResponseResourceName);
var inputObjects = files.Select(f => new Ds3Object(f.Key, f.Size)).ToList();
var queryParams = new Dictionary <string, string>()
{
{ "operation", operation }
};
foreach (var kvp in additionalQueryParams)
{
queryParams.Add(kvp.Key, kvp.Value);
}
var client = MockNetwork
.Expecting(HttpVerb.PUT, "/_rest_/bucket/bucket8192000000", queryParams, stringRequest)
.Returning(HttpStatusCode.OK, stringResponse, _emptyHeaders)
.AsClient;
CheckJobResponse(makeCall(client, inputObjects));
}
public void AllocateChunkReturnsRetryWhen503AndHeader()
{
var queryParams = new Dictionary <string, string> {
{ "operation", "allocate" }
};
var headers = new Dictionary <string, string> {
{ "Retry-After", "300" }
};
var client = MockNetwork
.Expecting(HttpVerb.PUT, "/_rest_/job_chunk/f58370c2-2538-4e78-a9f8-e4d2676bdf44", queryParams, "")
.Returning(HttpStatusCode.ServiceUnavailable, "", headers)
.AsClient;
var response = client.AllocateJobChunk(new AllocateJobChunkRequest(Guid.Parse("f58370c2-2538-4e78-a9f8-e4d2676bdf44")));
TimeSpan?retryResult = null;
response.Match(
chunk => Assert.Fail(),
retryAfter => retryResult = retryAfter,
Assert.Fail
);
Assert.NotNull(retryResult);
Assert.AreEqual(TimeSpan.FromMinutes(5), retryResult.Value);
}
public void AllocateChunkReturnsChunkWithNodeWhenAllocated()
{
var responseContent = ReadResource(AllocateJobChunkResponseResourceName);
var queryParams = new Dictionary <string, string> {
{ "operation", "allocate" }
};
var client = MockNetwork
.Expecting(HttpVerb.PUT, "/_rest_/job_chunk/f58370c2-2538-4e78-a9f8-e4d2676bdf44", queryParams, "")
.Returning(HttpStatusCode.OK, responseContent, _emptyHeaders)
.AsClient;
var response = client.AllocateJobChunk(new AllocateJobChunkRequest(Guid.Parse("f58370c2-2538-4e78-a9f8-e4d2676bdf44")));
JobObjectList chunkResult = null;
response.Match(
chunk => chunkResult = chunk,
retryAfter => Assert.Fail(),
Assert.Fail
);
Assert.NotNull(chunkResult);
Assert.AreEqual(Guid.Parse("a02053b9-0147-11e4-8d6a-002590c1177c"), chunkResult.NodeId);
Assert.AreEqual(Guid.Parse("f58370c2-2538-4e78-a9f8-e4d2676bdf44"), chunkResult.ChunkId);
Assert.AreEqual(0, chunkResult.ChunkNumber);
Assert.AreEqual(14, chunkResult.Objects.Count());
}
public static IMockNetworkWithExpectation Expecting(
HttpVerb verb,
string path,
IDictionary<string, string> queryParams,
IDictionary<string, string> requestHeaders,
string requestContent)
{
var mock = new MockNetwork
{
_verb = verb,
_path = path,
_queryParams = queryParams,
_requestHeaders = requestHeaders,
_requestContent = requestContent
};
return mock;
}
public void TestGetBucket()
{
var xmlResponse = "<ListBucketResult><Name>remoteTest16</Name><Prefix/><Marker/><MaxKeys>1000</MaxKeys><IsTruncated>false</IsTruncated><Contents><Key>user/hduser/gutenberg/20417.txt.utf-8</Key><LastModified>2014-01-03T13:26:47.000Z</LastModified><ETag>NOTRETURNED</ETag><Size>674570</Size><StorageClass>STANDARD</StorageClass><Owner><ID>ryan</ID><DisplayName>ryan</DisplayName></Owner></Contents><Contents><Key>user/hduser/gutenberg/5000.txt.utf-8</Key><LastModified>2014-01-03T13:26:47.000Z</LastModified><ETag>NOTRETURNED</ETag><Size>1423803</Size><StorageClass>STANDARD</StorageClass><Owner><ID>ryan</ID><DisplayName>ryan</DisplayName></Owner></Contents><Contents><Key>user/hduser/gutenberg/4300.txt.utf-8</Key><LastModified>2014-01-03T13:26:47.000Z</LastModified><ETag>NOTRETURNED</ETag><Size>1573150</Size><StorageClass>STANDARD</StorageClass><Owner><ID>ryan</ID><DisplayName>ryan</DisplayName></Owner></Contents></ListBucketResult>";
var expected = new {
Name = "remoteTest16",
Prefix = "",
Marker = "",
MaxKeys = 1000,
IsTruncated = false,
Objects = new[] {
new {
Key = "user/hduser/gutenberg/20417.txt.utf-8",
LastModified = DateTime.Parse("2014-01-03T13:26:47.000Z"),
ETag = "NOTRETURNED",
Size = 674570,
StorageClass = "STANDARD",
Owner = new { ID = "ryan", DisplayName = "ryan" }
},
new {
Key = "user/hduser/gutenberg/5000.txt.utf-8",
LastModified = DateTime.Parse("2014-01-03T13:26:47.000Z"),
ETag = "NOTRETURNED",
Size = 1423803,
StorageClass = "STANDARD",
Owner = new { ID = "ryan", DisplayName = "ryan" }
},
new {
Key = "user/hduser/gutenberg/4300.txt.utf-8",
LastModified = DateTime.Parse("2014-01-03T13:26:47.000Z"),
ETag = "NOTRETURNED",
Size = 1573150,
StorageClass = "STANDARD",
Owner = new { ID = "ryan", DisplayName = "ryan" }
}
}
};
var response = MockNetwork
.Expecting(HttpVerb.GET, "/remoteTest16", _emptyQueryParams, "")
.Returning(HttpStatusCode.OK, xmlResponse, _emptyHeaders)
.AsClient
.GetBucket(new GetBucketRequest("remoteTest16"));
Assert.AreEqual(expected.Name, response.Name);
Assert.AreEqual(expected.Prefix, response.Prefix);
Assert.AreEqual(expected.Marker, response.Marker);
Assert.AreEqual(expected.MaxKeys, response.MaxKeys);
Assert.AreEqual(expected.IsTruncated, response.IsTruncated);
var responseObjects = response.Objects.ToList();
Assert.AreEqual(expected.Objects.Length, responseObjects.Count);
for (var i = 0; i < expected.Objects.Length; i++)
{
Assert.AreEqual(expected.Objects[i].Key, responseObjects[i].Name);
Assert.AreEqual(expected.Objects[i].LastModified, responseObjects[i].LastModified);
Assert.AreEqual(expected.Objects[i].ETag, responseObjects[i].Etag);
Assert.AreEqual(expected.Objects[i].Size, responseObjects[i].Size);
Assert.AreEqual(expected.Objects[i].StorageClass, responseObjects[i].StorageClass);
Assert.AreEqual(expected.Objects[i].Owner.ID, responseObjects[i].Owner.Id);
Assert.AreEqual(expected.Objects[i].Owner.DisplayName, responseObjects[i].Owner.DisplayName);
}
}
