private static ContextRange <T> ItIsContextRange <T>(T context, long offset, long length)
where T : IComparable <T>
{
return(Match.Create(
it => it.Context.Equals(context) && it.Range.Start == offset && it.Range.Length == length,
() => ContextRange.Create(Range.ByLength(offset, length), context)
));
}
public void TestTransferItemComparer()
{
var comparer = new TransferItemComparer();
var client1 = new Mock <IDs3Client>(MockBehavior.Strict).Object;
var client2 = new Mock <IDs3Client>(MockBehavior.Strict).Object;
var blob1 = new Blob(Range.ByLength(10, 11), "foo");
var blob2 = new Blob(Range.ByLength(10, 11), "bar");
var instance1 = new TransferItem(client1, blob1);
Assert.AreEqual(0, comparer.Compare(instance1, instance1));
Assert.AreEqual(0, comparer.Compare(new TransferItem(client1, blob1), new TransferItem(client1, blob1)));
Assert.AreNotEqual(0, comparer.Compare(new TransferItem(client1, blob1), new TransferItem(client2, blob1)));
Assert.AreEqual(1, comparer.Compare(instance1, new TransferItem(client1, blob2)));
}
public void RangeTranslatorCompositionWorks()
{
var range_1_1 = ContextRange.Create(Range.ByLength(0L, 10L), 10L);
var range_2_1 = ContextRange.Create(Range.ByLength(0L, 5L), "foo");
var range_2_2 = ContextRange.Create(Range.ByLength(5L, 5L), "bar");
var range_3_1 = ContextRange.Create(Range.ByLength(0L, 2L), false);
var range_3_2 = ContextRange.Create(Range.ByLength(2L, 3L), false);
var range_3_3 = ContextRange.Create(Range.ByLength(5L, 2L), true);
var range_3_4 = ContextRange.Create(Range.ByLength(7L, 3L), true);
var first = new Mock <IRangeTranslator <long, string> >(MockBehavior.Strict);
first
.Setup(rt => rt.Translate(range_1_1))
.Returns(new[] { range_2_1, range_2_2 });
var second = new Mock <IRangeTranslator <string, bool> >(MockBehavior.Strict);
second
.Setup(rt => rt.Translate(range_2_1))
.Returns(new[] { range_3_1, range_3_2 });
second
.Setup(rt => rt.Translate(range_2_2))
.Returns(new[] { range_3_3, range_3_4 });
var composed = first.Object.ComposedWith(second.Object);
var result = composed.Translate(ContextRange.Create(Range.ByLength(0L, 10L), 10L));
CollectionAssert.AreEqual(
new[]
{
range_3_1,
range_3_2,
range_3_3,
range_3_4,
},
result
);
}
public void PartialReadTransfer()
{
var partialObjects = new[]
{
new Ds3PartialObject(Range.ByLength(0L, 4L), "foo"),
new Ds3PartialObject(Range.ByLength(6L, 10L), "foo"),
new Ds3PartialObject(Range.ByLength(18L, 1L), "foo"),
new Ds3PartialObject(Range.ByLength(10L, 26L), "bar"),
};
var fullObjects = new[] { "hello" };
var initialJobResponse = Stubs.BuildJobResponse(
Stubs.Chunk1(null, false, false),
Stubs.Chunk2(null, false, false),
Stubs.Chunk3(null, false, false)
);
var availableJobResponse = Stubs.BuildJobResponse(
Stubs.Chunk1(Stubs.NodeId2, true, true),
Stubs.Chunk2(Stubs.NodeId2, true, true),
Stubs.Chunk3(Stubs.NodeId1, true, true)
);
var node1Client = new Mock <IDs3Client>(MockBehavior.Strict);
SetupGetObject(node1Client, "hello", 0L, "ABCDefGHIJ", Range.ByLength(0L, 10L));
SetupGetObject(node1Client, "bar", 35L, "z", Range.ByLength(35L, 1L));
var node2Client = new Mock <IDs3Client>(MockBehavior.Strict);
SetupGetObject(node2Client, "bar", 0L, "abcde", Range.ByLength(10L, 5L));
SetupGetObject(node2Client, "foo", 10L, "klmnop!", Range.ByLength(10L, 6L), Range.ByLength(18L, 1L));
SetupGetObject(node2Client, "foo", 0L, "abcdghij", Range.ByLength(0L, 4L), Range.ByLength(6L, 4L));
SetupGetObject(node2Client, "bar", 15L, "fghijklmnopqrstuvwxy", Range.ByLength(15L, 20L));
var clientFactory = new Mock <IDs3ClientFactory>(MockBehavior.Strict);
clientFactory
.Setup(cf => cf.GetClientForNodeId(Stubs.NodeId1))
.Returns(node1Client.Object);
clientFactory
.Setup(cf => cf.GetClientForNodeId(Stubs.NodeId2))
.Returns(node2Client.Object);
var client = new Mock <IDs3Client>(MockBehavior.Strict);
client
.Setup(c => c.BuildFactory(Stubs.Nodes))
.Returns(clientFactory.Object);
client
.Setup(c => c.BulkGet(ItIsBulkGetRequest(
Stubs.BucketName,
ChunkOrdering.None,
fullObjects,
partialObjects
)))
.Returns(initialJobResponse);
client
.Setup(c => c.GetAvailableJobChunks(ItIsGetAvailableJobChunksRequest(Stubs.JobId)))
.Returns(GetAvailableJobChunksResponse.Success(TimeSpan.FromMinutes(1), availableJobResponse));
var job = new Ds3ClientHelpers(client.Object)
.StartPartialReadJob(Stubs.BucketName, fullObjects, partialObjects);
CollectionAssert.AreEquivalent(
partialObjects.Concat(new[] { new Ds3PartialObject(Range.ByLength(0L, 10L), "hello") }),
job.AllItems
);
var dataTransfers = new ConcurrentQueue <long>();
var itemsCompleted = new ConcurrentQueue <Ds3PartialObject>();
job.DataTransferred += dataTransfers.Enqueue;
job.ItemCompleted += itemsCompleted.Enqueue;
var streams = new ConcurrentDictionary <Ds3PartialObject, MockStream>();
job.Transfer(key => streams.GetOrAdd(key, k => new MockStream()));
node1Client.VerifyAll();
node2Client.VerifyAll();
clientFactory.VerifyAll();
client.VerifyAll();
var fullObjectPart = new Ds3PartialObject(Range.ByLength(0L, 10L), fullObjects[0]);
CollectionAssert.AreEqual(
new[]
{
new { Key = partialObjects[0], Value = "abcd" },
new { Key = partialObjects[1], Value = "ghijklmnop" },
new { Key = partialObjects[2], Value = "!" },
new { Key = partialObjects[3], Value = "abcdefghijklmnopqrstuvwxyz" },
new { Key = fullObjectPart, Value = "ABCDefGHIJ" },
}.OrderBy(it => it.Key).ToArray(),
(
from item in streams
orderby item.Key
select new { item.Key, Value = _encoding.GetString(item.Value.Result) }
).ToArray()
);
CollectionAssert.AreEquivalent(
new[] { 1L, 1L, 4L, 4L, 5L, 6L, 10L, 20L },
dataTransfers.Sorted().ToArray()
);
CollectionAssert.AreEquivalent(partialObjects.Concat(new[] { fullObjectPart }), itemsCompleted);
}
public void EnumerateTransfersStreamsNewlyAvailableTransferItems()
{
var jobResponse1 = Stubs.BuildJobResponse(
Stubs.Chunk1(null, false, false),
Stubs.Chunk2(null, false, false),
Stubs.Chunk3(null, false, false)
);
var jobResponse2 = Stubs.BuildJobResponse(
Stubs.Chunk2(Stubs.NodeId2, true, true)
);
var jobResponse3 = Stubs.BuildJobResponse(
Stubs.Chunk2(Stubs.NodeId2, true, true),
Stubs.Chunk1(Stubs.NodeId1, true, true)
);
var jobResponse4 = Stubs.BuildJobResponse(
Stubs.Chunk3(Stubs.NodeId2, true, true)
);
var node1Client = new Mock <IDs3Client>(MockBehavior.Strict).Object;
var node2Client = new Mock <IDs3Client>(MockBehavior.Strict).Object;
var clientFactory = new Mock <IDs3ClientFactory>(MockBehavior.Strict);
clientFactory.Setup(cf => cf.GetClientForNodeId(Stubs.NodeId1)).Returns(node1Client);
clientFactory.Setup(cf => cf.GetClientForNodeId(Stubs.NodeId2)).Returns(node2Client);
var actionSequence = Queue(new { Item = (object)null, Type = "" });
var client = new Mock <IDs3Client>(MockBehavior.Strict);
client.Setup(c => c.BuildFactory(Stubs.Nodes)).Returns(clientFactory.Object);
var chunkResponses = new[]
{
GetAvailableJobChunksResponse.RetryAfter(TimeSpan.FromMinutes(5)),
GetAvailableJobChunksResponse.Success(TimeSpan.FromMinutes(11), jobResponse2),
GetAvailableJobChunksResponse.Success(TimeSpan.FromMinutes(8), jobResponse2),
GetAvailableJobChunksResponse.Success(TimeSpan.FromMinutes(11), jobResponse3),
GetAvailableJobChunksResponse.Success(TimeSpan.FromMinutes(7), jobResponse3),
GetAvailableJobChunksResponse.Success(TimeSpan.FromMinutes(6), jobResponse3),
GetAvailableJobChunksResponse.RetryAfter(TimeSpan.FromMinutes(4)),
GetAvailableJobChunksResponse.Success(TimeSpan.FromMinutes(11), jobResponse4)
};
var chunkResponseQueue = new Queue <GetAvailableJobChunksResponse>(chunkResponses);
client
.Setup(c => c.GetAvailableJobChunks(AvailableChunks(Stubs.JobId)))
.Returns(() =>
{
var r = chunkResponseQueue.Dequeue();
actionSequence.Enqueue(new { Item = (object)r, Type = "Allocated" });
return(r);
});
var sleeps = new List <TimeSpan>();
var source = new ReadTransferItemSource(sleeps.Add, client.Object, jobResponse1);
var blobs = new[]
{
new Blob(Range.ByLength(0, 10), "foo"),
new Blob(Range.ByLength(15, 20), "bar"),
new Blob(Range.ByLength(0, 15), "bar"),
new Blob(Range.ByLength(10, 10), "foo"),
new Blob(Range.ByLength(0, 10), "hello"),
new Blob(Range.ByLength(35, 11), "bar")
};
var producerConsumer = new ProducerConsumer(1);
var completeBlobsTask = Task.Run(() =>
{
for (int i = 0; i < blobs.Length; i++)
{
producerConsumer.Read(() =>
actionSequence.Enqueue(new { Item = (object)blobs[i], Type = "Completed" })
);
source.CompleteBlob(blobs[i]);
}
});
CollectionAssert.AreEqual(
new[]
{
new TransferItem(node2Client, blobs[0]),
new TransferItem(node2Client, blobs[1]),
new TransferItem(node1Client, blobs[2]),
new TransferItem(node1Client, blobs[3]),
new TransferItem(node2Client, blobs[4]),
new TransferItem(node2Client, blobs[5]),
},
source
.EnumerateAvailableTransfers()
.Select(ti =>
{
producerConsumer.Write(() =>
actionSequence.Enqueue(new { Item = (object)ti.Blob, Type = "Returned" })
);
return(ti);
})
.ToArray(),
new TransferItemSourceHelpers.TransferItemComparer()
);
completeBlobsTask.Wait();
var results = actionSequence.ToArray();
CollectionAssert.AreEqual(
new[]
{
new { Item = (object)chunkResponses[0], Type = "Allocated" },
new { Item = (object)chunkResponses[1], Type = "Allocated" },
new { Item = (object)blobs[0], Type = "Returned" },
new { Item = (object)blobs[0], Type = "Completed" },
new { Item = (object)blobs[1], Type = "Returned" },
new { Item = (object)blobs[1], Type = "Completed" },
new { Item = (object)chunkResponses[2], Type = "Allocated" },
new { Item = (object)chunkResponses[3], Type = "Allocated" },
new { Item = (object)blobs[2], Type = "Returned" },
new { Item = (object)blobs[2], Type = "Completed" },
new { Item = (object)blobs[3], Type = "Returned" },
new { Item = (object)blobs[3], Type = "Completed" },
new { Item = (object)chunkResponses[4], Type = "Allocated" },
new { Item = (object)chunkResponses[5], Type = "Allocated" },
new { Item = (object)chunkResponses[6], Type = "Allocated" },
new { Item = (object)chunkResponses[7], Type = "Allocated" },
new { Item = (object)blobs[4], Type = "Returned" },
new { Item = (object)blobs[4], Type = "Completed" },
new { Item = (object)blobs[5], Type = "Returned" },
new { Item = (object)blobs[5], Type = "Completed" },
},
results
);
CollectionAssert.AreEqual(
new[]
{
TimeSpan.FromMinutes(5),
TimeSpan.FromMinutes(8),
TimeSpan.FromMinutes(7),
TimeSpan.FromMinutes(6),
TimeSpan.FromMinutes(4)
},
sleeps
);
clientFactory.VerifyAll();
client.VerifyAll();
}
public void StreamCanReadFromCorrectPlaces()
{
var translator = new Mock <IRangeTranslator <string, string> >(MockBehavior.Strict);
translator
.Setup(t => t.Translate(ItIsContextRange("a", 0L, 10L)))
.Returns(new[] { ContextRange.Create(Range.ByLength(0L, 10L), "lowercase") });
translator
.Setup(t => t.Translate(ItIsContextRange("b", 0L, 10L)))
.Returns(new[] { ContextRange.Create(Range.ByLength(10L, 10L), "lowercase") });
translator
.Setup(t => t.Translate(ItIsContextRange("c", 0L, 3L)))
.Returns(new[] { ContextRange.Create(Range.ByLength(20L, 3L), "lowercase") });
translator
.Setup(t => t.Translate(ItIsContextRange("c", 3L, 7L)))
.Returns(new[]
{
ContextRange.Create(Range.ByLength(23L, 3L), "lowercase"),
ContextRange.Create(Range.ByLength(0L, 4L), "uppercase"),
});
translator
.Setup(t => t.Translate(ItIsContextRange("d", 0L, 10L)))
.Returns(new[] { ContextRange.Create(Range.ByLength(4L, 10L), "uppercase") });
translator
.Setup(t => t.Translate(ItIsContextRange("e", 0L, 10L)))
.Returns(new[] { ContextRange.Create(Range.ByLength(14L, 10L), "uppercase") });
translator
.Setup(t => t.Translate(ItIsContextRange("f", 0L, 6L)))
.Returns(new[]
{
ContextRange.Create(Range.ByLength(24L, 2L), "uppercase"),
ContextRange.Create(Range.ByLength(0L, 4L), "numbers"),
});
translator
.Setup(t => t.Translate(ItIsContextRange("f", 6L, 4L)))
.Returns(new[] { ContextRange.Create(Range.ByLength(4L, 4L), "numbers") });
translator
.Setup(t => t.Translate(ItIsContextRange("g", 0L, 2L)))
.Returns(new[] { ContextRange.Create(Range.ByLength(8L, 2L), "numbers") });
var payloads = new[]
{
Tuple.Create("a", "abcdefghij"),
Tuple.Create("b", "klmnopqrst"),
Tuple.Create("c", "uvw"),
Tuple.Create("c", "xyzABCD"),
Tuple.Create("d", "EFGHIJKLMN"),
Tuple.Create("e", "OPQRSTUVWX"),
Tuple.Create("f", "YZ0123"),
Tuple.Create("f", "4567"),
Tuple.Create("g", "89"),
};
var streams = new Dictionary <string, MockStream>
{
{ "lowercase", new MockStream("abcdefghijklmnopqrstuvwxyz") },
{ "uppercase", new MockStream("ABCDEFGHIJKLMNOPQRSTUVWXYZ") },
{ "numbers", new MockStream("0123456789") },
};
var results = new Queue <Tuple <string, string> >();
using (var resourceStore = new ResourceStore <string, Stream>(key => streams[key]))
{
var buffer = new byte[100];
var rand = new Random();
foreach (var context in payloads.GroupBy(it => it.Item1, it => it.Item2.Length))
{
using (var stream = new StreamTranslator <string, string>(translator.Object, resourceStore, context.Key, context.Sum(it => it)))
{
foreach (var payloadSize in context)
{
var byteIndex = rand.Next(40);
stream.Read(buffer, byteIndex, payloadSize);
results.Enqueue(Tuple.Create(context.Key, _encoding.GetString(buffer, byteIndex, payloadSize)));
}
}
}
}
CollectionAssert.AreEqual(payloads, results.ToArray());
CollectionAssert.AreEquivalent(new[] { 26, 26, 10 }, streams.Values.Select(str => str.Result.Length).ToArray());
}
public void StreamCanWriteToCorrectPlaces()
{
var translator = new Mock <IRangeTranslator <string, string> >(MockBehavior.Strict);
var payloads = new[]
{
Tuple.Create("a", "abcdefghij"),
Tuple.Create("b", "klmnopqrst"),
Tuple.Create("c", "uvw"),
Tuple.Create("c", "xyzABCD"),
Tuple.Create("d", "EFGHIJKLMN"),
Tuple.Create("e", "OPQRSTUVWX"),
Tuple.Create("f", "YZ0123"),
Tuple.Create("f", "4567"),
Tuple.Create("g", "89"),
}.GroupBy(it => it.Item1, it => it.Item2);
translator
.Setup(t => t.Translate(ItIsContextRange("a", 0L, 10L)))
.Returns <ContextRange <string> >(cr => new[] { ContextRange.Create(Range.ByLength(0L, 10L), "lowercase") });
translator
.Setup(t => t.Translate(ItIsContextRange("b", 0L, 10L)))
.Returns <ContextRange <string> >(cr => new[] { ContextRange.Create(Range.ByLength(10L, 10L), "lowercase") });
translator
.Setup(t => t.Translate(ItIsContextRange("c", 0L, 3L)))
.Returns <ContextRange <string> >(cr => new[] { ContextRange.Create(Range.ByLength(20L, 3L), "lowercase") });
translator
.Setup(t => t.Translate(ItIsContextRange("c", 3L, 7L)))
.Returns <ContextRange <string> >(cr => new[]
{
ContextRange.Create(Range.ByLength(23L, 3L), "lowercase"),
ContextRange.Create(Range.ByLength(0L, 4L), "uppercase"),
});
translator
.Setup(t => t.Translate(ItIsContextRange("d", 0L, 10L)))
.Returns <ContextRange <string> >(cr => new[] { ContextRange.Create(Range.ByLength(4L, 10L), "uppercase"), });
translator
.Setup(t => t.Translate(ItIsContextRange("e", 0L, 10L)))
.Returns <ContextRange <string> >(cr => new[] { ContextRange.Create(Range.ByLength(14L, 10L), "uppercase"), });
translator
.Setup(t => t.Translate(ItIsContextRange("f", 0L, 6L)))
.Returns <ContextRange <string> >(cr => new[]
{
ContextRange.Create(Range.ByLength(24L, 2L), "uppercase"),
ContextRange.Create(Range.ByLength(0L, 4L), "numbers"),
});
translator
.Setup(t => t.Translate(ItIsContextRange("f", 6L, 4L)))
.Returns <ContextRange <string> >(cr => new[] { ContextRange.Create(Range.ByLength(4L, 4L), "numbers") });
translator
.Setup(t => t.Translate(ItIsContextRange("g", 0L, 2L)))
.Returns <ContextRange <string> >(cr => new[] { ContextRange.Create(Range.ByLength(8L, 2L), "numbers"), });
var streams = new ConcurrentDictionary <string, MockStream>();
using (var resourceStore = new ResourceStore <string, Stream>(key => streams.GetOrAdd(key, k => new MockStream())))
{
var buffer = new byte[100];
var rand = new Random();
foreach (var context in payloads)
{
using (var stream = new StreamTranslator <string, string>(translator.Object, resourceStore, context.Key, context.Sum(it => it.Length)))
{
foreach (var payload in context)
{
var byteIndex = rand.Next(40);
_encoding.GetBytes(payload, 0, payload.Length, buffer, byteIndex);
stream.Write(buffer, byteIndex, payload.Length);
}
}
}
}
CollectionAssert.AreEquivalent(
new[]
{
new { Key = "lowercase", Result = "abcdefghijklmnopqrstuvwxyz" },
new { Key = "uppercase", Result = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" },
new { Key = "numbers", Result = "0123456789" },
},
streams.Select(kvp => new { kvp.Key, Result = _encoding.GetString(kvp.Value.Result) })
);
}
public Ds3PartialObject(Range range, string objectName)
: base(range, objectName)
{
}
