private static ContextRange <T> ItIsContextRange <T>(T context, long offset, long length) where T : IComparable <T> { return(Match.Create( it => it.Context.Equals(context) && it.Range.Start == offset && it.Range.Length == length, () => ContextRange.Create(Range.ByLength(offset, length), context) )); }
public void TestTransferItemComparer() { var comparer = new TransferItemComparer(); var client1 = new Mock <IDs3Client>(MockBehavior.Strict).Object; var client2 = new Mock <IDs3Client>(MockBehavior.Strict).Object; var blob1 = new Blob(Range.ByLength(10, 11), "foo"); var blob2 = new Blob(Range.ByLength(10, 11), "bar"); var instance1 = new TransferItem(client1, blob1); Assert.AreEqual(0, comparer.Compare(instance1, instance1)); Assert.AreEqual(0, comparer.Compare(new TransferItem(client1, blob1), new TransferItem(client1, blob1))); Assert.AreNotEqual(0, comparer.Compare(new TransferItem(client1, blob1), new TransferItem(client2, blob1))); Assert.AreEqual(1, comparer.Compare(instance1, new TransferItem(client1, blob2))); }
public void RangeTranslatorCompositionWorks() { var range_1_1 = ContextRange.Create(Range.ByLength(0L, 10L), 10L); var range_2_1 = ContextRange.Create(Range.ByLength(0L, 5L), "foo"); var range_2_2 = ContextRange.Create(Range.ByLength(5L, 5L), "bar"); var range_3_1 = ContextRange.Create(Range.ByLength(0L, 2L), false); var range_3_2 = ContextRange.Create(Range.ByLength(2L, 3L), false); var range_3_3 = ContextRange.Create(Range.ByLength(5L, 2L), true); var range_3_4 = ContextRange.Create(Range.ByLength(7L, 3L), true); var first = new Mock <IRangeTranslator <long, string> >(MockBehavior.Strict); first .Setup(rt => rt.Translate(range_1_1)) .Returns(new[] { range_2_1, range_2_2 }); var second = new Mock <IRangeTranslator <string, bool> >(MockBehavior.Strict); second .Setup(rt => rt.Translate(range_2_1)) .Returns(new[] { range_3_1, range_3_2 }); second .Setup(rt => rt.Translate(range_2_2)) .Returns(new[] { range_3_3, range_3_4 }); var composed = first.Object.ComposedWith(second.Object); var result = composed.Translate(ContextRange.Create(Range.ByLength(0L, 10L), 10L)); CollectionAssert.AreEqual( new[] { range_3_1, range_3_2, range_3_3, range_3_4, }, result ); }
public void PartialReadTransfer() { var partialObjects = new[] { new Ds3PartialObject(Range.ByLength(0L, 4L), "foo"), new Ds3PartialObject(Range.ByLength(6L, 10L), "foo"), new Ds3PartialObject(Range.ByLength(18L, 1L), "foo"), new Ds3PartialObject(Range.ByLength(10L, 26L), "bar"), }; var fullObjects = new[] { "hello" }; var initialJobResponse = Stubs.BuildJobResponse( Stubs.Chunk1(null, false, false), Stubs.Chunk2(null, false, false), Stubs.Chunk3(null, false, false) ); var availableJobResponse = Stubs.BuildJobResponse( Stubs.Chunk1(Stubs.NodeId2, true, true), Stubs.Chunk2(Stubs.NodeId2, true, true), Stubs.Chunk3(Stubs.NodeId1, true, true) ); var node1Client = new Mock <IDs3Client>(MockBehavior.Strict); SetupGetObject(node1Client, "hello", 0L, "ABCDefGHIJ", Range.ByLength(0L, 10L)); SetupGetObject(node1Client, "bar", 35L, "z", Range.ByLength(35L, 1L)); var node2Client = new Mock <IDs3Client>(MockBehavior.Strict); SetupGetObject(node2Client, "bar", 0L, "abcde", Range.ByLength(10L, 5L)); SetupGetObject(node2Client, "foo", 10L, "klmnop!", Range.ByLength(10L, 6L), Range.ByLength(18L, 1L)); SetupGetObject(node2Client, "foo", 0L, "abcdghij", Range.ByLength(0L, 4L), Range.ByLength(6L, 4L)); SetupGetObject(node2Client, "bar", 15L, "fghijklmnopqrstuvwxy", Range.ByLength(15L, 20L)); var clientFactory = new Mock <IDs3ClientFactory>(MockBehavior.Strict); clientFactory .Setup(cf => cf.GetClientForNodeId(Stubs.NodeId1)) .Returns(node1Client.Object); clientFactory .Setup(cf => cf.GetClientForNodeId(Stubs.NodeId2)) .Returns(node2Client.Object); var client = new Mock <IDs3Client>(MockBehavior.Strict); client .Setup(c => c.BuildFactory(Stubs.Nodes)) .Returns(clientFactory.Object); client .Setup(c => c.BulkGet(ItIsBulkGetRequest( Stubs.BucketName, ChunkOrdering.None, fullObjects, partialObjects ))) .Returns(initialJobResponse); client .Setup(c => c.GetAvailableJobChunks(ItIsGetAvailableJobChunksRequest(Stubs.JobId))) .Returns(GetAvailableJobChunksResponse.Success(TimeSpan.FromMinutes(1), availableJobResponse)); var job = new Ds3ClientHelpers(client.Object) .StartPartialReadJob(Stubs.BucketName, fullObjects, partialObjects); CollectionAssert.AreEquivalent( partialObjects.Concat(new[] { new Ds3PartialObject(Range.ByLength(0L, 10L), "hello") }), job.AllItems ); var dataTransfers = new ConcurrentQueue <long>(); var itemsCompleted = new ConcurrentQueue <Ds3PartialObject>(); job.DataTransferred += dataTransfers.Enqueue; job.ItemCompleted += itemsCompleted.Enqueue; var streams = new ConcurrentDictionary <Ds3PartialObject, MockStream>(); job.Transfer(key => streams.GetOrAdd(key, k => new MockStream())); node1Client.VerifyAll(); node2Client.VerifyAll(); clientFactory.VerifyAll(); client.VerifyAll(); var fullObjectPart = new Ds3PartialObject(Range.ByLength(0L, 10L), fullObjects[0]); CollectionAssert.AreEqual( new[] { new { Key = partialObjects[0], Value = "abcd" }, new { Key = partialObjects[1], Value = "ghijklmnop" }, new { Key = partialObjects[2], Value = "!" }, new { Key = partialObjects[3], Value = "abcdefghijklmnopqrstuvwxyz" }, new { Key = fullObjectPart, Value = "ABCDefGHIJ" }, }.OrderBy(it => it.Key).ToArray(), ( from item in streams orderby item.Key select new { item.Key, Value = _encoding.GetString(item.Value.Result) } ).ToArray() ); CollectionAssert.AreEquivalent( new[] { 1L, 1L, 4L, 4L, 5L, 6L, 10L, 20L }, dataTransfers.Sorted().ToArray() ); CollectionAssert.AreEquivalent(partialObjects.Concat(new[] { fullObjectPart }), itemsCompleted); }
public void EnumerateTransfersStreamsNewlyAvailableTransferItems() { var jobResponse1 = Stubs.BuildJobResponse( Stubs.Chunk1(null, false, false), Stubs.Chunk2(null, false, false), Stubs.Chunk3(null, false, false) ); var jobResponse2 = Stubs.BuildJobResponse( Stubs.Chunk2(Stubs.NodeId2, true, true) ); var jobResponse3 = Stubs.BuildJobResponse( Stubs.Chunk2(Stubs.NodeId2, true, true), Stubs.Chunk1(Stubs.NodeId1, true, true) ); var jobResponse4 = Stubs.BuildJobResponse( Stubs.Chunk3(Stubs.NodeId2, true, true) ); var node1Client = new Mock <IDs3Client>(MockBehavior.Strict).Object; var node2Client = new Mock <IDs3Client>(MockBehavior.Strict).Object; var clientFactory = new Mock <IDs3ClientFactory>(MockBehavior.Strict); clientFactory.Setup(cf => cf.GetClientForNodeId(Stubs.NodeId1)).Returns(node1Client); clientFactory.Setup(cf => cf.GetClientForNodeId(Stubs.NodeId2)).Returns(node2Client); var actionSequence = Queue(new { Item = (object)null, Type = "" }); var client = new Mock <IDs3Client>(MockBehavior.Strict); client.Setup(c => c.BuildFactory(Stubs.Nodes)).Returns(clientFactory.Object); var chunkResponses = new[] { GetAvailableJobChunksResponse.RetryAfter(TimeSpan.FromMinutes(5)), GetAvailableJobChunksResponse.Success(TimeSpan.FromMinutes(11), jobResponse2), GetAvailableJobChunksResponse.Success(TimeSpan.FromMinutes(8), jobResponse2), GetAvailableJobChunksResponse.Success(TimeSpan.FromMinutes(11), jobResponse3), GetAvailableJobChunksResponse.Success(TimeSpan.FromMinutes(7), jobResponse3), GetAvailableJobChunksResponse.Success(TimeSpan.FromMinutes(6), jobResponse3), GetAvailableJobChunksResponse.RetryAfter(TimeSpan.FromMinutes(4)), GetAvailableJobChunksResponse.Success(TimeSpan.FromMinutes(11), jobResponse4) }; var chunkResponseQueue = new Queue <GetAvailableJobChunksResponse>(chunkResponses); client .Setup(c => c.GetAvailableJobChunks(AvailableChunks(Stubs.JobId))) .Returns(() => { var r = chunkResponseQueue.Dequeue(); actionSequence.Enqueue(new { Item = (object)r, Type = "Allocated" }); return(r); }); var sleeps = new List <TimeSpan>(); var source = new ReadTransferItemSource(sleeps.Add, client.Object, jobResponse1); var blobs = new[] { new Blob(Range.ByLength(0, 10), "foo"), new Blob(Range.ByLength(15, 20), "bar"), new Blob(Range.ByLength(0, 15), "bar"), new Blob(Range.ByLength(10, 10), "foo"), new Blob(Range.ByLength(0, 10), "hello"), new Blob(Range.ByLength(35, 11), "bar") }; var producerConsumer = new ProducerConsumer(1); var completeBlobsTask = Task.Run(() => { for (int i = 0; i < blobs.Length; i++) { producerConsumer.Read(() => actionSequence.Enqueue(new { Item = (object)blobs[i], Type = "Completed" }) ); source.CompleteBlob(blobs[i]); } }); CollectionAssert.AreEqual( new[] { new TransferItem(node2Client, blobs[0]), new TransferItem(node2Client, blobs[1]), new TransferItem(node1Client, blobs[2]), new TransferItem(node1Client, blobs[3]), new TransferItem(node2Client, blobs[4]), new TransferItem(node2Client, blobs[5]), }, source .EnumerateAvailableTransfers() .Select(ti => { producerConsumer.Write(() => actionSequence.Enqueue(new { Item = (object)ti.Blob, Type = "Returned" }) ); return(ti); }) .ToArray(), new TransferItemSourceHelpers.TransferItemComparer() ); completeBlobsTask.Wait(); var results = actionSequence.ToArray(); CollectionAssert.AreEqual( new[] { new { Item = (object)chunkResponses[0], Type = "Allocated" }, new { Item = (object)chunkResponses[1], Type = "Allocated" }, new { Item = (object)blobs[0], Type = "Returned" }, new { Item = (object)blobs[0], Type = "Completed" }, new { Item = (object)blobs[1], Type = "Returned" }, new { Item = (object)blobs[1], Type = "Completed" }, new { Item = (object)chunkResponses[2], Type = "Allocated" }, new { Item = (object)chunkResponses[3], Type = "Allocated" }, new { Item = (object)blobs[2], Type = "Returned" }, new { Item = (object)blobs[2], Type = "Completed" }, new { Item = (object)blobs[3], Type = "Returned" }, new { Item = (object)blobs[3], Type = "Completed" }, new { Item = (object)chunkResponses[4], Type = "Allocated" }, new { Item = (object)chunkResponses[5], Type = "Allocated" }, new { Item = (object)chunkResponses[6], Type = "Allocated" }, new { Item = (object)chunkResponses[7], Type = "Allocated" }, new { Item = (object)blobs[4], Type = "Returned" }, new { Item = (object)blobs[4], Type = "Completed" }, new { Item = (object)blobs[5], Type = "Returned" }, new { Item = (object)blobs[5], Type = "Completed" }, }, results ); CollectionAssert.AreEqual( new[] { TimeSpan.FromMinutes(5), TimeSpan.FromMinutes(8), TimeSpan.FromMinutes(7), TimeSpan.FromMinutes(6), TimeSpan.FromMinutes(4) }, sleeps ); clientFactory.VerifyAll(); client.VerifyAll(); }
public void StreamCanReadFromCorrectPlaces() { var translator = new Mock <IRangeTranslator <string, string> >(MockBehavior.Strict); translator .Setup(t => t.Translate(ItIsContextRange("a", 0L, 10L))) .Returns(new[] { ContextRange.Create(Range.ByLength(0L, 10L), "lowercase") }); translator .Setup(t => t.Translate(ItIsContextRange("b", 0L, 10L))) .Returns(new[] { ContextRange.Create(Range.ByLength(10L, 10L), "lowercase") }); translator .Setup(t => t.Translate(ItIsContextRange("c", 0L, 3L))) .Returns(new[] { ContextRange.Create(Range.ByLength(20L, 3L), "lowercase") }); translator .Setup(t => t.Translate(ItIsContextRange("c", 3L, 7L))) .Returns(new[] { ContextRange.Create(Range.ByLength(23L, 3L), "lowercase"), ContextRange.Create(Range.ByLength(0L, 4L), "uppercase"), }); translator .Setup(t => t.Translate(ItIsContextRange("d", 0L, 10L))) .Returns(new[] { ContextRange.Create(Range.ByLength(4L, 10L), "uppercase") }); translator .Setup(t => t.Translate(ItIsContextRange("e", 0L, 10L))) .Returns(new[] { ContextRange.Create(Range.ByLength(14L, 10L), "uppercase") }); translator .Setup(t => t.Translate(ItIsContextRange("f", 0L, 6L))) .Returns(new[] { ContextRange.Create(Range.ByLength(24L, 2L), "uppercase"), ContextRange.Create(Range.ByLength(0L, 4L), "numbers"), }); translator .Setup(t => t.Translate(ItIsContextRange("f", 6L, 4L))) .Returns(new[] { ContextRange.Create(Range.ByLength(4L, 4L), "numbers") }); translator .Setup(t => t.Translate(ItIsContextRange("g", 0L, 2L))) .Returns(new[] { ContextRange.Create(Range.ByLength(8L, 2L), "numbers") }); var payloads = new[] { Tuple.Create("a", "abcdefghij"), Tuple.Create("b", "klmnopqrst"), Tuple.Create("c", "uvw"), Tuple.Create("c", "xyzABCD"), Tuple.Create("d", "EFGHIJKLMN"), Tuple.Create("e", "OPQRSTUVWX"), Tuple.Create("f", "YZ0123"), Tuple.Create("f", "4567"), Tuple.Create("g", "89"), }; var streams = new Dictionary <string, MockStream> { { "lowercase", new MockStream("abcdefghijklmnopqrstuvwxyz") }, { "uppercase", new MockStream("ABCDEFGHIJKLMNOPQRSTUVWXYZ") }, { "numbers", new MockStream("0123456789") }, }; var results = new Queue <Tuple <string, string> >(); using (var resourceStore = new ResourceStore <string, Stream>(key => streams[key])) { var buffer = new byte[100]; var rand = new Random(); foreach (var context in payloads.GroupBy(it => it.Item1, it => it.Item2.Length)) { using (var stream = new StreamTranslator <string, string>(translator.Object, resourceStore, context.Key, context.Sum(it => it))) { foreach (var payloadSize in context) { var byteIndex = rand.Next(40); stream.Read(buffer, byteIndex, payloadSize); results.Enqueue(Tuple.Create(context.Key, _encoding.GetString(buffer, byteIndex, payloadSize))); } } } } CollectionAssert.AreEqual(payloads, results.ToArray()); CollectionAssert.AreEquivalent(new[] { 26, 26, 10 }, streams.Values.Select(str => str.Result.Length).ToArray()); }
public void StreamCanWriteToCorrectPlaces() { var translator = new Mock <IRangeTranslator <string, string> >(MockBehavior.Strict); var payloads = new[] { Tuple.Create("a", "abcdefghij"), Tuple.Create("b", "klmnopqrst"), Tuple.Create("c", "uvw"), Tuple.Create("c", "xyzABCD"), Tuple.Create("d", "EFGHIJKLMN"), Tuple.Create("e", "OPQRSTUVWX"), Tuple.Create("f", "YZ0123"), Tuple.Create("f", "4567"), Tuple.Create("g", "89"), }.GroupBy(it => it.Item1, it => it.Item2); translator .Setup(t => t.Translate(ItIsContextRange("a", 0L, 10L))) .Returns <ContextRange <string> >(cr => new[] { ContextRange.Create(Range.ByLength(0L, 10L), "lowercase") }); translator .Setup(t => t.Translate(ItIsContextRange("b", 0L, 10L))) .Returns <ContextRange <string> >(cr => new[] { ContextRange.Create(Range.ByLength(10L, 10L), "lowercase") }); translator .Setup(t => t.Translate(ItIsContextRange("c", 0L, 3L))) .Returns <ContextRange <string> >(cr => new[] { ContextRange.Create(Range.ByLength(20L, 3L), "lowercase") }); translator .Setup(t => t.Translate(ItIsContextRange("c", 3L, 7L))) .Returns <ContextRange <string> >(cr => new[] { ContextRange.Create(Range.ByLength(23L, 3L), "lowercase"), ContextRange.Create(Range.ByLength(0L, 4L), "uppercase"), }); translator .Setup(t => t.Translate(ItIsContextRange("d", 0L, 10L))) .Returns <ContextRange <string> >(cr => new[] { ContextRange.Create(Range.ByLength(4L, 10L), "uppercase"), }); translator .Setup(t => t.Translate(ItIsContextRange("e", 0L, 10L))) .Returns <ContextRange <string> >(cr => new[] { ContextRange.Create(Range.ByLength(14L, 10L), "uppercase"), }); translator .Setup(t => t.Translate(ItIsContextRange("f", 0L, 6L))) .Returns <ContextRange <string> >(cr => new[] { ContextRange.Create(Range.ByLength(24L, 2L), "uppercase"), ContextRange.Create(Range.ByLength(0L, 4L), "numbers"), }); translator .Setup(t => t.Translate(ItIsContextRange("f", 6L, 4L))) .Returns <ContextRange <string> >(cr => new[] { ContextRange.Create(Range.ByLength(4L, 4L), "numbers") }); translator .Setup(t => t.Translate(ItIsContextRange("g", 0L, 2L))) .Returns <ContextRange <string> >(cr => new[] { ContextRange.Create(Range.ByLength(8L, 2L), "numbers"), }); var streams = new ConcurrentDictionary <string, MockStream>(); using (var resourceStore = new ResourceStore <string, Stream>(key => streams.GetOrAdd(key, k => new MockStream()))) { var buffer = new byte[100]; var rand = new Random(); foreach (var context in payloads) { using (var stream = new StreamTranslator <string, string>(translator.Object, resourceStore, context.Key, context.Sum(it => it.Length))) { foreach (var payload in context) { var byteIndex = rand.Next(40); _encoding.GetBytes(payload, 0, payload.Length, buffer, byteIndex); stream.Write(buffer, byteIndex, payload.Length); } } } } CollectionAssert.AreEquivalent( new[] { new { Key = "lowercase", Result = "abcdefghijklmnopqrstuvwxyz" }, new { Key = "uppercase", Result = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" }, new { Key = "numbers", Result = "0123456789" }, }, streams.Select(kvp => new { kvp.Key, Result = _encoding.GetString(kvp.Value.Result) }) ); }