public async Task SplitJoinWithAllBlockTypes_CorrectlyBuild() { // arrange var result = new ConcurrentBag <string> (); var process = new ConcurrentBag <char> (); var process2 = new ConcurrentBag <int> (); var sut = DataflowFluent .ReceiveDataOfType <string> () .SplitTo <char> (s => s.ToCharArray()) .SplitProcess((s, c) => process.Add(c)) .SplitTransform((s, c) => (int)c) .SplitProcess((s, n) => process2.Add(n)) .SplitTransform((s, n) => (char)n) .SplitJoinInto(x => new string (x.SuccessfullyCompletedItems.ToArray())) .Action(result.Add); // act var dataflow = sut.CreateDataflow(); await dataflow.ProcessAsync(new[] { "a", "ab", "abc" }); // assert process.Should().BeEquivalentTo('a', 'a', 'b', 'a', 'b', 'c'); result.Should().BeEquivalentTo("a", "ab", "abc"); }
public async Task WithItemImplementingLogger_OneItemThrows_PassTheExceptionToContext() { // arrange var sut = DataflowFluent .ReceiveDataOfType <TestDataflowContext <string> > () .Action(x => { if (x.Data == "b") { throw new TestException(); } }) .WithMaxDegreeOfParallelism(); var contexts = new[] { "a", "b", "c" }.CreateDataflowContexts(); // act var dataflow = sut.CreateDataflow(); await dataflow.ProcessAsync(contexts); // assert contexts.SelectMany(x => x.Exceptions).Should().HaveCount(1).And.ContainItemsAssignableTo <TestException> (); }
public async Task Dataflow_CorrectlyRecordsTheSessionAsync() { // arrange var profiler_results_storage = new TestProfilerResultsStorage(); ProfilingLibrary.Setup(() => null); ProfilingLibrary.Container.RegisterSingleton <IProfilerLogger, TestsProfilerLogger>(); ProfilingLibrary.Container.RegisterInstance <IProfilerResultsStorage>(profiler_results_storage); ProfilingLibrary.Container.RegisterSingleton <IProfilerConfiguration, TestProfilerConfiguration>(); var exceptions = new List <Exception>(); var dataflow = DataflowFluent .ReceiveDataOfType <int>() .TransformAsync(async id => { var item = new DataflowItemContext { Id = id, ProfileSession = ProfilingLibrary.StartProfiling() }; using (ProfilingLibrary.Profile(item.ProfileSession, new ProfileOperationSpecification("a"))) { await Task.Delay(50).ConfigureAwait(true); } return(item); }) .ProcessAsync(async item => { using (ProfilingLibrary.Profile(item.ProfileSession, new ProfileOperationSpecification("b"))) { await Task.Delay(50).ConfigureAwait(true); } }) .ActionAsync(item => { ProfilingLibrary.StopProfiling(item.ProfileSession); return(Task.CompletedTask); }) .WithDefaultExceptionLogger((ex, obj) => exceptions.Add(ex)) .CreateDataflow(); // act await dataflow.ProcessAsync(Enumerable.Range(0, 10)).ConfigureAwait(false); // assert exceptions.Should().BeEmpty(); profiler_results_storage.ProfileSessions.ToArray() .SelectMany(x => x.Operations) .Select(x => x.Name) .GroupBy(x => x) .Select(x => new { Name = x.Key, Count = x.Count() }) .Should().BeEquivalentTo(new { Name = "a", Count = 10 }, new { Name = "b", Count = 10 }); }
public async Task TransformTransformAction_CorrectlyBuild() { // arrange var result = new ConcurrentBag <int>(); var sut = DataflowFluent .ReceiveDataOfType <int>() .TransformAsync(async x => { await Task.Yield(); return(x.ToString(CultureInfo.InvariantCulture)); }) .TransformAsync(async s => { await Task.Yield(); return(int.Parse(s)); }) .WithBoundedCapacity(100) .ActionAsync(async x => { await Task.Yield(); result.Add(x); }) .WithMaxDegreeOfParallelism(); // act var dataflow = sut.CreateDataflow(); await dataflow.ProcessAsync(new[] { 1, 2, 3 }).ConfigureAwait(false); // assert result.Should().BeEquivalentTo(1, 2, 3); }
public async Task WithDefaultExceptionLogger_OneItemThrows_LogsTheException() { // arrange var exceptions = new ConcurrentBag <Exception> (); var failed_items = new ConcurrentBag <object> (); var sut = DataflowFluent .ReceiveDataOfType <string> () .Transform(x => { if (x == "b") { throw new TestException(); } return(x); }) .Action(x => { }) .WithMaxDegreeOfParallelism() .WithDefaultExceptionLogger((exception, item) => { exceptions.Add(exception); failed_items.Add(item); }) .CreateDataflow(); // act await sut.ProcessAsync(new[] { "a", "b", "c" }); // assert exceptions.Should().HaveCount(1).And.ContainItemsAssignableTo <TestException> (); failed_items.Should().Equal("b"); }
public async Task TransformManyAction_CorrectlyBuild() { // arrange var result = new ConcurrentBag <string>(); var sut = DataflowFluent .ReceiveDataOfType <TestDataflowContext <int> >() .TransformManyAsync <TestDataflowContext <string> >(async x => { await Task.Yield(); return(new[] { new TestDataflowContext <string> { Data = x.ToString() } }); }) .WithBoundedCapacity(100) .ActionAsync(async x => { await Task.Yield(); result.Add(x.Data); }) .WithMaxDegreeOfParallelism(); // act var dataflow = sut.CreateDataflow(); await dataflow.ProcessAsync(new[] { 1, 2, 3 }.CreateDataflowContexts()).ConfigureAwait(false); // assert result.Should().BeEquivalentTo("1", "2", "3"); }
public async Task TransformSplitJoin_CorrectlyBuild() { // arrange var process = new ConcurrentBag <string> (); var process2 = new ConcurrentBag <string> (); var sut = DataflowFluent .ReceiveDataOfType <string> () .Transform(s => { process.Add(s); return(s); }) .SplitTo(s => { process2.Add(s); return(s.ToCharArray()); }) .SplitJoin(); // act var dataflow = sut.CreateDataflow(); await dataflow.ProcessAsync(new[] { "a", "ab", "abc" }); // assert process.Should().BeEquivalentTo("a", "ab", "abc"); process.Should().BeEquivalentTo(process2); }
public async Task TransformActionSync_WithNullReturn_CorrectlyExecuted() { // arrange var result = new ConcurrentBag <int>(); var sut = DataflowFluent .ReceiveDataOfType <TestDataflowContext <int> >() .TransformAsync(async x => { await Task.Yield(); if (x.Data == 2) { return(null); } return(x); }) .WithBoundedCapacity(100) .Action(x => result.Add(x.Data)) .WithMaxDegreeOfParallelism(); // act var dataflow = sut.CreateDataflow(); await dataflow.ProcessAsync(new[] { 1, 2, 3 }.CreateDataflowContexts()).ConfigureAwait(false); // assert result.Should().BeEquivalentTo(1, 3); }
public async Task SplitJoinInto_CorrectlyBuild() { // arrange var result = new ConcurrentBag <string> (); var sut = DataflowFluent .ReceiveDataOfType <string> () .SplitToAsync <char> (async s => { await Task.Yield(); return(s.ToCharArray()); }) .SplitJoinIntoAsync(async x => { await Task.Yield(); return(new string (x.SuccessfullyCompletedItems.ToArray())); }) .ActionAsync(async x => { await Task.Yield(); result.Add(x); }); // act var dataflow = sut.CreateDataflow(); await dataflow.ProcessAsync(new[] { "a", "ab", "abc" }); // assert result.Should().BeEquivalentTo("a", "ab", "abc"); }
public async Task SplitProcessJoin_CorrectlyBuild() { // arrange var process = new ConcurrentBag <char> (); var sut = DataflowFluent .ReceiveDataOfType <string> () .SplitToAsync <char> (async s => { await Task.Yield(); return(s.ToCharArray()); }) .SplitProcessAsync(async(s, c) => { await Task.Yield(); process.Add(c); }) .SplitJoin(); // act var dataflow = sut.CreateDataflow(); await dataflow.ProcessAsync(new[] { "a", "ab", "abc" }); // assert process.Should().BeEquivalentTo('a', 'a', 'b', 'a', 'b', 'c'); }
public async Task ProcessProcessAction_CorrectlyBuild() { // arrange var result = new ConcurrentBag <int>(); var sut = DataflowFluent .ReceiveDataOfType <int>() .Process(x => { }) .Process(x => { }) .WithBoundedCapacity(100) .Action(result.Add) .WithMaxDegreeOfParallelism(); // act var dataflow = sut.CreateDataflow(); await dataflow.ProcessAsync(new[] { 1, 2, 3 }); // assert result.Should().BeEquivalentTo(1, 2, 3); }
public async Task OneItemThrows_ProceedTheRest() { // arrange var result = new ConcurrentBag <string> (); var sut = DataflowFluent .ReceiveDataOfType <string> () .Transform(x => { if (x == "b") { throw new TestException(); } return(x); }) .Action(result.Add) .WithMaxDegreeOfParallelism() .CreateDataflow(); // act await sut.ProcessAsync(new[] { "a", "b", "c" }); // assert result.Should().BeEquivalentTo("a", "c"); }
public async Task SplitJoinWithAllBlockTypes_CorrectlyBuild() { // arrange var result = new ConcurrentBag <string>(); var process = new ConcurrentBag <char>(); var process2 = new ConcurrentBag <int>(); var sut = DataflowFluent .ReceiveDataOfType <string>() .SplitToAsync <char>(async s => { await Task.Yield(); return(s.ToCharArray()); }) .SplitProcessAsync(async(s, c) => { await Task.Yield(); process.Add(c); }) .SplitTransformAsync(async(s, c) => { await Task.Yield(); return((int)c); }) .SplitProcessAsync(async(s, n) => { await Task.Yield(); process2.Add(n); }) .SplitTransformAsync(async(s, n) => { await Task.Yield(); return((char)n); }) .SplitJoinIntoAsync(async x => { await Task.Yield(); return(new string(x.SuccessfullyCompletedItems.ToArray())); }) .ActionAsync(async x => { await Task.Yield(); result.Add(x); }); // act var dataflow = sut.CreateDataflow(); await dataflow.ProcessAsync(new[] { "a", "ab", "abc" }); // assert process.Should().BeEquivalentTo('a', 'a', 'b', 'a', 'b', 'c'); result.Should() .HaveCount(3) .And .OnlyContain(x => !x.Except(new[] { 'a', 'b', 'c' }).Any() && (x.Length == 1 || x.Length == 2 || x.Length == 3)); }
private Dataflow <MessageData> CreateDeleteDataflow() => DataflowFluent .ReceiveDataOfType <MessageData>() .ProcessAsync(data => this._dataAccess.Delete(data.Id, CancellationToken.None)) .WithMaxDegreeOfParallelism(this._config.WriteParallelsCount) .WithDefaultExceptionLogger((exception, o) => Console.WriteLine(exception)) .Action(x => { }) .CreateDataflow();
public async Task SplitTransformJoinIntoAction_WithDefaultExceptionLogger_OneItemsThrows_LogsTheException() { // arrange var exceptions = new ConcurrentBag <Exception> (); var failed_items = new ConcurrentBag <object> (); var sut = DataflowFluent .ReceiveDataOfType <string> () .SplitToAsync <char> (async x => { await Task.Yield(); return(x.ToCharArray()); }) .SplitTransformAsync(async(s, c) => { await Task.Yield(); return(c); }) .SplitJoinIntoAsync(async x => { await Task.Yield(); if (x.Parent == "b") { throw new TestException(); } return(x); }) .ActionAsync(async x => { await Task.Yield(); }) .WithDefaultExceptionLogger((exception, item) => { exceptions.Add(exception); failed_items.Add(item); }) .CreateDataflow(); // act await sut.ProcessAsync(new[] { "a", "b", "c" }); // assert exceptions.Should().HaveCount(1).And.ContainItemsAssignableTo <TestException> (); failed_items.Should().BeEquivalentTo (new[] { new SplitJoinResult <string, char> ("b", new[] { 'b' }, new SplitJoinFailedItem <char> [0], 1) }, options => options.Using <Exception> (x => x.Subject.Should().BeOfType(x.Expectation.GetType())) .WhenTypeIs <Exception> ()); }
public async Task Split_SplitTransform_Join_Action_WithFailedItems_CorrectlyBuild() { // arrange var result = new ConcurrentBag <string> (); var exceptions = new List <Exception> (); var sut = DataflowFluent .ReceiveDataOfType <string> () .SplitToAsync <char> (async s => { await Task.Yield(); return(s.ToCharArray()); }) .SplitTransformAsync(async(s, c) => { await Task.Yield(); return((int)c); }) .SplitTransformAsync(async(s, i) => { await Task.Yield(); var c = (char)i; if (c == 'b') { throw new TestException(); } return(c); }) .SplitJoinIntoAsync(async x => { await Task.Yield(); exceptions.AddRange(x.FailedItems.Select(f => f.Exception)); return(new string (x.SuccessfullyCompletedItems.ToArray())); }) .ActionAsync(async x => { await Task.Yield(); result.Add(x); }); // act var dataflow = sut.CreateDataflow(); await dataflow.ProcessAsync(new[] { "a", "ab", "abc" }); // assert result.Should().BeEquivalentTo("a", "a", "ac"); exceptions.Should().ContainItemsAssignableTo <TestException> (); }
public async Task SplitProcessJoin_WithException_PassTheExceptionToChildContext() { // arrange var failed_items_exceptions = new List <Exception> (); var split_items_exceptions = new List <Exception> (); var result = new List <char> (); var sut = DataflowFluent .ReceiveDataOfType <TestDataflowContext <string> > () .SplitToAsync <TestDataflowContext <char> > (async context => { await Task.Yield(); return(context.Data.CreateDataflowContexts()); }) .SplitProcessAsync(async(s, c) => { await Task.Yield(); if (c.Data == 'b') { throw new TestException(); } }) .SplitJoinAsync(async splitJoinResult => { await Task.Yield(); failed_items_exceptions.AddRange(splitJoinResult.FailedItems.Select(x => x.Exception)); split_items_exceptions.AddRange(splitJoinResult.FailedItems.SelectMany(x => x.Item.Exceptions)); result.AddRange(splitJoinResult.SuccessfullyCompletedItems.Select(x => x.Data)); }); var contexts = new[] { "a", "b", "c" }.CreateDataflowContexts(); // act var dataflow = sut.CreateDataflow(); await dataflow.ProcessAsync(contexts); // assert result.Should().BeEquivalentTo('a', 'c'); contexts.SelectMany(x => x.Exceptions).Should().BeEmpty(); failed_items_exceptions.Should().HaveCount(1); failed_items_exceptions.Should().ContainItemsAssignableTo <TestException> (); split_items_exceptions.Should().HaveCount(1); split_items_exceptions.Should().ContainItemsAssignableTo <TestException> (); }
public void SplitProcessJoinAction_TwoItems_SecondWaitsForTheFirst_DataflowEndedOnlyAfterBothProcessed() { // arrange var first_item_finished = new SemaphoreSlim(0, 1); var completed_items = new ConcurrentBag <string> (); var sut = DataflowFluent .ReceiveDataOfType <string> () .SplitToAsync <char> (async x => { await Task.Yield(); return(x.ToCharArray()); }) .SplitProcessAsync(async(s, c) => { await Task.Yield(); if (c == '1') { await Task.Delay(100); } else if (c == '2') { await first_item_finished.WaitAsync(); } }) .SplitJoinInto(result => result.Parent) .ActionAsync(async x => { await Task.Yield(); if (x == "111") { first_item_finished.Release(); } completed_items.Add(x); }) .CreateDataflow(); // act if (!sut.ProcessAsync(new[] { "111", "2" }).Wait(1000)) { throw new TimeoutException(); } // assert completed_items.Should().BeEquivalentTo("111", "2"); }
private Task WriteAll(IEnumerable <MessageData> messages, CancellationToken cancellationToken) { Console.WriteLine($"Start writing... {DateTimeFormatedString}"); var dataflow = DataflowFluent .ReceiveDataOfType <MessageData>() .ProcessAsync(data => this._dataAccess.Add(data, CancellationToken.None)) .WithMaxDegreeOfParallelism(this._config.WriteParallelsCount) .WithDefaultExceptionLogger((exception, o) => Console.WriteLine(exception)) .Action(x => { }) .CreateDataflow(); return(dataflow.ProcessAsync(messages, cancellationToken)); }
public void ActionAction_Throws() { // arrange // act Action action = () => DataflowFluent .ReceiveDataOfType <int> () .Action(x => { }) .Action(x => { }) .CreateDataflow(); // assert action.Should().Throw <InvalidOperationException> (); }
public async Task SplitProcess_WithDefaultExceptionLogger_OneItemsThrows_LogsTheException() { // arrange var exceptions = new ConcurrentBag <Exception> (); var failed_items = new ConcurrentBag <object> (); var sut = DataflowFluent .ReceiveDataOfType <string> () .SplitToAsync <char> (async x => { await Task.Yield(); return(x.ToCharArray()); }) .SplitProcessAsync(async(s, x) => { await Task.Yield(); if (x == 'b') { throw new TestException(); } }) .SplitJoin() .WithDefaultExceptionLogger((exception, item) => { exceptions.Add(exception); failed_items.Add(item); }) .CreateDataflow(); // act await sut.ProcessAsync(new[] { "a", "b", "c" }); // assert exceptions.Should().HaveCount(1).And.ContainItemsAssignableTo <TestException> (); var expected_failed_item = new SplitJoinItem <string, char> ("b", 'b', 1); expected_failed_item.Failed(new TestException()); failed_items.Should().BeEquivalentTo (new[] { expected_failed_item }, options => options.Using <Exception> (x => x.Subject.Should().BeOfType <TestException> ()) .WhenTypeIs <Exception> ()); }
public void ProcessAction_FirstWaitsForTheSecond_DataflowEndedOnlyAfterBothProcessed() { // arrange var second_item_finished = new SemaphoreSlim(0, 1); var completed_items = new ConcurrentBag <string>(); var sut = DataflowFluent .ReceiveDataOfType <string>() .ProcessAsync(async x => { await Task.Yield(); this.output.WriteLine("ProcessAsync({0})", x); if (x == "1") { await second_item_finished.WaitAsync(); } }) .WithMaxDegreeOfParallelism(10) .Action(x => { this.output.WriteLine("Action({0})", x); completed_items.Add(x); if (x == "2") { second_item_finished.Release(); } }) .CreateDataflow(); // act if (!sut.ProcessAsync(new[] { "1", "2" }).Wait(2000)) { throw new TimeoutException(); } // assert completed_items.Should().BeEquivalentTo("2", "1"); }
public async Task StartsWithBatch_WithTimeout_CorrectlyBuild() { // arrange var result = new ConcurrentBag <int[]>(); var sut = DataflowFluent .ReceiveDataOfType <int>() .Batch(5, TimeSpan.FromMilliseconds(100)) .WithEnsureOrdered(true) .ActionAsync(async x => { await Task.Yield(); result.Add(x); }) .WithEnsureOrdered(true) .WithMaxDegreeOfParallelism(1); // act var dataflow = sut.CreateDataflow(); dataflow.Start(); foreach (var x in Enumerable.Range(1, 2)) { await dataflow.SendAsync(x); } await Task.Delay(200); foreach (var x in Enumerable.Range(3, 3)) { await dataflow.SendAsync(x); } await dataflow.CompleteAsync(); // assert result.Should().BeEquivalentTo(new[] { 1, 2 }, new[] { 3, 4, 5 }); }
public async Task TransformAction_CorrectlyBuild() { // arrange var result = new ConcurrentBag <string> (); var sut = DataflowFluent .ReceiveDataOfType <int> () .Transform(x => x.ToString(CultureInfo.InvariantCulture)) .WithBoundedCapacity(100) .Action(result.Add) .WithMaxDegreeOfParallelism(); // act var dataflow = sut.CreateDataflow(); await dataflow.ProcessAsync(new[] { 1, 2, 3 }); // assert result.Should().BeEquivalentTo("1", "2", "3"); }
public async Task ContinuesWithBatch_WithoutTimeout_CorrectlyBuild() { // arrange var result = new ConcurrentBag <int[]>(); var sut = DataflowFluent .ReceiveDataOfType <int>() .Transform(x => x) .Batch(3) .ActionAsync(async x => { await Task.Yield(); result.Add(x); }) .WithMaxDegreeOfParallelism(); // act var dataflow = sut.CreateDataflow(); dataflow.Start(); foreach (var x in Enumerable.Range(1, 2)) { await dataflow.SendAsync(x); } await Task.Delay(100); foreach (var x in Enumerable.Range(3, 3)) { await dataflow.SendAsync(x); } await dataflow.CompleteAsync(); // assert result.Should().BeEquivalentTo(new[] { 1, 2, 3 }, new[] { 4, 5 }); }
public void ProcessAction_FirstWaitsForTheSecond_DataflowEndedOnlyAfterBothProcessed() { // arrange var second_item_finished = new SemaphoreSlim(0, 1); var completed_items = new ConcurrentBag <string>(); var sut = DataflowFluent .ReceiveDataOfType <string>() .ProcessAsync(async x => { await Task.Yield(); if (x == "1") { await second_item_finished.WaitAsync().ConfigureAwait(false); } }) .Action(x => { completed_items.Add(x); if (x == "2") { second_item_finished.Release(); } }) .CreateDataflow(); // act if (!sut.ProcessAsync(new[] { "1", "2" }).Wait(2000)) { throw new TimeoutException(); } // assert completed_items.Should().Equal("2", "1"); }
private async Task ProcessOperation(Func <IReadOnlyList <MarginTradingAccountHistoryEntity>, Task> operation) { var errorTcs = new TaskCompletionSource <object>(); var dataflow = DataflowFluent .ReceiveDataOfType <IReadOnlyList <MarginTradingAccountHistoryEntity> >() .TransformMany(batch => batch.GroupBy(p => p.PartitionKey, (k, gr) => gr.ToList())) .ProcessAsync(operation) .WithMaxDegreeOfParallelism(10) .Action(batch => { var oldCounterValue = _counter; var currCounterValue = _counter += batch.Count; if (currCounterValue / 1000 != oldCounterValue / 1000) { Log($"Completed: {currCounterValue}; elapsed: {_clock.Elapsed}, speed: {currCounterValue / _clock.Elapsed.TotalMinutes:f2}/min"); } }) .WithMaxDegreeOfParallelism(1) .WithDefaultExceptionLogger((ex, obj) => { Error(ex); errorTcs.TrySetException(ex); throw ex; }) .CreateDataflow(); dataflow.Start(); await _repository.Read( new TableQuery <MarginTradingAccountHistoryEntity>().Select(MarginTradingAccountHistoryEntity.ColumnNames), entities => dataflow.SendAsync(entities)); await dataflow.CompleteAsync(); errorTcs.TrySetResult(null); await errorTcs.Task; Log($"Fin. Completed: {_counter}; elapsed: {_clock.Elapsed}, speed: {_counter / _clock.Elapsed.TotalMinutes:f2}/min"); }
public async Task Split_SplitTransform_Join_Action_CorrectlyBuild() { // arrange var result = new ConcurrentBag <string> (); var sut = DataflowFluent .ReceiveDataOfType <string> () .SplitTo(s => s.ToCharArray()) .SplitTransform((s, c) => (int)c) .SplitTransform((s, i) => (char)i) .SplitJoinInto(x => new string (x.SuccessfullyCompletedItems.ToArray())) .Action(result.Add); // act var dataflow = sut.CreateDataflow(); await dataflow.ProcessAsync(new[] { "a", "ab", "abc" }); // assert result.Should().BeEquivalentTo("a", "ab", "abc"); }
public async Task SplitTo_WithDefaultExceptionLogger_OneItemsThrows_LogsTheException() { // arrange var exceptions = new ConcurrentBag <Exception> (); var failed_items = new ConcurrentBag <object> (); var sut = DataflowFluent .ReceiveDataOfType <string> () .SplitToAsync <char> (async x => { await Task.Yield(); if (x == "b") { throw new TestException(); } return(x.ToCharArray()); }) .SplitJoin() .WithDefaultExceptionLogger((exception, item) => { exceptions.Add(exception); failed_items.Add(item); }) .CreateDataflow(); // act await sut.ProcessAsync(new[] { "a", "b", "c" }); // assert exceptions.Should().HaveCount(1).And.ContainItemsAssignableTo <TestException> (); failed_items.Should().Equal("b"); }