public void A_Flow_can_have_a_nested_flow_with_a_different_dispatcher() { Source.Single(1) .Via( Flow.Create <int>() .Select(x => SentThreadNameTo(TestActor, x)) .WithAttributes(ActorAttributes.CreateDispatcher("my-dispatcher"))) .To(Sink.Ignore <int>()) .Run(Materializer); ExpectMsg <string>().Should().Contain("my-dispatcher1"); }
public void ActorPublisher_should_use_dispatcher_from_props() { var materializer = Sys.Materializer(); var s = this.CreateManualSubscriberProbe <string>(); var actorRef = Source.ActorPublisher <string>(TestPublisher.Props(TestActor, useTestDispatcher: false).WithDispatcher("my-dispatcher1")) .WithAttributes(ActorAttributes.CreateDispatcher("my-dispatcher2")) .To(Sink.FromSubscriber(s)) .Run(materializer); actorRef.Tell(ThreadName.Instance); ExpectMsg <string>().Should().Contain("my-dispatcher1"); }
public void GroupBy_must_resume_stream_when_GroupBy_function_throws() { this.AssertAllStagesStopped(() => { var publisherProbe = this.CreateManualPublisherProbe <int>(); var ex = new TestException("test"); var publisher = Source.FromPublisher(publisherProbe).GroupBy(2, i => { if (i == 2) { throw ex; } return(i % 2); }) .Lift(x => x % 2) .WithAttributes(ActorAttributes.CreateSupervisionStrategy(Deciders.ResumingDecider)) .RunWith(Sink.AsPublisher <Tuple <int, Source <int, NotUsed> > >(false), Materializer); var subscriber = this.CreateManualSubscriberProbe <Tuple <int, Source <int, NotUsed> > >(); publisher.Subscribe(subscriber); var upstreamSubscription = publisherProbe.ExpectSubscription(); var downstreamSubscription = subscriber.ExpectSubscription(); downstreamSubscription.Request(100); upstreamSubscription.SendNext(1); var substream = subscriber.ExpectNext().Item2; var substreamPuppet1 = new StreamPuppet(substream.RunWith(Sink.AsPublisher <int>(false), Materializer), this); substreamPuppet1.Request(10); substreamPuppet1.ExpectNext(1); upstreamSubscription.SendNext(2); upstreamSubscription.SendNext(4); var substream2 = subscriber.ExpectNext().Item2; var substreamPuppet2 = new StreamPuppet(substream2.RunWith(Sink.AsPublisher <int>(false), Materializer), this); substreamPuppet2.Request(10); substreamPuppet2.ExpectNext(4); upstreamSubscription.SendNext(3); substreamPuppet1.ExpectNext(3); upstreamSubscription.SendNext(6); substreamPuppet2.ExpectNext(6); upstreamSubscription.SendComplete(); subscriber.ExpectComplete(); substreamPuppet1.ExpectComplete(); substreamPuppet2.ExpectComplete(); }, Materializer); }
public async Task SupervisionStrategy_Decider_on_PlainSink_should_work() { var callCount = 0; Directive Decider(Exception cause) { callCount++; switch (cause) { case ProduceException <Null, string> ex when ex.Error.IsSerializationError(): return(Directive.Resume); default: return(Directive.Stop); } } var topic1 = CreateTopic(1); var group1 = CreateGroup(1); var producerSettings = ProducerSettings <Null, string> .Create(Sys, null, new FailingSerializer()) .WithBootstrapServers(Fixture.KafkaServer); // Exception is injected into the sink by the FailingSerializer serializer, it throws an exceptions // when the message "5" is encountered. var sourceTask = Source .From(new [] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }) .Select(elem => new ProducerRecord <Null, string>(new TopicPartition(topic1, 0), elem.ToString())) .RunWith( KafkaProducer.PlainSink(producerSettings) .WithAttributes(ActorAttributes.CreateSupervisionStrategy(Decider)), Materializer); await GuardWithTimeoutAsync(sourceTask, TimeSpan.FromSeconds(5)); var settings = CreateConsumerSettings <Null, string>(group1).WithValueDeserializer(new StringDeserializer()); var probe = KafkaConsumer .PlainSource(settings, Subscriptions.Assignment(new TopicPartition(topic1, 0))) .Select(c => c.Value) .RunWith(this.SinkProbe <string>(), Materializer); probe.Request(10); for (var i = 0; i < 9; i++) { var message = probe.ExpectNext(); Log.Info($"> [{i}]: {message}"); } callCount.Should().Be(1); probe.Cancel(); }
public void A_Flow_with_SelectAsyncUnordered_must_resume_when_task_is_completed_with_null() { var c = TestSubscriber.CreateManualProbe <string>(this); Source.From(new[] { "a", "b", "c" }) .SelectAsyncUnordered(4, s => s.Equals("b") ? Task.FromResult(null as string) : Task.FromResult(s)) .WithAttributes(ActorAttributes.CreateSupervisionStrategy(Deciders.ResumingDecider)) .To(Sink.FromSubscriber(c)).Run(Materializer); var sub = c.ExpectSubscription(); sub.Request(10); c.ExpectNextUnordered("a", "c"); c.ExpectComplete(); }
public async Task SupervisionStrategy_Decider_on_Consumer_Downstream_should_work() { var topic = CreateTopic(1); var group = CreateGroup(1); var topicPartition = new TopicPartition(topic, 0); var callCount = 0; Directive Decider(Exception cause) { callCount++; if (cause.Message == "BOOM!") { return(Directive.Restart); } return(Directive.Stop); } var consumerSettings = CreateConsumerSettings <string>(group); var counter = 0; await Source.From(Enumerable.Range(1, 11)) .Select(elem => new ProducerRecord <Null, string>(topicPartition, elem.ToString())) .RunWith(KafkaProducer.PlainSink(ProducerSettings), Materializer); var(_, probe) = KafkaConsumer .PlainSource(consumerSettings, Subscriptions.Assignment(topicPartition)) .Select(c => { counter++; // fail once on counter 5 if (counter == 5) { throw new Exception("BOOM!"); } return(c.Message.Value); }) .WithAttributes(ActorAttributes.CreateSupervisionStrategy(Decider)) .ToMaterialized(this.SinkProbe <string>(), Keep.Both) .Run(Materializer); probe.Request(10); for (var i = 0; i < 9; i++) { var message = probe.ExpectNext(TimeSpan.FromSeconds(10)); Log.Info(message); } probe.Cancel(); callCount.Should().Be(1); }
public void A_QueueSink_should_retry_failing_messages_if_supervision_strategy_is_resume() { Queue.DeleteIfExists(); var messages = new[] { "1", "2" }; var queueSink = QueueSink.Create(Queue) .WithAttributes(ActorAttributes.CreateSupervisionStrategy(Deciders.ResumingDecider)); var t = Source.From(messages) .Select(x => new CloudQueueMessage(x)) .RunWith(queueSink, Materializer); Thread.Sleep(1000); Queue.Create(); t.Wait(TimeSpan.FromSeconds(3)).Should().BeTrue(); Queue.GetMessages(2).Select(x => x.AsString).ShouldAllBeEquivalentTo(messages); }
public void Conflate_must_restart_when_seed_throws_and_a_RestartDescider_is_used() { var sourceProbe = this.CreatePublisherProbe <int>(); var sinkProbe = this.CreateManualSubscriberProbe <int>(); var exceptionlath = new TestLatch(); var graph = Source.FromPublisher(sourceProbe).ConflateWithSeed(i => { if (i % 2 == 0) { exceptionlath.Open(); throw new TestException("I hate even seed numbers"); } return(i); }, (sum, i) => sum + i) .WithAttributes(ActorAttributes.CreateSupervisionStrategy(Deciders.RestartingDecider)) .To(Sink.FromSubscriber(sinkProbe)) .WithAttributes(Attributes.CreateInputBuffer(1, 1)); RunnableGraph.FromGraph(graph).Run(Materializer); var sub = sourceProbe.ExpectSubscription(); var sinkSub = sinkProbe.ExpectSubscription(); // push the first value sub.ExpectRequest(1); sub.SendNext(1); // and consume it, so that the next element // will trigger seed sinkSub.Request(1); sinkProbe.ExpectNext(1); sub.ExpectRequest(1); sub.SendNext(2); // make sure the seed exception happened // before going any further exceptionlath.Ready(TimeSpan.FromSeconds(3)); sub.ExpectRequest(1); sub.SendNext(3); // now we should have lost the 2 and the accumulated state sinkSub.Request(1); sinkProbe.ExpectNext(3); }
public void A_Collect_must_restart_when_Collect_throws() { bool ThrowOnTwo(int x) => x == 2 ? throw new TestException("") : true; var probe = Source.From(Enumerable.Range(1, 3)) .Collect(ThrowOnTwo, x => x) .WithAttributes(ActorAttributes.CreateSupervisionStrategy(Deciders.RestartingDecider)) .RunWith(this.SinkProbe <int>(), Materializer); probe.Request(1); probe.ExpectNext(1); probe.Request(1); probe.ExpectNext(3); probe.Request(1); probe.ExpectComplete(); }
public void A_Flow_with_SelectAsyncUnordered_must_resume_when_SelectAsyncUnordered_throws() { Source.From(Enumerable.Range(1, 5)) .SelectAsyncUnordered(4, n => { if (n == 3) { throw new TestException("err4"); } return(Task.FromResult(n)); }) .WithAttributes(ActorAttributes.CreateSupervisionStrategy(Deciders.ResumingDecider)) .RunWith(this.SinkProbe <int>(), Materializer) .Request(10) .ExpectNextUnordered(1, 2, 4, 5) .ExpectComplete(); }
private void StartServer() { if (InternalTransport != TransportMode.Tcp) { throw new NotSupportedException("Currently Akka.Streams server supports only TCP transport mode."); } var addressFamily = Settings.DnsUseIpv6 ? AddressFamily.InterNetworkV6 : AddressFamily.InterNetwork; var serverSource = System.TcpStream().Bind(Settings.Hostname, Settings.Port, Settings.Backlog) .AddAttributes(ActorAttributes.CreateDispatcher(System.Settings.Config.GetString("akka.remote.use-dispatcher"))); serverSource.RunForeach(connection => { connection.Flow.Join(StreamTransportFlows.OutboundConnectionHandler(Settings, connection.)); }); }
public void A_AggregateAsync_must_finish_after_task_failure() { this.AssertAllStagesStopped(() => { Source.From(Enumerable.Range(1, 3)).AggregateAsync(1, (_, n) => Task.Run(() => { if (n == 3) { throw new Exception("err3b"); } return(n); })) .WithAttributes(ActorAttributes.CreateSupervisionStrategy(Deciders.ResumingDecider)) .Grouped(10) .RunWith(Sink.First <IEnumerable <int> >(), Materializer) .AwaitResult().ShouldAllBeEquivalentTo(2); }, Materializer); }
public void A_Scan_must_resume_properly() { var scan = Flow.Create <int>().Scan(0, (old, current) => { if (current <= 0) { throw new ArgumentException("current must be greater than zero"); } return(old + current); }).WithAttributes(ActorAttributes.CreateSupervisionStrategy(Deciders.ResumingDecider)); Source.From(new[] { 1, 3, -1, 5, 7 }) .Via(scan) .RunWith(this.SinkProbe <int>(), Materializer) .ToStrict(TimeSpan.FromSeconds(1)) .ShouldAllBeEquivalentTo(new[] { 0, 1, 4, 9, 16 }); }
public void A_SkipWhile_must_continue_if_error() { this.AssertAllStagesStopped(() => { Source.From(Enumerable.Range(1, 4)).SkipWhile(x => { if (x < 3) { return(true); } throw new TestException(""); }) .WithAttributes(ActorAttributes.CreateSupervisionStrategy(Deciders.ResumingDecider)) .RunWith(this.SinkProbe <int>(), Materializer) .Request(1) .ExpectComplete(); }, Materializer); }
public void Stream_supervision_must_resume_stream_when_null_is_emitted() { var nullMap = Flow.Create <string>().Select(element => { if (element == "b") { return(null); } return(element); }).WithAttributes(ActorAttributes.CreateSupervisionStrategy(Deciders.ResumingDecider)); var task = Source.From(new[] { "a", "b", "c" }) .Via(nullMap) .Limit(1000) .RunWith(Sink.Seq <string>(), Materializer); task.Wait(TimeSpan.FromSeconds(3)).Should().BeTrue(); task.Result.ShouldAllBeEquivalentTo(new [] { "a", "c" }); }
public void Flow_with_ask_must_resume_after_ask_failure() => this.AssertAllStagesStopped(() => { var c = this.CreateManualSubscriberProbe <Reply>(); var aref = ReplierFailOn(3); var p = Source.From(Enumerable.Range(1, 5)) .Ask <Reply>(aref, _timeout, 4) .WithAttributes(ActorAttributes.CreateSupervisionStrategy(Supervision.Deciders.ResumingDecider)) .To(Sink.FromSubscriber(c)) .Run(_materializer); var sub = c.ExpectSubscription(); sub.Request(10); foreach (var i in new[] { 1, 2, 4, 5 }) { c.ExpectNext(new Reply(i)); } c.ExpectComplete(); }, _materializer);
public void A_Flow_can_have_multiple_levels_of_nesting() { var probe1 = CreateTestProbe(); var probe2 = CreateTestProbe(); var flow1 = Flow.Create <int>() .Select(x => SentThreadNameTo(probe1.Ref, x)) .WithAttributes(ActorAttributes.CreateDispatcher("my-dispatcher1")); var flow2 = flow1 .Via(Flow.Create <int>().Select(x => SentThreadNameTo(probe2.Ref, x))) .WithAttributes(ActorAttributes.CreateDispatcher("my-dispatcher2")); Source.Single(1).Via(flow2).To(Sink.Ignore <int>()).Run(Materializer); probe1.ExpectMsg <string>().Should().Contain("my-dispatcher1"); probe2.ExpectMsg <string>().Should().Contain("my-dispatcher2"); }
public async Task A_QueueSink_should_retry_failing_messages_if_supervision_strategy_is_resume() { await Queue.DeleteIfExistsAsync(); var messages = new[] { "1", "2" }; var queueSink = QueueSink.Create(Queue) .WithAttributes(ActorAttributes.CreateSupervisionStrategy(Deciders.ResumingDecider)); var t = Source.From(messages) //.Select(x => new QueueMessage(x)) .RunWith(queueSink, Materializer); await Task.Delay(1000); await Queue.CreateAsync(); t.Wait(TimeSpan.FromSeconds(3)).Should().BeTrue(); (await Queue.ReceiveMessagesAsync(2)).Value.Select(x => x.MessageText).Should().BeEquivalentTo(messages); }
public async Task SupervisionStrategy_Decider_on_Producer_Upstream_should_work() { var topic = CreateTopic(1); var group = CreateGroup(1); var topicPartition = new TopicPartition(topic, 0); var callCount = 0; // create a custom Decider with a "Restart" directive in the event of DivideByZeroException Directive Decider(Exception cause) { callCount++; return(cause is DivideByZeroException ? Directive.Restart : Directive.Stop); } var consumerSettings = CreateConsumerSettings <string>(group); var numbers = Source.From(new [] { 9, 8, 7, 6, 0, 5, 4, 3, 2, 1 }); await numbers // a DivideByZeroException will be thrown here, and since this happens upstream of the producer sink, // the whole stream got restarted when the exception happened, and the offending message will be ignored. // All the messages prior and after the exception are sent to the Kafka producer. .Via(Flow.Create <int>().Select(x => $"1/{x} is {1/x} w/ integer division")) .WithAttributes(ActorAttributes.CreateSupervisionStrategy(Decider)) .Select(elem => new ProducerRecord <Null, string>(topicPartition, elem)) .RunWith(KafkaProducer.PlainSink(ProducerSettings), Materializer); var(_, probe) = KafkaConsumer .PlainSource(consumerSettings, Subscriptions.Assignment(topicPartition)) .Select(c => c.Message.Value) .ToMaterialized(this.SinkProbe <string>(), Keep.Both) .Run(Materializer); probe.Request(10); for (var i = 0; i < 9; i++) { Log.Info(probe.ExpectNext(TimeSpan.FromSeconds(10))); } probe.Cancel(); callCount.Should().BeGreaterThan(0); }
public void A_Flow_with_SelectAsync_must_finish_after_task_failure() { this.AssertAllStagesStopped(() => { var t = Source.From(Enumerable.Range(1, 3)) .SelectAsync(1, n => Task.Run(() => { if (n == 3) { throw new TestException("err3b"); } return(n); })) .WithAttributes(ActorAttributes.CreateSupervisionStrategy(Deciders.ResumingDecider)) .Grouped(10) .RunWith(Sink.First <IEnumerable <int> >(), Materializer); t.AwaitResult().Should().BeEquivalentTo(new[] { 1, 2 }); }, Materializer); }
public void A_Aggregate_must_resume_and_reset_the_state_when_the_reduce_funtion_throws_and_the_supervisor_strategy_decides_to_restart() { this.AssertAllStagesStopped(() => { var error = new Exception("boom"); var sum = Sink.Sum((int x, int y) => { if (y == 50) { throw error; } return(x + y); }); var task = InputSource.RunWith( sum.WithAttributes(ActorAttributes.CreateSupervisionStrategy(Deciders.RestartingDecider)), Materializer); task.AwaitResult().Should().Be(Enumerable.Range(51, 50).Sum()); }, Materializer); }
public void A_Aggregate_must_resume_with_the_accumulated_state_when_the_aggregating_funtion_throws_and_the_supervisor_strategy_decides_to_resume() { this.AssertAllStagesStopped(() => { var error = new Exception("boom"); var aggregate = Sink.Aggregate(0, (int x, int y) => { if (y == 50) { throw error; } return(x + y); }); var task = InputSource.RunWith( aggregate.WithAttributes(ActorAttributes.CreateSupervisionStrategy(Deciders.ResumingDecider)), Materializer); task.AwaitResult().Should().Be(Expected - 50); }, Materializer); }
public void A_Flow_can_have_an_op_section_with_different_dispatcher_and_name() { var defaultDispatcher = CreateTestProbe(); var customDispatcher = CreateTestProbe(); var f1 = Flow.Create <int>().Select(x => SentThreadNameTo(defaultDispatcher.Ref, x)); var f2 = Flow.Create <int>() .Select(x => SentThreadNameTo(defaultDispatcher.Ref, x)) .Select(x => x) .WithAttributes( ActorAttributes.CreateDispatcher("my-dispatcher") .And(Attributes.CreateName("seperate-dispatcher"))); Source.From(new[] { 0, 1, 2 }).Via(f1).Via(f2).RunWith(Sink.Ignore <int>(), Materializer); defaultDispatcher.ReceiveN(3).ForEach(o => o.ToString().Should().Contain("akka.test.stream-dispatcher")); customDispatcher.ReceiveN(3).ForEach(o => o.ToString().Should().Contain("my-dispatcher")); }
public void A_QueueSource_should_not_fail_if_the_supervision_strategy_is_not_stop_when_an_error_occurs() { Queue.DeleteIfExists(); var probe = QueueSource.Create(Queue, pollInterval: TimeSpan.FromSeconds(1)) .Take(3) .Select(x => x.AsString) .WithAttributes(ActorAttributes.CreateSupervisionStrategy(Deciders.ResumingDecider)) .RunWith(this.SinkProbe <string>(), Materializer); probe.Request(3).ExpectNoMsg(); Queue.CreateIfNotExists(); Queue.AddMessage(new CloudQueueMessage("Test1")); Queue.AddMessage(new CloudQueueMessage("Test2")); Queue.AddMessage(new CloudQueueMessage("Test3")); probe.ExpectNext("Test1", "Test2", "Test3") .ExpectComplete(); }
public void A_TakeWhile_must_continue_if_error() { this.AssertAllStagesStopped(() => { var testException = new Exception("test"); Source.From(Enumerable.Range(1, 4)).TakeWhile(a => { if (a == 3) { throw testException; } return(true); }) .WithAttributes(ActorAttributes.CreateSupervisionStrategy(Deciders.ResumingDecider)) .RunWith(this.SinkProbe <int>(), Materializer) .Request(4) .ExpectNext(1, 2, 4) .ExpectComplete(); }, Materializer); }
public void A_Flow_with_SelectAsync_must_resume_when_SelectAsync_throws() { var c = this.CreateManualSubscriberProbe <int>(); Source.From(Enumerable.Range(1, 5)) .SelectAsync(4, n => { if (n == 3) { throw new TestException("err4"); } return(Task.FromResult(n)); }) .WithAttributes(ActorAttributes.CreateSupervisionStrategy(Deciders.ResumingDecider)) .RunWith(Sink.FromSubscriber(c), Materializer); var sub = c.ExpectSubscription(); sub.Request(10); new[] { 1, 2, 4, 5 }.ForEach(i => c.ExpectNext(i)); c.ExpectComplete(); }
private TestSubscriber.ManualProbe <int> WhenFailedScan(ICollection <int> elements, int zero, Exception exception = null, Decider decider = null) { exception = exception ?? new Exception("boom"); decider = decider ?? Deciders.StoppingDecider; return(Source.From(elements) .ScanAsync(zero, (i, i1) => { if (i1 >= 0) { return Task.FromResult(i + i1); } throw exception; }) .WithAttributes(ActorAttributes.CreateSupervisionStrategy(decider)) .RunWith(this.SinkProbe <int>(), Materializer) .Request(elements.Count + 1) .ExpectNext(zero)); }
public void A_AggregateAsync_must_resume_after_multiple_failures() { this.AssertAllStagesStopped(() => { var tasks = new [] { FailedTask("failure1"), FailedTask("failure2"), FailedTask("failure3"), FailedTask("failure4"), FailedTask("failure5"), Task.FromResult("happy!") }; Source.From(tasks) .AggregateAsync(string.Empty, (_, t) => t) .WithAttributes(ActorAttributes.CreateSupervisionStrategy(Deciders.ResumingDecider)) .RunWith(Sink.First <string>(), Materializer) .AwaitResult().Should().Be("happy!"); }, Materializer); }
public void A_QueueSink_should_skip_failing_messages_if_supervision_strategy_is_restart() { Queue.DeleteIfExists(); var queueSink = QueueSink.Create(Queue) .WithAttributes(ActorAttributes.CreateSupervisionStrategy(Deciders.RestartingDecider)); var t = this.SourceProbe <string>() .Select(x => new CloudQueueMessage(x)) .ToMaterialized(queueSink, Keep.Both) .Run(Materializer); var probe = t.Item1; var task = t.Item2; probe.SendNext("1"); Thread.Sleep(500); Queue.Create(); probe.SendNext("2"); probe.SendComplete(); task.Wait(TimeSpan.FromSeconds(3)).Should().BeTrue(); Queue.GetMessage().AsString.Should().Be("2"); }
public void A_LazySink_must_fail_gracefully_when_factory_task_failed() { this.AssertAllStagesStopped(() => { var failedTask = new TaskFactory <Sink <int, TestSubscriber.Probe <int> > >().StartNew(() => { throw Ex; }); var sourceProbe = this.CreateManualPublisherProbe <int>(); var lazySink = Sink.LazySink((int _) => failedTask, Fallback <TestSubscriber.Probe <int> >()); var taskProbe = Source.FromPublisher(sourceProbe) .ToMaterialized(lazySink, Keep.Right) .WithAttributes(ActorAttributes.CreateSupervisionStrategy(Deciders.StoppingDecider)) .Run(Materializer); var sourceSub = sourceProbe.ExpectSubscription(); sourceSub.ExpectRequest(1); sourceSub.SendNext(0); taskProbe.Invoking(t => t.Wait(TimeSpan.FromMilliseconds(300))).ShouldThrow <TestException>(); }, Materializer); }