public void A_Flow_can_have_a_nested_flow_with_a_different_dispatcher()
{
Source.Single(1)
.Via(
Flow.Create <int>()
.Select(x => SentThreadNameTo(TestActor, x))
.WithAttributes(ActorAttributes.CreateDispatcher("my-dispatcher")))
.To(Sink.Ignore <int>())
.Run(Materializer);
ExpectMsg <string>().Should().Contain("my-dispatcher1");
}
public void ActorPublisher_should_use_dispatcher_from_props()
{
var materializer = Sys.Materializer();
var s = this.CreateManualSubscriberProbe <string>();
var actorRef = Source.ActorPublisher <string>(TestPublisher.Props(TestActor, useTestDispatcher: false).WithDispatcher("my-dispatcher1"))
.WithAttributes(ActorAttributes.CreateDispatcher("my-dispatcher2"))
.To(Sink.FromSubscriber(s))
.Run(materializer);
actorRef.Tell(ThreadName.Instance);
ExpectMsg <string>().Should().Contain("my-dispatcher1");
}
public void GroupBy_must_resume_stream_when_GroupBy_function_throws()
{
this.AssertAllStagesStopped(() =>
{
var publisherProbe = this.CreateManualPublisherProbe <int>();
var ex = new TestException("test");
var publisher = Source.FromPublisher(publisherProbe).GroupBy(2, i =>
{
if (i == 2)
{
throw ex;
}
return(i % 2);
})
.Lift(x => x % 2)
.WithAttributes(ActorAttributes.CreateSupervisionStrategy(Deciders.ResumingDecider))
.RunWith(Sink.AsPublisher <Tuple <int, Source <int, NotUsed> > >(false), Materializer);
var subscriber = this.CreateManualSubscriberProbe <Tuple <int, Source <int, NotUsed> > >();
publisher.Subscribe(subscriber);
var upstreamSubscription = publisherProbe.ExpectSubscription();
var downstreamSubscription = subscriber.ExpectSubscription();
downstreamSubscription.Request(100);
upstreamSubscription.SendNext(1);
var substream = subscriber.ExpectNext().Item2;
var substreamPuppet1 = new StreamPuppet(substream.RunWith(Sink.AsPublisher <int>(false), Materializer), this);
substreamPuppet1.Request(10);
substreamPuppet1.ExpectNext(1);
upstreamSubscription.SendNext(2);
upstreamSubscription.SendNext(4);
var substream2 = subscriber.ExpectNext().Item2;
var substreamPuppet2 = new StreamPuppet(substream2.RunWith(Sink.AsPublisher <int>(false), Materializer), this);
substreamPuppet2.Request(10);
substreamPuppet2.ExpectNext(4);
upstreamSubscription.SendNext(3);
substreamPuppet1.ExpectNext(3);
upstreamSubscription.SendNext(6);
substreamPuppet2.ExpectNext(6);
upstreamSubscription.SendComplete();
subscriber.ExpectComplete();
substreamPuppet1.ExpectComplete();
substreamPuppet2.ExpectComplete();
}, Materializer);
}
public async Task SupervisionStrategy_Decider_on_PlainSink_should_work()
{
var callCount = 0;
Directive Decider(Exception cause)
{
callCount++;
switch (cause)
{
case ProduceException <Null, string> ex when ex.Error.IsSerializationError():
return(Directive.Resume);
default:
return(Directive.Stop);
}
}
var topic1 = CreateTopic(1);
var group1 = CreateGroup(1);
var producerSettings = ProducerSettings <Null, string>
.Create(Sys, null, new FailingSerializer())
.WithBootstrapServers(Fixture.KafkaServer);
// Exception is injected into the sink by the FailingSerializer serializer, it throws an exceptions
// when the message "5" is encountered.
var sourceTask = Source
.From(new [] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 })
.Select(elem => new ProducerRecord <Null, string>(new TopicPartition(topic1, 0), elem.ToString()))
.RunWith(
KafkaProducer.PlainSink(producerSettings)
.WithAttributes(ActorAttributes.CreateSupervisionStrategy(Decider)),
Materializer);
await GuardWithTimeoutAsync(sourceTask, TimeSpan.FromSeconds(5));
var settings = CreateConsumerSettings <Null, string>(group1).WithValueDeserializer(new StringDeserializer());
var probe = KafkaConsumer
.PlainSource(settings, Subscriptions.Assignment(new TopicPartition(topic1, 0)))
.Select(c => c.Value)
.RunWith(this.SinkProbe <string>(), Materializer);
probe.Request(10);
for (var i = 0; i < 9; i++)
{
var message = probe.ExpectNext();
Log.Info($"> [{i}]: {message}");
}
callCount.Should().Be(1);
probe.Cancel();
}
public void A_Flow_with_SelectAsyncUnordered_must_resume_when_task_is_completed_with_null()
{
var c = TestSubscriber.CreateManualProbe <string>(this);
Source.From(new[] { "a", "b", "c" })
.SelectAsyncUnordered(4, s => s.Equals("b") ? Task.FromResult(null as string) : Task.FromResult(s))
.WithAttributes(ActorAttributes.CreateSupervisionStrategy(Deciders.ResumingDecider))
.To(Sink.FromSubscriber(c)).Run(Materializer);
var sub = c.ExpectSubscription();
sub.Request(10);
c.ExpectNextUnordered("a", "c");
c.ExpectComplete();
}
public async Task SupervisionStrategy_Decider_on_Consumer_Downstream_should_work()
{
var topic = CreateTopic(1);
var group = CreateGroup(1);
var topicPartition = new TopicPartition(topic, 0);
var callCount = 0;
Directive Decider(Exception cause)
{
callCount++;
if (cause.Message == "BOOM!")
{
return(Directive.Restart);
}
return(Directive.Stop);
}
var consumerSettings = CreateConsumerSettings <string>(group);
var counter = 0;
await Source.From(Enumerable.Range(1, 11))
.Select(elem => new ProducerRecord <Null, string>(topicPartition, elem.ToString()))
.RunWith(KafkaProducer.PlainSink(ProducerSettings), Materializer);
var(_, probe) = KafkaConsumer
.PlainSource(consumerSettings, Subscriptions.Assignment(topicPartition))
.Select(c =>
{
counter++;
// fail once on counter 5
if (counter == 5)
{
throw new Exception("BOOM!");
}
return(c.Message.Value);
})
.WithAttributes(ActorAttributes.CreateSupervisionStrategy(Decider))
.ToMaterialized(this.SinkProbe <string>(), Keep.Both)
.Run(Materializer);
probe.Request(10);
for (var i = 0; i < 9; i++)
{
var message = probe.ExpectNext(TimeSpan.FromSeconds(10));
Log.Info(message);
}
probe.Cancel();
callCount.Should().Be(1);
}
public void A_QueueSink_should_retry_failing_messages_if_supervision_strategy_is_resume()
{
Queue.DeleteIfExists();
var messages = new[] { "1", "2" };
var queueSink = QueueSink.Create(Queue)
.WithAttributes(ActorAttributes.CreateSupervisionStrategy(Deciders.ResumingDecider));
var t = Source.From(messages)
.Select(x => new CloudQueueMessage(x))
.RunWith(queueSink, Materializer);
Thread.Sleep(1000);
Queue.Create();
t.Wait(TimeSpan.FromSeconds(3)).Should().BeTrue();
Queue.GetMessages(2).Select(x => x.AsString).ShouldAllBeEquivalentTo(messages);
}
public void Conflate_must_restart_when_seed_throws_and_a_RestartDescider_is_used()
{
var sourceProbe = this.CreatePublisherProbe <int>();
var sinkProbe = this.CreateManualSubscriberProbe <int>();
var exceptionlath = new TestLatch();
var graph = Source.FromPublisher(sourceProbe).ConflateWithSeed(i =>
{
if (i % 2 == 0)
{
exceptionlath.Open();
throw new TestException("I hate even seed numbers");
}
return(i);
}, (sum, i) => sum + i)
.WithAttributes(ActorAttributes.CreateSupervisionStrategy(Deciders.RestartingDecider))
.To(Sink.FromSubscriber(sinkProbe))
.WithAttributes(Attributes.CreateInputBuffer(1, 1));
RunnableGraph.FromGraph(graph).Run(Materializer);
var sub = sourceProbe.ExpectSubscription();
var sinkSub = sinkProbe.ExpectSubscription();
// push the first value
sub.ExpectRequest(1);
sub.SendNext(1);
// and consume it, so that the next element
// will trigger seed
sinkSub.Request(1);
sinkProbe.ExpectNext(1);
sub.ExpectRequest(1);
sub.SendNext(2);
// make sure the seed exception happened
// before going any further
exceptionlath.Ready(TimeSpan.FromSeconds(3));
sub.ExpectRequest(1);
sub.SendNext(3);
// now we should have lost the 2 and the accumulated state
sinkSub.Request(1);
sinkProbe.ExpectNext(3);
}
public void A_Collect_must_restart_when_Collect_throws()
{
bool ThrowOnTwo(int x) => x == 2 ? throw new TestException("") : true;
var probe =
Source.From(Enumerable.Range(1, 3))
.Collect(ThrowOnTwo, x => x)
.WithAttributes(ActorAttributes.CreateSupervisionStrategy(Deciders.RestartingDecider))
.RunWith(this.SinkProbe <int>(), Materializer);
probe.Request(1);
probe.ExpectNext(1);
probe.Request(1);
probe.ExpectNext(3);
probe.Request(1);
probe.ExpectComplete();
}
public void A_Flow_with_SelectAsyncUnordered_must_resume_when_SelectAsyncUnordered_throws()
{
Source.From(Enumerable.Range(1, 5))
.SelectAsyncUnordered(4, n =>
{
if (n == 3)
{
throw new TestException("err4");
}
return(Task.FromResult(n));
})
.WithAttributes(ActorAttributes.CreateSupervisionStrategy(Deciders.ResumingDecider))
.RunWith(this.SinkProbe <int>(), Materializer)
.Request(10)
.ExpectNextUnordered(1, 2, 4, 5)
.ExpectComplete();
}
private void StartServer()
{
if (InternalTransport != TransportMode.Tcp)
{
throw new NotSupportedException("Currently Akka.Streams server supports only TCP transport mode.");
}
var addressFamily = Settings.DnsUseIpv6 ? AddressFamily.InterNetworkV6 : AddressFamily.InterNetwork;
var serverSource = System.TcpStream().Bind(Settings.Hostname, Settings.Port, Settings.Backlog)
.AddAttributes(ActorAttributes.CreateDispatcher(System.Settings.Config.GetString("akka.remote.use-dispatcher")));
serverSource.RunForeach(connection =>
{
connection.Flow.Join(StreamTransportFlows.OutboundConnectionHandler(Settings, connection.));
});
}
public void A_AggregateAsync_must_finish_after_task_failure()
{
this.AssertAllStagesStopped(() =>
{
Source.From(Enumerable.Range(1, 3)).AggregateAsync(1, (_, n) => Task.Run(() =>
{
if (n == 3)
{
throw new Exception("err3b");
}
return(n);
}))
.WithAttributes(ActorAttributes.CreateSupervisionStrategy(Deciders.ResumingDecider))
.Grouped(10)
.RunWith(Sink.First <IEnumerable <int> >(), Materializer)
.AwaitResult().ShouldAllBeEquivalentTo(2);
}, Materializer);
}
public void A_Scan_must_resume_properly()
{
var scan = Flow.Create <int>().Scan(0, (old, current) =>
{
if (current <= 0)
{
throw new ArgumentException("current must be greater than zero");
}
return(old + current);
}).WithAttributes(ActorAttributes.CreateSupervisionStrategy(Deciders.ResumingDecider));
Source.From(new[] { 1, 3, -1, 5, 7 })
.Via(scan)
.RunWith(this.SinkProbe <int>(), Materializer)
.ToStrict(TimeSpan.FromSeconds(1))
.ShouldAllBeEquivalentTo(new[] { 0, 1, 4, 9, 16 });
}
public void A_SkipWhile_must_continue_if_error()
{
this.AssertAllStagesStopped(() =>
{
Source.From(Enumerable.Range(1, 4)).SkipWhile(x =>
{
if (x < 3)
{
return(true);
}
throw new TestException("");
})
.WithAttributes(ActorAttributes.CreateSupervisionStrategy(Deciders.ResumingDecider))
.RunWith(this.SinkProbe <int>(), Materializer)
.Request(1)
.ExpectComplete();
}, Materializer);
}
public void Stream_supervision_must_resume_stream_when_null_is_emitted()
{
var nullMap = Flow.Create <string>().Select(element =>
{
if (element == "b")
{
return(null);
}
return(element);
}).WithAttributes(ActorAttributes.CreateSupervisionStrategy(Deciders.ResumingDecider));
var task = Source.From(new[] { "a", "b", "c" })
.Via(nullMap)
.Limit(1000)
.RunWith(Sink.Seq <string>(), Materializer);
task.Wait(TimeSpan.FromSeconds(3)).Should().BeTrue();
task.Result.ShouldAllBeEquivalentTo(new [] { "a", "c" });
}
public void Flow_with_ask_must_resume_after_ask_failure() => this.AssertAllStagesStopped(() =>
{
var c = this.CreateManualSubscriberProbe <Reply>();
var aref = ReplierFailOn(3);
var p = Source.From(Enumerable.Range(1, 5))
.Ask <Reply>(aref, _timeout, 4)
.WithAttributes(ActorAttributes.CreateSupervisionStrategy(Supervision.Deciders.ResumingDecider))
.To(Sink.FromSubscriber(c))
.Run(_materializer);
var sub = c.ExpectSubscription();
sub.Request(10);
foreach (var i in new[] { 1, 2, 4, 5 })
{
c.ExpectNext(new Reply(i));
}
c.ExpectComplete();
}, _materializer);
public void A_Flow_can_have_multiple_levels_of_nesting()
{
var probe1 = CreateTestProbe();
var probe2 = CreateTestProbe();
var flow1 =
Flow.Create <int>()
.Select(x => SentThreadNameTo(probe1.Ref, x))
.WithAttributes(ActorAttributes.CreateDispatcher("my-dispatcher1"));
var flow2 = flow1
.Via(Flow.Create <int>().Select(x => SentThreadNameTo(probe2.Ref, x)))
.WithAttributes(ActorAttributes.CreateDispatcher("my-dispatcher2"));
Source.Single(1).Via(flow2).To(Sink.Ignore <int>()).Run(Materializer);
probe1.ExpectMsg <string>().Should().Contain("my-dispatcher1");
probe2.ExpectMsg <string>().Should().Contain("my-dispatcher2");
}
public async Task A_QueueSink_should_retry_failing_messages_if_supervision_strategy_is_resume()
{
await Queue.DeleteIfExistsAsync();
var messages = new[] { "1", "2" };
var queueSink = QueueSink.Create(Queue)
.WithAttributes(ActorAttributes.CreateSupervisionStrategy(Deciders.ResumingDecider));
var t = Source.From(messages)
//.Select(x => new QueueMessage(x))
.RunWith(queueSink, Materializer);
await Task.Delay(1000);
await Queue.CreateAsync();
t.Wait(TimeSpan.FromSeconds(3)).Should().BeTrue();
(await Queue.ReceiveMessagesAsync(2)).Value.Select(x => x.MessageText).Should().BeEquivalentTo(messages);
}
public async Task SupervisionStrategy_Decider_on_Producer_Upstream_should_work()
{
var topic = CreateTopic(1);
var group = CreateGroup(1);
var topicPartition = new TopicPartition(topic, 0);
var callCount = 0;
// create a custom Decider with a "Restart" directive in the event of DivideByZeroException
Directive Decider(Exception cause)
{
callCount++;
return(cause is DivideByZeroException
? Directive.Restart
: Directive.Stop);
}
var consumerSettings = CreateConsumerSettings <string>(group);
var numbers = Source.From(new [] { 9, 8, 7, 6, 0, 5, 4, 3, 2, 1 });
await numbers
// a DivideByZeroException will be thrown here, and since this happens upstream of the producer sink,
// the whole stream got restarted when the exception happened, and the offending message will be ignored.
// All the messages prior and after the exception are sent to the Kafka producer.
.Via(Flow.Create <int>().Select(x => $"1/{x} is {1/x} w/ integer division"))
.WithAttributes(ActorAttributes.CreateSupervisionStrategy(Decider))
.Select(elem => new ProducerRecord <Null, string>(topicPartition, elem))
.RunWith(KafkaProducer.PlainSink(ProducerSettings), Materializer);
var(_, probe) = KafkaConsumer
.PlainSource(consumerSettings, Subscriptions.Assignment(topicPartition))
.Select(c => c.Message.Value)
.ToMaterialized(this.SinkProbe <string>(), Keep.Both)
.Run(Materializer);
probe.Request(10);
for (var i = 0; i < 9; i++)
{
Log.Info(probe.ExpectNext(TimeSpan.FromSeconds(10)));
}
probe.Cancel();
callCount.Should().BeGreaterThan(0);
}
public void A_Flow_with_SelectAsync_must_finish_after_task_failure()
{
this.AssertAllStagesStopped(() =>
{
var t = Source.From(Enumerable.Range(1, 3))
.SelectAsync(1, n => Task.Run(() =>
{
if (n == 3)
{
throw new TestException("err3b");
}
return(n);
}))
.WithAttributes(ActorAttributes.CreateSupervisionStrategy(Deciders.ResumingDecider))
.Grouped(10)
.RunWith(Sink.First <IEnumerable <int> >(), Materializer);
t.AwaitResult().Should().BeEquivalentTo(new[] { 1, 2 });
}, Materializer);
}
public void A_Aggregate_must_resume_and_reset_the_state_when_the_reduce_funtion_throws_and_the_supervisor_strategy_decides_to_restart()
{
this.AssertAllStagesStopped(() =>
{
var error = new Exception("boom");
var sum = Sink.Sum((int x, int y) =>
{
if (y == 50)
{
throw error;
}
return(x + y);
});
var task = InputSource.RunWith(
sum.WithAttributes(ActorAttributes.CreateSupervisionStrategy(Deciders.RestartingDecider)),
Materializer);
task.AwaitResult().Should().Be(Enumerable.Range(51, 50).Sum());
}, Materializer);
}
public void A_Aggregate_must_resume_with_the_accumulated_state_when_the_aggregating_funtion_throws_and_the_supervisor_strategy_decides_to_resume()
{
this.AssertAllStagesStopped(() =>
{
var error = new Exception("boom");
var aggregate = Sink.Aggregate(0, (int x, int y) =>
{
if (y == 50)
{
throw error;
}
return(x + y);
});
var task = InputSource.RunWith(
aggregate.WithAttributes(ActorAttributes.CreateSupervisionStrategy(Deciders.ResumingDecider)),
Materializer);
task.AwaitResult().Should().Be(Expected - 50);
}, Materializer);
}
public void A_Flow_can_have_an_op_section_with_different_dispatcher_and_name()
{
var defaultDispatcher = CreateTestProbe();
var customDispatcher = CreateTestProbe();
var f1 = Flow.Create <int>().Select(x => SentThreadNameTo(defaultDispatcher.Ref, x));
var f2 =
Flow.Create <int>()
.Select(x => SentThreadNameTo(defaultDispatcher.Ref, x))
.Select(x => x)
.WithAttributes(
ActorAttributes.CreateDispatcher("my-dispatcher")
.And(Attributes.CreateName("seperate-dispatcher")));
Source.From(new[] { 0, 1, 2 }).Via(f1).Via(f2).RunWith(Sink.Ignore <int>(), Materializer);
defaultDispatcher.ReceiveN(3).ForEach(o => o.ToString().Should().Contain("akka.test.stream-dispatcher"));
customDispatcher.ReceiveN(3).ForEach(o => o.ToString().Should().Contain("my-dispatcher"));
}
public void A_QueueSource_should_not_fail_if_the_supervision_strategy_is_not_stop_when_an_error_occurs()
{
Queue.DeleteIfExists();
var probe = QueueSource.Create(Queue, pollInterval: TimeSpan.FromSeconds(1))
.Take(3)
.Select(x => x.AsString)
.WithAttributes(ActorAttributes.CreateSupervisionStrategy(Deciders.ResumingDecider))
.RunWith(this.SinkProbe <string>(), Materializer);
probe.Request(3).ExpectNoMsg();
Queue.CreateIfNotExists();
Queue.AddMessage(new CloudQueueMessage("Test1"));
Queue.AddMessage(new CloudQueueMessage("Test2"));
Queue.AddMessage(new CloudQueueMessage("Test3"));
probe.ExpectNext("Test1", "Test2", "Test3")
.ExpectComplete();
}
public void A_TakeWhile_must_continue_if_error()
{
this.AssertAllStagesStopped(() =>
{
var testException = new Exception("test");
Source.From(Enumerable.Range(1, 4)).TakeWhile(a =>
{
if (a == 3)
{
throw testException;
}
return(true);
})
.WithAttributes(ActorAttributes.CreateSupervisionStrategy(Deciders.ResumingDecider))
.RunWith(this.SinkProbe <int>(), Materializer)
.Request(4)
.ExpectNext(1, 2, 4)
.ExpectComplete();
}, Materializer);
}
public void A_Flow_with_SelectAsync_must_resume_when_SelectAsync_throws()
{
var c = this.CreateManualSubscriberProbe <int>();
Source.From(Enumerable.Range(1, 5))
.SelectAsync(4, n =>
{
if (n == 3)
{
throw new TestException("err4");
}
return(Task.FromResult(n));
})
.WithAttributes(ActorAttributes.CreateSupervisionStrategy(Deciders.ResumingDecider))
.RunWith(Sink.FromSubscriber(c), Materializer);
var sub = c.ExpectSubscription();
sub.Request(10);
new[] { 1, 2, 4, 5 }.ForEach(i => c.ExpectNext(i));
c.ExpectComplete();
}
private TestSubscriber.ManualProbe <int> WhenFailedScan(ICollection <int> elements, int zero, Exception exception = null,
Decider decider = null)
{
exception = exception ?? new Exception("boom");
decider = decider ?? Deciders.StoppingDecider;
return(Source.From(elements)
.ScanAsync(zero, (i, i1) =>
{
if (i1 >= 0)
{
return Task.FromResult(i + i1);
}
throw exception;
})
.WithAttributes(ActorAttributes.CreateSupervisionStrategy(decider))
.RunWith(this.SinkProbe <int>(), Materializer)
.Request(elements.Count + 1)
.ExpectNext(zero));
}
public void A_AggregateAsync_must_resume_after_multiple_failures()
{
this.AssertAllStagesStopped(() =>
{
var tasks = new []
{
FailedTask("failure1"),
FailedTask("failure2"),
FailedTask("failure3"),
FailedTask("failure4"),
FailedTask("failure5"),
Task.FromResult("happy!")
};
Source.From(tasks)
.AggregateAsync(string.Empty, (_, t) => t)
.WithAttributes(ActorAttributes.CreateSupervisionStrategy(Deciders.ResumingDecider))
.RunWith(Sink.First <string>(), Materializer)
.AwaitResult().Should().Be("happy!");
}, Materializer);
}
public void A_QueueSink_should_skip_failing_messages_if_supervision_strategy_is_restart()
{
Queue.DeleteIfExists();
var queueSink = QueueSink.Create(Queue)
.WithAttributes(ActorAttributes.CreateSupervisionStrategy(Deciders.RestartingDecider));
var t = this.SourceProbe <string>()
.Select(x => new CloudQueueMessage(x))
.ToMaterialized(queueSink, Keep.Both)
.Run(Materializer);
var probe = t.Item1;
var task = t.Item2;
probe.SendNext("1");
Thread.Sleep(500);
Queue.Create();
probe.SendNext("2");
probe.SendComplete();
task.Wait(TimeSpan.FromSeconds(3)).Should().BeTrue();
Queue.GetMessage().AsString.Should().Be("2");
}
public void A_LazySink_must_fail_gracefully_when_factory_task_failed()
{
this.AssertAllStagesStopped(() =>
{
var failedTask = new TaskFactory <Sink <int, TestSubscriber.Probe <int> > >().StartNew(() =>
{
throw Ex;
});
var sourceProbe = this.CreateManualPublisherProbe <int>();
var lazySink = Sink.LazySink((int _) => failedTask, Fallback <TestSubscriber.Probe <int> >());
var taskProbe =
Source.FromPublisher(sourceProbe)
.ToMaterialized(lazySink, Keep.Right)
.WithAttributes(ActorAttributes.CreateSupervisionStrategy(Deciders.StoppingDecider))
.Run(Materializer);
var sourceSub = sourceProbe.ExpectSubscription();
sourceSub.ExpectRequest(1);
sourceSub.SendNext(0);
taskProbe.Invoking(t => t.Wait(TimeSpan.FromMilliseconds(300))).ShouldThrow <TestException>();
}, Materializer);
}
