public void SynchronousFileSink_should_write_each_element_if_auto_flush_is_set() { this.AssertAllStagesStopped(() => { TargetFile(f => { var(actor, task) = Source.ActorRef <string>(64, OverflowStrategy.DropNew) .Select(ByteString.FromString) .ToMaterialized(FileIO.ToFile(f, fileMode: FileMode.OpenOrCreate, startPosition: 0, autoFlush: true), (a, t) => (a, t)) .Run(_materializer); actor.Tell("a\n"); actor.Tell("b\n"); // wait for flush Thread.Sleep(100); CheckFileContent(f, "a\nb\n"); actor.Tell("a\n"); actor.Tell("b\n"); actor.Tell(new Status.Success(NotUsed.Instance)); task.Wait(TimeSpan.FromSeconds(3)).Should().BeTrue(); f.Length.ShouldBe(8); CheckFileContent(f, "a\nb\na\nb\n"); }); }, _materializer); }
private void Handle(Setup m) { Become(Ready); Stash.UnstashAll(); var source = Source.ActorRef <object>(5000, OverflowStrategy.DropTail); var graph = GraphDsl.Create(source, (builder, start) => { var sink = builder.Add(SinksHelper.SimpleWithBackoff(option => { option.HostAndPorts = m.HostAndPorts; option.UserName = m.UserName; option.Password = m.Password; option.QueueName = m.QueueName; option.VirtualHost = m.VirtualHost; })); var flow = builder.Add(FlowsHelper.Serialize() .Recover(ex => { Logger.Error(ex, ""); return(Option <ByteString> .None); })); builder.From(start) .Via(flow) .To(sink); return(ClosedShape.Instance); }); SourceActor = Context.Materializer().Materialize(graph); }
OutboundConnectionHandler(StreamsTransport transport, Address remoteAddress, EndPoint remoteSocketAddr) { var settings = transport.Settings; var transmissionHandle = Source.ActorRef <Google.Protobuf.ByteString>(settings.BufferedMessages, OverflowStrategy.Backpressure); var fromLocalActors = FromProtobuf().Via(Encode(4, settings.MaxFrameSize, settings.ByteOrder)); var fromRemoteActors = Decode(4, settings.MaxFrameSize, settings.ByteOrder).Via(ToProtobuf()); var transmissionRef = transmissionHandle.PreMaterialize(transport.StreamMaterializer); var finalOutput = new RemoteOutboundAssociationSink(transport, remoteAddress, remoteSocketAddr, transmissionRef.Item1); var dsl = GraphDsl.Create(finalOutput, (builder, remoteOutput) => { // local stages var localInput = builder.Add(transmissionRef.Item2); var compilerCeremony = builder.Add(Flow.Create <Google.Protobuf.ByteString>()); var local = builder.Add(fromLocalActors); var merge = builder.Add(new Merge <Google.Protobuf.ByteString, Google.Protobuf.ByteString>(2, false)); builder.From(localInput.Outlet).To(merge.In(0)); builder.From(compilerCeremony.Outlet).To(merge.In(1)); builder.From(merge.Out).To(local.Inlet); // remote stages var remote = builder.Add(fromRemoteActors); builder.From(remote.Outlet).To(remoteOutput.Inlet); return(new BidiShape <ByteString, Google.Protobuf.ByteString, Google.Protobuf.ByteString, ByteString>(remote.Inlet, remote.Outlet, compilerCeremony.Inlet, local.Outlet)); }); return(BidiFlow.FromGraph(dsl)); }
static async Task Main(string[] args) { var system = ActorSystem.Create("MySystem"); var streamBits = Source.ActorRef <int>(1000, OverflowStrategy.DropHead) .GroupedWithin(100, TimeSpan.FromMilliseconds(100)) .Scan(new SortedSet <int>(), (set, ints) => { foreach (var i in ints) { set.Add(i); } return(set); }).PreMaterialize(system.Materializer()); var source = streamBits.Item2; var actor = streamBits.Item1; system.Scheduler.ScheduleTellRepeatedly(TimeSpan.FromMilliseconds(10), TimeSpan.FromMilliseconds(1), actor, ThreadLocalRandom.Current.Next(), ActorRefs.NoSender); system.Scheduler.ScheduleTellOnce(TimeSpan.FromMinutes(10), actor, PoisonPill.Instance, ActorRefs.NoSender); // terminate stream after 10 minutes await source.RunForeach(i => { Console.WriteLine("[{0}]", string.Join(",", i)); }, system.Materializer()); await system.Terminate(); }
private void WaitForDelimiter() { var source = Source.ActorRef <ByteString>(1000, OverflowStrategy.Fail); var sink = Sink.ActorRef <string>(Self, new DelimitedStreamComplete()); var parseLogic = Flow.Create <ByteString>() .Via(Framing.Delimiter( _delimiter, maximumFrameLength: 2000)) .Select(bs => bs.ToString(Encoding.ASCII)); var mat = source.Via(parseLogic).To(sink).Run(Context.System.Materializer()); source.RunForeach(x => Console.WriteLine(x), Context.System.Materializer()); Receive <ByteString>(msg => { mat.Tell(msg); }); Receive <string>(msg => { _logger.Info(msg); }); }
public static IRunnableGraph <IActorRef> CreateRunnableGraph() { var tweetSource = Source.ActorRef <ITweet>(100, OverflowStrategy.DropHead); var formatFlow = Flow.Create <ITweet>().Select(Utils.FormatTweet); var writeSink = Sink.ForEach <string>(Console.WriteLine); return(tweetSource.Via(formatFlow).To(writeSink)); }
public DownloadCoordinator(CrawlJob job, IActorRef commander, IActorRef downloadsTracker, long maxConcurrentDownloads) { Job = job; DownloadsTracker = downloadsTracker; MaxConcurrentDownloads = maxConcurrentDownloads; Commander = commander; Stats = new CrawlJobStats(Job); var selfHtmlSink = Sink.ActorRef <CheckDocuments>(Self, StreamCompleteTick.Instance); var selfDocSink = Sink.ActorRef <CompletedDocument>(Self, StreamCompleteTick.Instance); var selfImgSink = Sink.ActorRef <CompletedDocument>(Self, StreamCompleteTick.Instance); var htmlFlow = Flow.Create <CrawlDocument>().Via(DownloadFlow.SelectDocType()) .Throttle(30, TimeSpan.FromSeconds(5), 100, ThrottleMode.Shaping) .Via(DownloadFlow.ProcessHtmlDownloadFor(DefaultMaxConcurrentDownloads, HttpClientFactory.GetClient())); var imageFlow = Flow.Create <CrawlDocument>() .Via(DownloadFlow.SelectDocType()) .Throttle(30, TimeSpan.FromSeconds(1), 100, ThrottleMode.Shaping) .Via(DownloadFlow.ProcessImageDownloadFor(DefaultMaxConcurrentDownloads, HttpClientFactory.GetClient())) .Via(DownloadFlow.ProcessCompletedDownload()); var source = Source.ActorRef <CrawlDocument>(5000, OverflowStrategy.DropTail); var graph = GraphDsl.Create(source, (builder, s) => { // html flows var downloadHtmlFlow = builder.Add(htmlFlow); var downloadBroadcast = builder.Add(new Broadcast <DownloadHtmlResult>(2)); var completedDownload = builder.Add(DownloadFlow.ProcessCompletedHtmlDownload()); var parseCompletedDownload = builder.Add(ParseFlow.GetParseFlow(Job)); var htmlSink = builder.Add(selfHtmlSink); var docSink = builder.Add(selfDocSink); builder.From(downloadHtmlFlow).To(downloadBroadcast); builder.From(downloadBroadcast.Out(0)).To(completedDownload.Inlet); builder.From(downloadBroadcast.Out(1)).To(parseCompletedDownload.Inlet); builder.From(parseCompletedDownload).To(htmlSink); builder.From(completedDownload).To(docSink); // image flows var imgSink = builder.Add(selfImgSink); var downloadImageFlow = builder.Add(imageFlow); builder.From(downloadImageFlow).To(imgSink); var sourceBroadcast = builder.Add(new Broadcast <CrawlDocument>(2)); builder.From(sourceBroadcast.Out(0)).To(downloadImageFlow.Inlet); builder.From(sourceBroadcast.Out(1)).To(downloadHtmlFlow.Inlet); builder.From(s.Outlet).To(sourceBroadcast.In); return(ClosedShape.Instance); }); SourceActor = Context.Materializer().Materialize(graph); Receiving(); }
public void A_ActorRefSource_must_completes_the_stream_immediately_when_receiving_PoisonPill() { this.AssertAllStagesStopped(() => { var s = this.CreateManualSubscriberProbe <int>(); var actorRef = Source.ActorRef <int>(10, OverflowStrategy.Fail) .To(Sink.FromSubscriber(s)) .Run(Materializer); s.ExpectSubscription(); actorRef.Tell(PoisonPill.Instance); s.ExpectComplete(); }, Materializer); }
static void Main(string[] args) { var tweetSource = Source.ActorRef <ITweet>(100, OverflowStrategy.DropHead); var formatFlow = Flow.Create <ITweet>().Select(FormatTweet); var writeSink = Sink.ForEach <string>(Console.WriteLine); var countauthors = Flow.Create <ITweet>() .StatefulSelectMany(() => { var dict = new Dictionary <string, int>(); Func <ITweet, IEnumerable <string> > result = (tweet => { var user = tweet.CreatedBy.Name; if (!dict.ContainsKey(user)) { dict.Add(user, 1); } return(new[] { $"{dict[user]++} tweet from {user}\n" }); }); return(result); }); var graph = GraphDsl.Create(countauthors, writeSink, (notUsed, _) => notUsed, (b, count, write) => { var broadcast = b.Add(new Broadcast <ITweet>(2)); var output = b.From(broadcast.Out(0)).Via(formatFlow); b.From(broadcast.Out(1)).Via(count).To(write); return(new FlowShape <ITweet, string>(broadcast.In, output.Out)); }); using (var sys = ActorSystem.Create("Reactive-Tweets")) { using (var mat = sys.Materializer()) { // Start Akka.Net stream var actor = tweetSource.Via(graph).To(writeSink).Run(mat); // Start Twitter stream Auth.SetCredentials(new TwitterCredentials(ConsumerKey, ConsumerSecret, AccessToken, AccessTokenSecret)); var stream = Stream.CreateFilteredStream(); stream.AddLocation(CenterOfNewYork); stream.MatchingTweetReceived += (_, arg) => actor.Tell(arg.Tweet); // push the tweets into the stream stream.StartStreamMatchingAllConditions(); Console.ReadLine(); } } }
static void Main(string[] args) { using (var sys = ActorSystem.Create("Reactive-Tweets")) { var consumerKey = ConfigurationManager.AppSettings["ConsumerKey"]; var consumerSecret = ConfigurationManager.AppSettings["ConsumerSecret"]; var accessToken = ConfigurationManager.AppSettings["AccessToken"]; var accessTokenSecret = ConfigurationManager.AppSettings["AccessTokenSecret"]; Console.OutputEncoding = System.Text.Encoding.UTF8; Console.ForegroundColor = ConsoleColor.Cyan; Console.WriteLine("Press Enter to Start"); Console.ReadLine(); var useCachedTweets = true; using (var mat = sys.Materializer()) { Auth.SetCredentials(new TwitterCredentials(consumerKey, consumerSecret, accessToken, accessTokenSecret)); if (useCachedTweets) { var tweetSource = Source.FromEnumerator(() => new TweetEnumerator(true)); var graph = CreateRunnableGraph(tweetSource); graph.Run(mat); } else { var tweetSource = Source.ActorRef <ITweet>(100, OverflowStrategy.Backpressure); var graph = CreateRunnableGraph(tweetSource); var actor = graph.Run(mat); Utils.StartSampleTweetStream(actor); } Console.WriteLine("Press Enter to exit"); TweetsToEmotion.form?.ShowDialog(); Console.ReadLine(); } } IRunnableGraph <TMat> CreateRunnableGraph <TMat>(Source <ITweet, TMat> tweetSource) => //TweetsToConsole.CreateRunnableGraph(tweetSource); //TweetsWithBroadcast.CreateRunnableGraph(tweetSource); // TweetsWithThrottle.CreateRunnableWethaerGraph(tweetSource); //TweetsWithThrottle.CreateRunnableGraph(tweetSource); // TweetsWithWeather.CreateRunnableGraph(tweetSource); TweetsToEmotion.CreateRunnableGraph(tweetSource); }
public void A_ActorRefSource_must_terminate_when_the_stream_is_cancelled() { this.AssertAllStagesStopped(() => { var s = this.CreateManualSubscriberProbe <int>(); var actorRef = Source.ActorRef <int>(0, OverflowStrategy.Fail) .To(Sink.FromSubscriber(s)) .Run(Materializer); Watch(actorRef); var sub = s.ExpectSubscription(); sub.Cancel(); ExpectTerminated(actorRef); }, Materializer); }
public void A_ActorRefSource_must_set_actor_name_equal_to_stage_name() { this.AssertAllStagesStopped(() => { var s = this.CreateManualSubscriberProbe <int>(); const string name = "SomeCustomName"; var actorRef = Source.ActorRef <int>(10, OverflowStrategy.Fail) .WithAttributes(Attributes.CreateName(name)) .To(Sink.FromSubscriber(s)) .Run(Materializer); actorRef.Path.ToString().Should().Contain(name); actorRef.Tell(PoisonPill.Instance); }, Materializer); }
public void A_ActorRefSource_must_fail_the_stream_when_receiving_Status_Failure() { this.AssertAllStagesStopped(() => { var s = this.CreateManualSubscriberProbe <int>(); var actorRef = Source.ActorRef <int>(10, OverflowStrategy.Fail) .To(Sink.FromSubscriber(s)) .Run(Materializer); s.ExpectSubscription(); var ex = new TestException("testfailure"); actorRef.Tell(new Status.Failure(ex)); s.ExpectError().Should().Be(ex); }, Materializer); }
public void A_ActorRefSource_must_not_fail_when_0_buffer_space_and_demand_is_signalled() { this.AssertAllStagesStopped(() => { var s = this.CreateManualSubscriberProbe <int>(); var actorRef = Source.ActorRef <int>(0, OverflowStrategy.DropHead) .To(Sink.FromSubscriber(s)) .Run(Materializer); Watch(actorRef); var sub = s.ExpectSubscription(); sub.Request(100); sub.Cancel(); ExpectTerminated(actorRef); }, Materializer); }
public void A_ActorRefSource_must_emit_received_messages_to_the_stream() { var s = this.CreateManualSubscriberProbe <int>(); var actorRef = Source.ActorRef <int>(10, OverflowStrategy.Fail) .To(Sink.FromSubscriber(s)) .Run(Materializer); var sub = s.ExpectSubscription(); sub.Request(2); actorRef.Tell(1); s.ExpectNext(1); actorRef.Tell(2); s.ExpectNext(2); actorRef.Tell(3); s.ExpectNoMsg(TimeSpan.FromMilliseconds(500)); }
public void A_ActorRefSource_must_drop_new_when_full_and_with_DropNew_strategy() { var t = Source.ActorRef <int>(100, OverflowStrategy.DropNew) .ToMaterialized(this.SinkProbe <int>(), Keep.Both) .Run(Materializer); var actorRef = t.Item1; var sub = t.Item2; Enumerable.Range(1, 20).ForEach(x => actorRef.Tell(x)); sub.Request(10); Enumerable.Range(1, 10).ForEach(x => sub.ExpectNext(x)); sub.Request(10); Enumerable.Range(11, 10).ForEach(x => sub.ExpectNext(x)); Enumerable.Range(200, 200).ForEach(x => actorRef.Tell(x)); sub.Request(100); Enumerable.Range(200, 100).ForEach(x => sub.ExpectNext(x)); }
public void A_ActorRefSource_must_signal_buffered_elements_and_complete_the_stream_after_receiving_Status_Success() { this.AssertAllStagesStopped(() => { var s = this.CreateManualSubscriberProbe <int>(); var actorRef = Source.ActorRef <int>(10, OverflowStrategy.Fail) .To(Sink.FromSubscriber(s)) .Run(Materializer); var sub = s.ExpectSubscription(); actorRef.Tell(1); actorRef.Tell(2); actorRef.Tell(3); actorRef.Tell(new Status.Success("ok")); sub.Request(10); s.ExpectNext(1, 2, 3); s.ExpectComplete(); }, Materializer); }
public void A_ActorRefSource_must_buffer_when_needed() { var s = this.CreateManualSubscriberProbe <int>(); var actorRef = Source.ActorRef <int>(100, OverflowStrategy.DropHead) .To(Sink.FromSubscriber(s)) .Run(Materializer); var sub = s.ExpectSubscription(); Enumerable.Range(1, 20).ForEach(x => actorRef.Tell(x)); sub.Request(10); Enumerable.Range(1, 10).ForEach(x => s.ExpectNext(x)); sub.Request(10); Enumerable.Range(11, 10).ForEach(x => s.ExpectNext(x)); Enumerable.Range(200, 200).ForEach(x => actorRef.Tell(x)); sub.Request(100); Enumerable.Range(300, 100).ForEach(x => s.ExpectNext(x)); }
public void A_ActorRefSource_must_after_receiving_Status_Success_allow_for_earlier_completion_with_PoisonPill() { this.AssertAllStagesStopped(() => { var s = this.CreateManualSubscriberProbe <int>(); var actorRef = Source.ActorRef <int>(3, OverflowStrategy.DropBuffer) .To(Sink.FromSubscriber(s)) .Run(Materializer); var sub = s.ExpectSubscription(); actorRef.Tell(1); actorRef.Tell(2); actorRef.Tell(3); actorRef.Tell(new Status.Success("ok")); sub.Request(2); // not all elements drained yet s.ExpectNext(1, 2); actorRef.Tell(PoisonPill.Instance); s.ExpectComplete(); // element `3` not signaled }, Materializer); }
public void Source_prematerialization() { #region source-prematerialization var matPoweredSource = Source.ActorRef <string>(bufferSize: 100, overflowStrategy: OverflowStrategy.Fail); (IActorRef, Source <string, NotUsed>)materialized = matPoweredSource.PreMaterialize(Sys.Materializer()); var actorRef = materialized.Item1; var source = materialized.Item2; actorRef.Tell("hit"); // pass source around for materialization source.RunWith(Sink.ForEach <string>(Console.WriteLine), Sys.Materializer()); #endregion }
public static void RunGraph() { using (var sys = ActorSystem.Create("Reactive-Tweets")) { var consumerKey = ConfigurationManager.AppSettings["ConsumerKey"]; var consumerSecret = ConfigurationManager.AppSettings["ConsumerSecret"]; var accessToken = ConfigurationManager.AppSettings["AccessToken"]; var accessTokenSecret = ConfigurationManager.AppSettings["AccessTokenSecret"]; Console.OutputEncoding = System.Text.Encoding.UTF8; Console.ForegroundColor = ConsoleColor.Cyan; Console.WriteLine("Press Enter to Start"); Console.ReadLine(); var useCachedTweets = true; using (var mat = sys.Materializer()) { if (useCachedTweets) { var tweetSource = Source.FromEnumerator(() => new TweetEnumerator(true)); var graph = CreateRunnableGraph(tweetSource); graph.Run(mat); } else { Auth.SetCredentials(new TwitterCredentials(consumerKey, consumerSecret, accessToken, accessTokenSecret)); var tweetSource = Source.ActorRef <ITweet>(100, OverflowStrategy.Backpressure); var graph = CreateRunnableGraph(tweetSource); var actor = graph.Run(mat); Utils.StartSampleTweetStream(actor); } Console.WriteLine("Press Enter to exit"); Console.ReadLine(); } } }
public void A_ActorRefSource_must_not_buffer_elements_after_receiving_Status_Success() { this.AssertAllStagesStopped(() => { var s = TestSubscriber.CreateManualProbe <int>(this); var actorRef = Source.ActorRef <int>(3, OverflowStrategy.DropBuffer) .To(Sink.FromSubscriber(s)) .Run(Materializer); var sub = s.ExpectSubscription(); actorRef.Tell(1); actorRef.Tell(2); actorRef.Tell(3); actorRef.Tell(new Status.Success("ok")); actorRef.Tell(100); actorRef.Tell(100); actorRef.Tell(100); sub.Request(10); s.ExpectNext(1, 2, 3); s.ExpectComplete(); }, Materializer); }
private void Handle(Setup m) { Become(Ready); Stash.UnstashAll(); var source = Source.ActorRef <object>(5000, OverflowStrategy.DropTail); var sinkActor = Sink.ActorRef <(object, ICommitable)>(Self, new CompleteMessage()); var graph = GraphDsl.Create(source, (builder, start) => { var serializeFlow = builder.Add(FlowsHelper.Serialize() .Recover(ex => { Logger.Error(ex, ""); return(Option <ByteString> .None); }) .Select(x => new OutgoingMessage(x, true, true))); var rpcFlow = builder.Add(FlowsHelper.SimpleRpc(option => { option.HostAndPorts = m.HostAndPorts; option.UserName = m.UserName; option.Password = m.Password; option.QueueName = m.QueueName; option.VirtualHost = m.VirtualHost; })); builder.From(start) .Via(serializeFlow) .Via(rpcFlow) .Via(FlowsHelper.Deserialize()) .To(sinkActor); return(ClosedShape.Instance); }); SourceActor = Context.Materializer().Materialize(graph); }
public void PublishSubscribeOnHubsAddsAndRemovesPublishersAndSubscribers() { const int publisherMaxCount = 16; const int subscriberMaxCount = 16; const int bufferSize = 4; // Source ToMat Bidi // +------------+ +------------+ // | MergeHub | |BroadcastHub| // | Source | ~> Message ~> | Sink | // | | | | // +------------+ +------------+ // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ // ~~ (Sink<Message>, Source<Message>) ~~ // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ (Sink <string, NotUsed> mergeSink, Source <string, NotUsed> mergeSource) = MergeHub.Source <string>(perProducerBufferSize: publisherMaxCount) .ToMaterialized(BroadcastHub.Sink <string>(bufferSize: subscriberMaxCount), Keep.Both) .Run(_materializer); TestProbe sub0 = CreateTestProbe(); TestProbe sub1 = CreateTestProbe(); // Flow JoinMat Bidi // +------------+ +------------+ // | FromSink | ~> Message ~> |KillSwitches| ~> Message // | And | | Single | // | Source | <~ Message <~ | Bidi | <~ Message // +------------+ +------------+ // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ // ~~ UniqueKillSwitch ~~ // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Flow <string, string, UniqueKillSwitch> busFlow = Flow.FromSinkAndSource(mergeSink, mergeSource) .JoinMaterialized(KillSwitches.SingleBidi <string, string>(), Keep.Right); var(pub0, uniqueKillSwitch0) = Source.ActorRef <string>(bufferSize, OverflowStrategy.Fail) .ViaMaterialized(busFlow, Keep.Both) .To(Sink.ActorRef <string>(sub0, "complete")) .Run(_materializer); pub0.Tell("It's chat member 0!"); sub0.ExpectMsg("It's chat member 0!"); // Echo. sub0.ExpectNoMsg(TimeSpan.FromMilliseconds(50)); var(pub1, uniqueKillSwitch1) = Source.ActorRef <string>(bufferSize, OverflowStrategy.Fail) .ViaMaterialized(busFlow, Keep.Both) .To(Sink.ActorRef <string>(sub1, "complete")) .Run(_materializer); pub1.Tell("Hi! It's chat member 1!"); sub1.ExpectMsg("Hi! It's chat member 1!"); // Echo. sub0.ExpectMsg("Hi! It's chat member 1!"); pub0.Tell("Oh, Hi! Sry, but I gotta go, bye!"); sub0.ExpectMsg("Oh, Hi! Sry, but I gotta go, bye!"); // Echo. uniqueKillSwitch0.Shutdown(); // Looks like this Shutdown is non-blocking. sub0.ExpectMsg("complete", TimeSpan.FromMilliseconds(1000)); // Wait for the running graph to stop. sub1.ExpectMsg("Oh, Hi! Sry, but I gotta go, bye!"); pub1.Tell("Oh, looks like I stayed alone."); sub1.ExpectMsg("Oh, looks like I stayed alone."); // Echo. sub0.ExpectNoMsg(); }