// Gzip encoding or output framing don't apply to this method.
public async Task <IReadOnlyList <string> > GetText(string id, Stream stream, ModuleLogFilter filter)
{
Preconditions.CheckNotNull(stream, nameof(stream));
Preconditions.CheckNotNull(filter, nameof(filter));
Preconditions.CheckNonWhiteSpace(id, nameof(id));
IRunnableGraph <Task <IImmutableList <string> > > GetGraph()
{
if (filter.Regex.HasValue || filter.LogLevel.HasValue)
{
GraphBuilder graphBuilder = GraphBuilder.CreateParsingGraphBuilder(stream, b => this.logMessageParser.Parse(b, id));
filter.LogLevel.ForEach(l => graphBuilder.AddFilter(m => m.LogLevel == l));
filter.Regex.ForEach(r => graphBuilder.AddFilter(m => r.IsMatch(m.Text)));
return(graphBuilder.GetMaterializingGraph(m => m.FullText));
}
else
{
return(GraphBuilder.BuildMaterializedGraph(stream));
}
}
IRunnableGraph <Task <IImmutableList <string> > > graph = GetGraph();
IImmutableList <string> result = await graph.Run(this.materializer);
return(filter.Tail.Match <IReadOnlyList <string> >(
t =>
{
return result.Skip(Math.Max(0, result.Count - t)).ToList().AsReadOnly();
},
() => result));
}
public void Hubs_must_demonstrate_creating_a_dynamic_partition_hub()
{
#region partition-hub
// A simple producer that publishes a new "message-" every second
Source <string, NotUsed> producer = Source.Tick(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(1), "message")
.MapMaterializedValue(_ => NotUsed.Instance)
.ZipWith(Source.From(Enumerable.Range(1, 100)), (msg, i) => $"{msg}-{i}");
// Attach a PartitionHub Sink to the producer. This will materialize to a
// corresponding Source.
// (We need to use toMat and Keep.right since by default the materialized
// value to the left is used)
IRunnableGraph <Source <string, NotUsed> > runnableGraph =
producer.ToMaterialized(PartitionHub.Sink <string>(
(size, element) => Math.Abs(element.GetHashCode()) % size,
startAfterNrOfConsumers: 2, bufferSize: 256), Keep.Right);
// By running/materializing the producer, we get back a Source, which
// gives us access to the elements published by the producer.
Source <string, NotUsed> fromProducer = runnableGraph.Run(Materializer);
// Print out messages from the producer in two independent consumers
fromProducer.RunForeach(msg => Console.WriteLine("Consumer1: " + msg), Materializer);
fromProducer.RunForeach(msg => Console.WriteLine("Consumer2: " + msg), Materializer);
#endregion
}
public async Task ProcessLogsStream(string id, Stream stream, ModuleLogOptions logOptions, Func <ArraySegment <byte>, Task> callback)
{
GraphBuilder graphBuilder = GraphBuilder.CreateParsingGraphBuilder(stream, b => this.logMessageParser.Parse(b, id));
logOptions.Filter.LogLevel.ForEach(l => graphBuilder.AddFilter(m => m.LogLevel == l));
logOptions.Filter.Regex.ForEach(r => graphBuilder.AddFilter(m => r.IsMatch(m.Text)));
async Task <bool> ConsumerCallback(ArraySegment <byte> a)
{
await callback(a);
return(true);
}
ArraySegment <byte> BasicMapper(ModuleLogMessageData l)
=> logOptions.ContentType == LogsContentType.Text
? new ArraySegment <byte>(l.FullFrame.ToArray())
: new ArraySegment <byte>(l.ToBytes());
var mappers = new List <Func <ArraySegment <byte>, ArraySegment <byte> > >();
if (logOptions.ContentEncoding == LogsContentEncoding.Gzip)
{
mappers.Add(m => new ArraySegment <byte>(Compression.CompressToGzip(m.Array)));
}
IRunnableGraph <Task> graph = graphBuilder.GetStreamingGraph(
ConsumerCallback,
BasicMapper,
mappers);
await graph.Run(this.materializer);
}
public void Hubs_must_demonstrate_creating_a_dynamic_partition_hub_routing_to_fastest_consumer()
{
#region partition-hub-fastest
// A simple producer that publishes a new "message-" every second
Source <int, NotUsed> producer = Source.From(Enumerable.Range(0, 100));
// Attach a PartitionHub Sink to the producer. This will materialize to a
// corresponding Source.
// (We need to use toMat and Keep.right since by default the materialized
// value to the left is used)
IRunnableGraph <Source <int, NotUsed> > runnableGraph =
producer.ToMaterialized(PartitionHub.StatefulSink <int>(
() => ((info, element) => info.ConsumerIds.Min(info.QueueSize)),
startAfterNrOfConsumers: 2, bufferSize: 256), Keep.Right);
// By running/materializing the producer, we get back a Source, which
// gives us access to the elements published by the producer.
Source <int, NotUsed> fromProducer = runnableGraph.Run(Materializer);
// Print out messages from the producer in two independent consumers
fromProducer.RunForeach(msg => Console.WriteLine("Consumer1: " + msg), Materializer);
fromProducer.Throttle(10, TimeSpan.FromMilliseconds(100), 10, ThrottleMode.Shaping)
.RunForeach(msg => Console.WriteLine("Consumer2: " + msg), Materializer);
#endregion
}
public void Hubs_must_demonstrate_creating_a_dynamic_merge()
{
void WriteLine(string s) => TestActor.Tell(s);
#region merge-hub
// A simple consumer that will print to the console for now
Sink <string, Task> consumer = Sink.ForEach <string>(WriteLine);
// Attach a MergeHub Source to the consumer. This will materialize to a
// corresponding Sink.
IRunnableGraph <Sink <string, NotUsed> > runnableGraph =
MergeHub.Source <string>(perProducerBufferSize: 16).To(consumer);
// By running/materializing the consumer we get back a Sink, and hence
// now have access to feed elements into it. This Sink can be materialized
// any number of times, and every element that enters the Sink will
// be consumed by our consumer.
Sink <string, NotUsed> toConsumer = runnableGraph.Run(Materializer);
// Feeding two independent sources into the hub.
Source.Single("Hello!").RunWith(toConsumer, Materializer);
Source.Single("Hub!").RunWith(toConsumer, Materializer);
#endregion
ExpectMsgAllOf("Hello!", "Hub!");
}
public void Setup()
{
system = ActorSystem.Create("system");
materializer = system.Materializer();
simpleGraph = Source.Single(1)
.Select(x => x + 1)
.ToMaterialized(Sink.ForEach <int>(i => { }), Keep.Right);
}
public static IRunnableGraph <Task <IImmutableList <string> > > BuildMaterializedGraph(Stream stream)
{
var source = StreamConverters.FromInputStream(() => stream);
var seqSink = Sink.Seq <string>();
IRunnableGraph <Task <IImmutableList <string> > > graph = source
.Via(FramingFlow)
.Select(b => b.Slice(8))
.Select(b => b.ToString(Encoding.UTF8))
.ToMaterialized(seqSink, Keep.Right);
return(graph);
}
public async Task <IReadOnlyList <string> > GetText(Stream stream)
{
Preconditions.CheckNotNull(stream, nameof(stream));
var source = StreamConverters.FromInputStream(() => stream);
var seqSink = Sink.Seq <string>();
IRunnableGraph <Task <IImmutableList <string> > > graph = source
.Via(FramingFlow)
.Select(b => b.Slice(8))
.Select(b => b.ToString(Encoding.UTF8))
.ToMaterialized(seqSink, Keep.Right);
IImmutableList <string> result = await graph.Run(this.materializer);
return(result);
}
public async Task <IReadOnlyList <ModuleLogMessage> > GetMessages(Stream stream, string moduleId)
{
Preconditions.CheckNotNull(stream, nameof(stream));
Preconditions.CheckNonWhiteSpace(moduleId, nameof(moduleId));
var source = StreamConverters.FromInputStream(() => stream);
var seqSink = Sink.Seq <ModuleLogMessage>();
IRunnableGraph <Task <IImmutableList <ModuleLogMessage> > > graph = source
.Via(FramingFlow)
.Select(b => this.logMessageParser.Parse(b, moduleId))
.ToMaterialized(seqSink, Keep.Right);
IImmutableList <ModuleLogMessage> result = await graph.Run(this.materializer);
return(result);
}
// Gzip encoding or output framing don't apply to this method.
public async Task <IReadOnlyList <ModuleLogMessage> > GetMessages(string id, Stream stream, ModuleLogFilter filter)
{
Preconditions.CheckNotNull(stream, nameof(stream));
Preconditions.CheckNotNull(filter, nameof(filter));
Preconditions.CheckNonWhiteSpace(id, nameof(id));
GraphBuilder graphBuilder = GraphBuilder.CreateParsingGraphBuilder(stream, b => this.logMessageParser.Parse(b, id));
filter.LogLevel.ForEach(l => graphBuilder.AddFilter(m => m.LogLevel == l));
filter.Regex.ForEach(r => graphBuilder.AddFilter(m => r.IsMatch(m.Text)));
IRunnableGraph <Task <IImmutableList <ModuleLogMessage> > > graph = graphBuilder.GetMaterializingGraph(m => (ModuleLogMessage)m);
IImmutableList <ModuleLogMessage> result = await graph.Run(this.materializer);
return(result);
}
static Task <int> Stream()
{
// 1秒ごとに下流に各要素を放出する Source
Source <int, NotUsed> source = Source.From(Enumerable.Range(1, 100));
// 上流から流れてきた要素を足し合わせる Sink
Sink <int, Task <int> > sink = Sink.Aggregate <int, int>(0, (l, r) => l + r);
// Stream を正常（もしくはキャンセル扱いで）に停止させるための KillSwitch
Flow <int, int, UniqueKillSwitch> killSwitch = Flow.Create <int>().ViaMaterialized(KillSwitches.Single <int>(), Keep.Right);
// Stream（の特定の部分）を通過する要素の流量を制御するための Throttle
Flow <int, int, NotUsed> throttle = Flow.Create <int>().Throttle(1, TimeSpan.FromSeconds(1), 1, ThrottleMode.Shaping);
// Stream を動作させる Actor をホストする ActorSystem
ActorSystem system = ActorSystem.Create("akkanet");
// ActorSystem を使用して Stream をマテリアル化するマテリアライザ
ActorMaterializer materializer = ActorMaterializer.Create(system);
// Source、Sink、Throttle、KillSwitch を使用して RunnableGraph（実行可能なグラフ）を組み立てる
IRunnableGraph <Tuple <UniqueKillSwitch, Task <int> > > runnable =
source
.Via(throttle)
.ViaMaterialized(killSwitch, Keep.Right)
.ToMaterialized(sink, Keep.Both);
// RunnableGraph をマテリアライズして Stream を作動させる
var(ks, mat) = runnable.Run(materializer);
// 10秒後に KillSwitch を使用して Stream を途中で停止させる（完了扱い）
ICancelable canceller = materializer.ScheduleOnce(TimeSpan.FromSeconds(10), () =>
{
Console.WriteLine("Stream is cancelled");
ks.Shutdown();
});
// Stream 完了後に ActorSystem と ActorMaterializer を破棄する
return(mat.ContinueWith(prev =>
{
canceller.Cancel();
materializer.Dispose();
system.Dispose();
return prev.Result;
}));
}
public void Setup(BenchmarkContext context)
{
_actorSystem = ActorSystem.Create("MergeManyBenchmark",
ConfigurationFactory.FromResource<ScriptedTest>("Akka.Streams.TestKit.Tests.reference.conf"));
_actorSystem.Settings.InjectTopLevelFallback(ActorMaterializer.DefaultConfig());
_materializerSettings = ActorMaterializerSettings.Create(_actorSystem).WithDispatcher("akka.test.stream-dispatcher");
_materializer = _actorSystem.Materializer(_materializerSettings);
var takeSource = CreateSource(NumberOfElements);
var singleSubSource = CreateSource(NumberOfElements);
var singleSource = Source.Repeat(0).Take(1).MergeMany(1, _ => singleSubSource);
var tenSubSources = CreateSource(NumberOfElements/10);
var tenSources = Source.Repeat(0).Take(10).MergeMany(10, _ => tenSubSources);
_takeGraph = ToSource(takeSource);
_singleGraph = ToSource(singleSource);
_tenGraph = ToSource(tenSources);
}
public void Setup(BenchmarkContext context)
{
_actorSystem = ActorSystem.Create("MergeManyBenchmark",
ConfigurationFactory.FromResource <ScriptedTest>("Akka.Streams.TestKit.Tests.reference.conf"));
_actorSystem.Settings.InjectTopLevelFallback(ActorMaterializer.DefaultConfig());
_materializerSettings = ActorMaterializerSettings.Create(_actorSystem).WithDispatcher("akka.test.stream-dispatcher");
_materializer = _actorSystem.Materializer(_materializerSettings);
var takeSource = CreateSource(NumberOfElements);
var singleSubSource = CreateSource(NumberOfElements);
var singleSource = Source.Repeat(0).Take(1).MergeMany(1, _ => singleSubSource);
var tenSubSources = CreateSource(NumberOfElements / 10);
var tenSources = Source.Repeat(0).Take(10).MergeMany(10, _ => tenSubSources);
_takeGraph = ToSource(takeSource);
_singleGraph = ToSource(singleSource);
_tenGraph = ToSource(tenSources);
}
public void Hubs_must_demonstrate_creating_a_dynamic_broadcast()
{
#region broadcast-hub
Source <string, ICancelable> producer = Source.Tick(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(1), "New message");
// Attach a BroadcastHub Sink to the producer. This will materialize to a
// corresponding Source.
// (We need to use ToMaterialized and Keep.Right since by default the materialized
// value to the left is used)
IRunnableGraph <Source <string, NotUsed> > runnableGraph =
producer.ToMaterialized(BroadcastHub.Sink <string>(bufferSize: 256), Keep.Right);
// By running/materializing the producer, we get back a Source, which
// gives us access to the elements published by the producer.
Source <string, NotUsed> fromProducer = runnableGraph.Run(Materializer);
// Print out messages from the producer in two independent consumers
fromProducer.RunForeach(msg => Console.WriteLine($"consumer1:{msg}"), Materializer);
fromProducer.RunForeach(msg => Console.WriteLine($"consumer2:{msg}"), Materializer);
#endregion
}
public async Task ProcessLogsStream(string id, Stream stream, ModuleLogOptions logOptions, Func <ArraySegment <byte>, Task> callback)
{
GraphBuilder graphBuilder = GraphBuilder.CreateParsingGraphBuilder(stream, b => this.logMessageParser.Parse(b, id));
logOptions.Filter.LogLevel.ForEach(l => graphBuilder.AddFilter(m => m.LogLevel == l));
logOptions.Filter.Regex.ForEach(r => graphBuilder.AddFilter(m => r.IsMatch(m.Text)));
async Task <bool> ConsumerCallback(ArraySegment <byte> a)
{
await callback(a);
return(true);
}
ArraySegment <byte> BasicMapper(ModuleLogMessageData l)
=> logOptions.ContentType == LogsContentType.Text
? new ArraySegment <byte>(l.FullText.ToBytes())
: new ArraySegment <byte>(l.ToBytes());
var sourceMappers = new List <Func <Source <ArraySegment <byte>, AkkaNet.NotUsed>, Source <ArraySegment <byte>, AkkaNet.NotUsed> > >();
if (logOptions.ContentEncoding == LogsContentEncoding.Gzip)
{
sourceMappers.Add(
s => logOptions.OutputGroupingConfig.Map(o => GroupingGzipMapper(s, o))
.GetOrElse(() => NonGroupingGzipMapper(s)));
}
if (logOptions.OutputFraming == LogOutputFraming.SimpleLength)
{
sourceMappers.Add(SimpleLengthFramingMapper);
}
IRunnableGraph <Task> graph = graphBuilder.GetStreamingGraph(
ConsumerCallback,
BasicMapper,
sourceMappers);
await graph.Run(this.materializer);
}
public void Hubs_must_demonstrate_creating_a_dynamic_steful_partition_hub()
{
#region partition-hub-stateful
// A simple producer that publishes a new "message-" every second
Source <string, NotUsed> producer = Source.Tick(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(1), "message")
.MapMaterializedValue(_ => NotUsed.Instance)
.ZipWith(Source.From(Enumerable.Range(1, 100)), (msg, i) => $"{msg}-{i}");
// New instance of the partitioner function and its state is created
// for each materialization of the PartitionHub.
Func <PartitionHub.IConsumerInfo, string, long> RoundRobbin()
{
var i = -1L;
return((info, element) =>
{
i++;
return info.ConsumerByIndex((int)(i % info.Size));
});
}
// Attach a PartitionHub Sink to the producer. This will materialize to a
// corresponding Source.
// (We need to use toMat and Keep.right since by default the materialized
// value to the left is used)
IRunnableGraph <Source <string, NotUsed> > runnableGraph =
producer.ToMaterialized(PartitionHub.StatefulSink(RoundRobbin,
startAfterNrOfConsumers: 2, bufferSize: 256), Keep.Right);
// By running/materializing the producer, we get back a Source, which
// gives us access to the elements published by the producer.
Source <string, NotUsed> fromProducer = runnableGraph.Run(Materializer);
// Print out messages from the producer in two independent consumers
fromProducer.RunForeach(msg => Console.WriteLine("Consumer1: " + msg), Materializer);
fromProducer.RunForeach(msg => Console.WriteLine("Consumer2: " + msg), Materializer);
#endregion
}
/// <summary>
/// Creates a new instance of the <see cref="MultiProcessorFactory"/>
/// </summary>
/// <param name="graph">The graph that should be run</param>
/// <param name="materializer">The materializer</param>
public MultiProcessorFactory(IRunnableGraph <IEventProcessor> graph, IMaterializer materializer)
{
_graph = graph;
_materializer = materializer;
}
private static RunnableGraph <TestLatch> Fuse(IRunnableGraph <TestLatch> graph) => RunnableGraph.FromGraph(Fusing.Aggressive(graph));
public sealed record ChangeTrackeState(ActorMaterializer Materializer, ISourceQueueWithComplete <AppInfo> AppInfos, IRunnableGraph <Source <AppInfo, NotUsed> > Hub);
