/// <summary>
/// Initializes a new instance of the <see cref="ParallelFixedLength{TIn, TOut}"/> class.
/// </summary>
/// <param name="pipeline">The pipeline to add the component to.</param>
/// <param name="vectorSize">Vector size.</param>
/// <param name="transform">Function mapping keyed input producers to output producers.</param>
/// <param name="outputDefaultIfDropped">When true, a result is produced even if a message is dropped in processing one of the input elements. In this case the corresponding output element is set to a default value.</param>
/// <param name="defaultValue">Default value to use when messages are dropped in processing one of the input elements.</param>
/// <param name="name">Name for this component (defaults to ParallelFixedLength).</param>
/// <param name="defaultDeliveryPolicy">Pipeline-level default delivery policy to be used by this component (defaults to <see cref="DeliveryPolicy.Unlimited"/> if unspecified).</param>
public ParallelFixedLength(Pipeline pipeline, int vectorSize, Func <int, IProducer <TIn>, IProducer <TOut> > transform, bool outputDefaultIfDropped, TOut defaultValue = default, string name = null, DeliveryPolicy defaultDeliveryPolicy = null)
: base(pipeline, name ?? nameof(ParallelFixedLength <TIn, TOut>), defaultDeliveryPolicy)
{
this.inConnector = this.CreateInputConnectorFrom <TIn[]>(pipeline, nameof(this.inConnector));
this.splitter = this.CreateReceiver <TIn[]>(this, this.Receive, nameof(this.splitter));
this.inConnector.PipeTo(this.splitter);
this.branches = new Emitter <TIn> [vectorSize];
var branchResults = new IProducer <TOut> [vectorSize];
for (int i = 0; i < vectorSize; i++)
{
var subpipeline = Subpipeline.Create(this, $"subpipeline{i}");
var connectorIn = new Connector <TIn>(this, subpipeline, $"connectorIn{i}");
var connectorOut = new Connector <TOut>(subpipeline, this, $"connectorOut{i}");
this.branches[i] = this.CreateEmitter <TIn>(this, $"branch{i}");
this.branches[i].PipeTo(connectorIn);
transform(i, connectorIn.Out).PipeTo(connectorOut.In);
branchResults[i] = connectorOut;
}
var interpolator = outputDefaultIfDropped ? Reproducible.ExactOrDefault <TOut>(defaultValue) : Reproducible.Exact <TOut>();
this.join = Operators.Join(branchResults, interpolator);
this.outConnector = this.CreateOutputConnectorTo <TOut[]>(pipeline, nameof(this.outConnector));
this.join.PipeTo(this.outConnector);
}
public void ErrorHandlingWithSubpipelineTest()
{
using (var p = Pipeline.Create("root"))
{
using (var s = Subpipeline.Create(p, "sub"))
{
var caughtException = false;
try
{
Generators.Return(p, 1).Select(i => i / 0);
p.Run(enableExceptionHandling: true);
}
catch (AggregateException exception)
{
caughtException = true;
if (exception.InnerException == null)
{
Assert.Fail("AggregateException contains no inner exception");
}
else
{
Assert.IsInstanceOfType(exception.InnerException, typeof(DivideByZeroException), "Unexpected inner exception type: {0}", exception.InnerException.GetType().ToString());
}
}
Assert.IsTrue(caughtException);
}
}
}
private void Receive(TIn[] message, Envelope e)
{
for (int i = 0; i < message.Length; i++)
{
if (this.branches.Count == i)
{
var subpipeline = Subpipeline.Create(this.pipeline, $"subpipeline{i}");
var branch = this.pipeline.CreateEmitter <TIn>(this, $"branch{i}");
var connectorIn = new Connector <TIn>(this.pipeline, subpipeline, $"connectorIn{i}");
branch.PipeTo(connectorIn, true); // allows connections in running pipelines
this.branches.Add(branch);
if (this.parallelTransform != null)
{
var branchResult = this.parallelTransform(i, connectorIn.Out);
var connectorOut = new Connector <TOut>(subpipeline, this.pipeline, $"connectorOut{i}");
branchResult.PipeTo(connectorOut.In, true);
connectorOut.Out.PipeTo(this.join.AddInput(), true);
}
else
{
this.parallelAction(i, connectorIn.Out);
}
subpipeline.RunAsync(this.pipeline.ReplayDescriptor);
}
this.branches[i].Post(message[i], e.OriginatingTime);
}
this.activeBranchesEmitter?.Post(message.Length, e.OriginatingTime);
}
private void Receive(TIn[] message, Envelope e)
{
for (int i = 0; i < message.Length; i++)
{
if (this.branches.Count == i)
{
var subpipeline = Subpipeline.Create(this.pipeline, $"subpipeline{i}");
var branch = subpipeline.CreateEmitter <TIn>(subpipeline, $"branch{i}");
this.branches.Add(branch);
if (this.parallelTransform != null)
{
var branchResult = this.parallelTransform(i, branch);
branchResult.PipeTo(this.join.AddInput());
}
else
{
this.parallelAction(i, branch);
}
subpipeline.RunAsync(this.pipeline.ReplayDescriptor);
}
this.branches[i].Post(message[i], e.OriginatingTime);
}
this.activeBranchesEmitter?.Post(message.Length, e.OriginatingTime);
}
private void Receive(Dictionary <TKey, TIn> message, Envelope e)
{
this.activeBranches.Clear();
foreach (var pair in message)
{
if (!this.branches.ContainsKey(pair.Key))
{
this.keyToBranchMapping[pair.Key] = this.branchKey++;
var subpipeline = Subpipeline.Create(this.pipeline, $"subpipeline{pair.Key}");
var branch = this.branches[pair.Key] = subpipeline.CreateEmitter <TIn>(subpipeline, $"branch{pair.Key}");
var branchResult = this.transformSelector(pair.Key, branch);
branchResult.PipeTo(this.join.AddInput());
subpipeline.RunAsync(this.pipeline.ReplayDescriptor);
}
this.branches[pair.Key].Post(pair.Value, e.OriginatingTime);
this.activeBranches[pair.Key] = this.keyToBranchMapping[pair.Key];
}
foreach (var branch in this.branches.ToArray())
{
if (this.branchTerminationPolicy(branch.Key, message))
{
(branch.Value.Pipeline as Subpipeline).Suspend();
this.branches.Remove(branch.Key);
}
}
this.activeBranchesEmitter.Post(this.activeBranches, e.OriginatingTime);
}
/// <summary>
/// Initializes a new instance of the <see cref="ParallelFixedLength{TIn, TOut}"/> class.
/// </summary>
/// <param name="pipeline">Pipeline to which this component belongs.</param>
/// <param name="vectorSize">Vector size.</param>
/// <param name="action">Action to apply to output producers.</param>
public ParallelFixedLength(Pipeline pipeline, int vectorSize, Action <int, IProducer <TIn> > action)
{
this.In = pipeline.CreateReceiver <TIn[]>(this, this.Receive, nameof(this.In));
this.branches = new Emitter <TIn> [vectorSize];
for (int i = 0; i < vectorSize; i++)
{
var subpipeline = Subpipeline.Create(pipeline, $"subpipeline{i}");
this.branches[i] = subpipeline.CreateEmitter <TIn>(subpipeline, $"branch{i}");
action(i, this.branches[i]);
}
}
public void Subpipelines()
{
using (var p = Pipeline.Create("root"))
{
using (var s = Subpipeline.Create(p, "sub"))
{
// add to sub-pipeline
var seq = Generators.Sequence(s, new[] { 1, 2, 3 }).ToObservable().ToListObservable();
p.Run(); // run parent pipeline
Assert.IsTrue(Enumerable.SequenceEqual(new int[] { 1, 2, 3 }, seq.AsEnumerable()));
}
}
}
/// <summary>
/// Initializes a new instance of the <see cref="ParallelFixedLength{TIn, TOut}"/> class.
/// </summary>
/// <param name="pipeline">The pipeline to add the component to.</param>
/// <param name="vectorSize">Vector size.</param>
/// <param name="action">Action to apply to output producers.</param>
/// <param name="name">Name for this component (defaults to ParallelFixedLength).</param>
/// <param name="defaultDeliveryPolicy">Pipeline-level default delivery policy to be used by this component (defaults to <see cref="DeliveryPolicy.Unlimited"/> if unspecified).</param>
public ParallelFixedLength(Pipeline pipeline, int vectorSize, Action <int, IProducer <TIn> > action, string name = null, DeliveryPolicy defaultDeliveryPolicy = null)
: base(pipeline, name ?? nameof(ParallelFixedLength <TIn, TOut>), defaultDeliveryPolicy)
{
this.inConnector = this.CreateInputConnectorFrom <TIn[]>(pipeline, nameof(this.inConnector));
this.splitter = this.CreateReceiver <TIn[]>(this, this.Receive, nameof(this.splitter));
this.inConnector.PipeTo(this.splitter);
this.branches = new Emitter <TIn> [vectorSize];
for (int i = 0; i < vectorSize; i++)
{
var subpipeline = Subpipeline.Create(pipeline, $"subpipeline{i}");
var connector = new Connector <TIn>(pipeline, subpipeline, $"connector{i}");
this.branches[i] = pipeline.CreateEmitter <TIn>(this, $"branch{i}");
this.branches[i].PipeTo(connector);
action(i, connector.Out);
}
}
/// <summary>
/// Initializes a new instance of the <see cref="Parallel{TIn, TOut}"/> class.
/// </summary>
/// <param name="pipeline">Pipeline to which this component belongs.</param>
/// <param name="vectorSize">Vector size.</param>
/// <param name="transformSelector">Function mapping keyed input producers to output producers.</param>
/// <param name="joinOrDefault">Whether to do an "...OrDefault" join.</param>
public Parallel(Pipeline pipeline, int vectorSize, Func <int, IProducer <TIn>, IProducer <TOut> > transformSelector, bool joinOrDefault)
{
this.input = pipeline.CreateReceiver <TIn[]>(this, this.Receive, nameof(this.In));
this.branches = new Emitter <TIn> [vectorSize];
var branchResults = new IProducer <TOut> [vectorSize];
for (int i = 0; i < vectorSize; i++)
{
var subpipeline = Subpipeline.Create(pipeline, $"subpipeline{i}");
this.branches[i] = subpipeline.CreateEmitter <TIn>(subpipeline, $"branch{i}");
branchResults[i] = transformSelector(i, this.branches[i]);
}
var interpolator = joinOrDefault ? Match.ExactOrDefault <TOut>() : Match.Exact <TOut>();
this.join = Operators.Join(branchResults, interpolator, pipeline);
}
/// <summary>
/// Initializes a new instance of the <see cref="ParallelFixedLength{TIn, TOut}"/> class.
/// </summary>
/// <param name="pipeline">Pipeline to which this component belongs.</param>
/// <param name="vectorSize">Vector size.</param>
/// <param name="transform">Function mapping keyed input producers to output producers.</param>
/// <param name="joinOrDefault">Whether to do an "...OrDefault" join.</param>
public ParallelFixedLength(Pipeline pipeline, int vectorSize, Func <int, IProducer <TIn>, IProducer <TOut> > transform, bool joinOrDefault)
{
this.In = pipeline.CreateReceiver <TIn[]>(this, this.Receive, nameof(this.In));
this.branches = new Emitter <TIn> [vectorSize];
var branchResults = new IProducer <TOut> [vectorSize];
for (int i = 0; i < vectorSize; i++)
{
var subpipeline = Subpipeline.Create(pipeline, $"subpipeline{i}");
var connectorIn = new Connector <TIn>(pipeline, subpipeline, $"connectorIn{i}");
var connectorOut = new Connector <TOut>(subpipeline, pipeline, $"connectorOut{i}");
this.branches[i] = pipeline.CreateEmitter <TIn>(this, $"branch{i}");
this.branches[i].PipeTo(connectorIn);
transform(i, connectorIn.Out).PipeTo(connectorOut.In);
branchResults[i] = connectorOut;
}
var interpolator = joinOrDefault ? Match.ExactOrDefault <TOut>() : Match.Exact <TOut>();
this.join = Operators.Join(branchResults, interpolator, pipeline: pipeline);
}
public void SubscriptionAcrossPipelinesShouldThrow()
{
var exceptionThrown = false;
try
{
using (var p = Pipeline.Create())
{
var s = Subpipeline.Create(p, "subpipeline");
var emitter = p.CreateEmitter<int>(this, "pipelineEmitter");
var receiver = s.CreateReceiver<int>(this, x => emitter.Post(x, p.GetCurrentTime()), "subpipelineReceiver");
Generators.Return(p, 123).PipeTo(receiver);
p.Run();
}
}
catch (InvalidOperationException ex)
{
exceptionThrown = true;
Assert.IsTrue(ex.Message.StartsWith("Receiver cannot subscribe to an emitter from a different pipeline"));
}
Assert.IsTrue(exceptionThrown);
}
public void ExceptionFromSubpipelineRunHandler()
{
using (var p = Pipeline.Create("pipeline"))
{
var sub = new Subpipeline(p, "subpipeline");
var generator = Generators.Sequence(p, 1, x => x + 1, 100, TimeSpan.FromTicks(1));
sub.PipelineRun += (s, e) =>
{
int x = 0;
x /= 0; // trigger an exception
};
try
{
p.Run();
}
catch (AggregateException e)
{
// exceptions from within the pipeline (including subpipelines) are wrapped in an AggregateException
throw e.InnerException;
}
}
}
public void PostAcrossPipelinesShouldThrow()
{
var exceptionThrown = false;
try
{
using (var p = Pipeline.Create())
{
var s = Subpipeline.Create(p, "subpipeline");
var emitter = p.CreateEmitter <int>(this, "pipelineEmitter");
var receiver = s.CreateReceiver <int>(this, x => emitter.Post(x, p.GetCurrentTime()), "subpipelineReceiver");
Generators.Return(p, 123).PipeTo(receiver);
p.Run(enableExceptionHandling: true);
}
}
catch (AggregateException ex)
{
exceptionThrown = true;
Assert.AreEqual <int>(1, ex.InnerExceptions.Count);
Assert.IsTrue(ex.InnerExceptions[0].Message.StartsWith("Emitter created in one pipeline unexpectedly received post from a receiver in another pipeline"));
}
Assert.IsTrue(exceptionThrown);
}
public void ReadFromMultipleSubpipelines()
{
// This test ensures that stores in subpipelines correctly propose their replay intervals,
// and that this is properly accounted for in the parent pipeline at runtime.
// first, create multiple test stores to read from
var count = 100;
var name0 = nameof(this.ReadFromMultipleSubpipelines) + "_0";
var name1 = nameof(this.ReadFromMultipleSubpipelines) + "_1";
var name2 = nameof(this.ReadFromMultipleSubpipelines) + "_2";
using (var p = Pipeline.Create("write0"))
{
var writeStore = Store.Create(p, name0, this.path);
var seq = Generators.Sequence(p, 1, i => i + 1, count, TimeSpan.FromTicks(1));
seq.Write("seq0", writeStore);
p.Run();
}
using (var p = Pipeline.Create("write1"))
{
var writeStore = Store.Create(p, name1, this.path);
var seq = Generators.Sequence(p, 1, i => i + 1, count, TimeSpan.FromTicks(1));
seq.Write("seq1", writeStore);
p.Run();
}
using (var p = Pipeline.Create("write2"))
{
var writeStore = Store.Create(p, name2, this.path);
var seq = Generators.Sequence(p, 1, i => i + 1, count, TimeSpan.FromTicks(1));
seq.Write("seq2", writeStore);
p.Run();
}
double sum0 = 0;
double sum1 = 0;
double sum2 = 0;
// now open and replay the streams in parent/sub-pipelines and verify
using (var p = Pipeline.Create("parent"))
{
// read store0 in parent pipeline
var store0 = Store.Open(p, name0, this.path);
var seq0 = store0.OpenStream <int>("seq0");
seq0.Do(s => sum0 = sum0 + s);
// read store1 in sub-pipeline
var sub1 = Subpipeline.Create(p, "sub1");
var store1 = Store.Open(sub1, name1, this.path);
var seq1 = store1.OpenStream <int>("seq1");
seq1.Do(s => sum1 = sum1 + s);
// read store2 in sub-pipeline
var sub2 = Subpipeline.Create(p, "sub2");
var store2 = Store.Open(sub2, name2, this.path);
var seq2 = store2.OpenStream <int>("seq2");
seq1.Do(s => sum2 = sum2 + s);
p.Run();
}
Assert.AreEqual(count * (count + 1) / 2, sum0);
Assert.AreEqual(count * (count + 1) / 2, sum1);
Assert.AreEqual(count * (count + 1) / 2, sum2);
}
public void StartStopOrderingTestWithSubpipelines()
{
// Run test for a range of scheduler thread counts, including 1. This prevents the
// output from appearing in the correct order by fluke due to concurrent scheduling.
for (int maxThreads = Environment.ProcessorCount * 2; maxThreads > 0; maxThreads >>= 1)
{
var cLog = new ConcurrentQueue <string>();
using (var p = Pipeline.Create("P", null, maxThreads))
using (var q = Subpipeline.Create(p, "Q"))
using (var r = Subpipeline.Create(q, "R"))
{
var a = new TestSourceComponent(p, "A", cLog);
var b = new TestSourceComponent(p, "B", cLog);
var c = new TestSourceComponent(p, "C", cLog);
var d = new TestSourceComponent(q, "D", cLog);
var e = new TestSourceComponent(q, "E", cLog);
var f = new TestSourceComponent(q, "F", cLog);
var g = new TestSourceComponent(r, "G", cLog);
var h = new TestSourceComponent(r, "H", cLog);
var i = new TestSourceComponent(r, "I", cLog);
a.Emitter.Do(m => cLog.Enqueue(m));
b.Emitter.Do(m => cLog.Enqueue(m));
c.Emitter.Do(m => cLog.Enqueue(m));
d.Emitter.Do(m => cLog.Enqueue(m));
e.Emitter.Do(m => cLog.Enqueue(m));
f.Emitter.Do(m => cLog.Enqueue(m));
g.Emitter.Do(m => cLog.Enqueue(m));
h.Emitter.Do(m => cLog.Enqueue(m));
i.Emitter.Do(m => cLog.Enqueue(m));
Generators.Repeat(p, "PGen", 10, TimeSpan.FromTicks(1)).Do(m => cLog.Enqueue(m));
Generators.Repeat(q, "QGen", 10, TimeSpan.FromTicks(1)).Do(m => cLog.Enqueue(m));
Generators.Repeat(r, "RGen", 10, TimeSpan.FromTicks(1)).Do(m => cLog.Enqueue(m));
p.Run();
}
var log = cLog.ToList();
log.ForEach(m => Console.WriteLine(m));
// all components should have started
Assert.IsTrue(log.Contains("AStart"));
Assert.IsTrue(log.Contains("BStart"));
Assert.IsTrue(log.Contains("CStart"));
Assert.IsTrue(log.Contains("DStart"));
Assert.IsTrue(log.Contains("EStart"));
Assert.IsTrue(log.Contains("FStart"));
Assert.IsTrue(log.Contains("GStart"));
Assert.IsTrue(log.Contains("HStart"));
Assert.IsTrue(log.Contains("IStart"));
int pLatestStartIndex = new[] { log.IndexOf("AStart"), log.IndexOf("BStart"), log.IndexOf("CStart") }.Max();
int qLatestStartIndex = new[] { log.IndexOf("DStart"), log.IndexOf("EStart"), log.IndexOf("FStart") }.Max();
int rLatestStartIndex = new[] { log.IndexOf("GStart"), log.IndexOf("HStart"), log.IndexOf("IStart") }.Max();
// messages should only be delivered after all components have started within their respective pipelines
Assert.IsTrue(pLatestStartIndex < log.IndexOf("APostFromStart"));
Assert.IsTrue(pLatestStartIndex < log.IndexOf("BPostFromStart"));
Assert.IsTrue(pLatestStartIndex < log.IndexOf("CPostFromStart"));
Assert.IsTrue(pLatestStartIndex < log.IndexOf("PGen"));
Assert.IsTrue(qLatestStartIndex < log.IndexOf("DPostFromStart"));
Assert.IsTrue(qLatestStartIndex < log.IndexOf("EPostFromStart"));
Assert.IsTrue(qLatestStartIndex < log.IndexOf("FPostFromStart"));
Assert.IsTrue(qLatestStartIndex < log.IndexOf("QGen"));
Assert.IsTrue(rLatestStartIndex < log.IndexOf("GPostFromStart"));
Assert.IsTrue(rLatestStartIndex < log.IndexOf("HPostFromStart"));
Assert.IsTrue(rLatestStartIndex < log.IndexOf("IPostFromStart"));
Assert.IsTrue(rLatestStartIndex < log.IndexOf("RGen"));
// all components should be stopped after they were started
Assert.IsTrue(log.IndexOf("AStart") < log.IndexOf("AStop"));
Assert.IsTrue(log.IndexOf("BStart") < log.IndexOf("BStop"));
Assert.IsTrue(log.IndexOf("CStart") < log.IndexOf("CStop"));
Assert.IsTrue(log.IndexOf("DStart") < log.IndexOf("DStop"));
Assert.IsTrue(log.IndexOf("EStart") < log.IndexOf("EStop"));
Assert.IsTrue(log.IndexOf("FStart") < log.IndexOf("FStop"));
Assert.IsTrue(log.IndexOf("GStart") < log.IndexOf("GStop"));
Assert.IsTrue(log.IndexOf("HStart") < log.IndexOf("HStop"));
Assert.IsTrue(log.IndexOf("IStart") < log.IndexOf("IStop"));
}
}
